From 5b776a1d626e43ac6ecf173821b688e57ba97bcc Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 18 Nov 2026 15:38:23 +0100 Subject: [PATCH 1/2] Add the ASYNC_CALL and ASYNC_CALL_KW instruction families The compiler emits ASYNC_CALL (or ASYNC_CALL_KW) instead of CALL (or CALL_KW) for a call whose result is awaited at once: `await f(...)`, the __aenter__ and __aexit__ calls of `async with`, and the call of an async comprehension. The instruction is always followed by GET_AWAITABLE, PUSH_NULL, LOAD_COMMON_CONSTANT None and SEND. Unspecialized, it does the same as CALL. When the callee is a coroutine function, the specializer rewrites it to ASYNC_CALL_PY_EXACT_ARGS, ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, ASYNC_CALL_PY_GENERAL, ASYNC_CALL_BOUND_METHOD_GENERAL, ASYNC_CALL_KW_PY or ASYNC_CALL_KW_BOUND_METHOD. These merge the call with the first execution of the instructions that follow it: the new coroutine is started in place, without being returned to the caller first. They reuse the uops of the matching CALL_* specialization, with the new _ASYNC_CALL_GEN_FRAME uop in place of _SAVE_RETURN_OFFSET. RETURN_GENERATOR does nothing in a frame that already belongs to a generator, so that coroutine functions whose prologue has COPY_FREE_VARS or MAKE_CELL use the fast path too. Calls to callables that are not coroutine functions are not specialized: the ASYNC_CALL families hold only the coroutine specializations. Co-Authored-By: Claude Fable 5.1 --- Doc/library/dis.rst | 18 + Include/internal/pycore_ceval.h | 1 + Include/internal/pycore_code.h | 28 + Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 100 +- Include/internal/pycore_uop_ids.h | 14 +- Include/internal/pycore_uop_metadata.h | 50 +- Include/opcode_ids.h | 378 ++--- InternalDocs/generators.md | 16 + Lib/_opcode_metadata.py | 388 ++--- Lib/opcode.py | 8 + Lib/test/test_capi/test_opt.py | 43 + Lib/test/test_compile.py | 31 + Lib/test/test_coroutines.py | 354 +++++ Lib/test/test_dis.py | 4 +- Lib/test/test_monitoring.py | 84 + Lib/test/test_opcache.py | 159 ++ Lib/test/test_sys_settrace.py | 21 + Modules/_testinternalcapi/test_cases.c.h | 1740 +++++++++++++++++++-- Modules/_testinternalcapi/test_targets.h | 90 +- Objects/frameobject.c | 4 +- Objects/genobject.c | 9 +- Programs/test_frozenmain.h | 26 +- Python/bytecodes.c | 247 ++- Python/ceval_macros.h | 26 + Python/codegen.c | 62 +- Python/executor_cases.c.h | 348 ++++- Python/generated_cases.c.h | 1740 +++++++++++++++++++-- Python/instrumentation.c | 10 + Python/opcode_targets.h | 90 +- Python/optimizer.c | 12 +- Python/optimizer_bytecodes.c | 12 + Python/optimizer_cases.c.h | 23 + Python/pystats.c | 2 + Python/record_functions.c.h | 14 + Python/specialize.c | 141 +- Tools/cases_generator/analyzer.py | 3 + 37 files changed, 5496 insertions(+), 803 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 5a93a328d487289..0383c79979ebe77 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1630,6 +1630,24 @@ iterations of the loop. .. versionadded:: 3.13 +.. opcode:: ASYNC_CALL (argc) + + Like :opcode:`CALL`, for a call whose result is awaited at once, as in + ``await f(...)``, ``async with`` and asynchronous comprehensions. + It is always followed by :opcode:`GET_AWAITABLE`, :opcode:`PUSH_NULL`, + :opcode:`LOAD_COMMON_CONSTANT` (``None``) and :opcode:`SEND`. + + .. versionadded:: 3.16 + + +.. opcode:: ASYNC_CALL_KW (argc) + + Like :opcode:`CALL_KW`, for a call whose result is awaited at once. + See :opcode:`ASYNC_CALL`. + + .. versionadded:: 3.16 + + .. opcode:: CALL_FUNCTION_EX (flags) Calls a callable object with variable set of positional and keyword diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a0fea389b6502c0..f60605e609c3bee 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -267,6 +267,7 @@ extern int _PyEval_EnsureBuiltinsWithModule( PyObject **p_builtins); PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); +PyAPI_FUNC(PyObject *)_Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index aeb2ad06ad07cb7..3b145089761f930 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -9,6 +9,8 @@ extern "C" { #endif #include "pycore_backoff.h" // _Py_BackoffCounter +#include "pycore_opcode_utils.h" // CONSTANT_NONE +#include "opcode_ids.h" // GET_AWAITABLE #include "pycore_structs.h" // _Py_CODEUNIT #include "pycore_tstate.h" // _PyThreadStateImpl @@ -128,6 +130,8 @@ typedef struct { #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) #define INLINE_CACHE_ENTRIES_CALL_KW CACHE_ENTRIES(_PyCallCache) +#define INLINE_CACHE_ENTRIES_ASYNC_CALL CACHE_ENTRIES(_PyCallCache) +#define INLINE_CACHE_ENTRIES_ASYNC_CALL_KW CACHE_ENTRIES(_PyCallCache) typedef struct { _Py_BackoffCounter counter; @@ -317,6 +321,10 @@ PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _PyStackRef self_or_n _Py_CODEUNIT *instr, int nargs); PyAPI_FUNC(void) _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); +PyAPI_FUNC(void) _Py_Specialize_AsyncCall(_PyStackRef callable, _Py_CODEUNIT *instr, + int nargs); +PyAPI_FUNC(void) _Py_Specialize_AsyncCallKw(_PyStackRef callable, _Py_CODEUNIT *instr, + int nargs); PyAPI_FUNC(void) _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, int oparg, _PyStackRef *locals); PyAPI_FUNC(void) _Py_Specialize_CompareOp(_PyStackRef lhs, _PyStackRef rhs, @@ -542,6 +550,26 @@ PyAPI_FUNC(int) _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(_Py_CODEUNIT) _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); +PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; + +/* The SEND that awaits the result of the ASYNC_CALL or ASYNC_CALL_KW whose + * cache ends at `tail`, or NULL if the instructions after the call are not + * the ones the compiler emits for an await. */ +static inline _Py_CODEUNIT * +_PyCode_AwaitedCallSend(_Py_CODEUNIT *tail) +{ + /* None of the first three instructions can end the code. */ + if (tail[0].op.code == GET_AWAITABLE + && tail[1].op.code == PUSH_NULL + && tail[2].op.code == LOAD_COMMON_CONSTANT + && tail[2].op.arg == CONSTANT_NONE + && _PyOpcode_Deopt[tail[3].op.code] == SEND) + { + return &tail[3]; + } + return NULL; +} + extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); extern PyObject *_PyInstrumentation_BranchesIterator(PyCodeObject *code); diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 15d3ad6f3a336b7..587fe8674558905 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -304,6 +304,7 @@ Known values: Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) Python 3.16a1 3706 (Add INTRINSIC_MATCH_CLASS_ISINSTANCE) + Python 3.16a2 3707 (Add ASYNC_CALL and ASYNC_CALL_KW) Python 3.17 will start with 3750 @@ -313,7 +314,7 @@ Known values: */ -#define PYC_MAGIC_NUMBER 3706 +#define PYC_MAGIC_NUMBER 3707 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 04b826ed09144e7..e301dff3cc6fd14 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -38,6 +38,22 @@ int _PyOpcode_num_popped(int opcode, int oparg) { switch(opcode) { case ANNOTATIONS_PLACEHOLDER: return 0; + case ASYNC_CALL: + return 2 + oparg; + case ASYNC_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case ASYNC_CALL_BOUND_METHOD_GENERAL: + return 2 + oparg; + case ASYNC_CALL_KW: + return 3 + oparg; + case ASYNC_CALL_KW_BOUND_METHOD: + return 3 + oparg; + case ASYNC_CALL_KW_PY: + return 3 + oparg; + case ASYNC_CALL_PY_EXACT_ARGS: + return 2 + oparg; + case ASYNC_CALL_PY_GENERAL: + return 2 + oparg; case BINARY_OP: return 2; case BINARY_OP_ADD_FLOAT: @@ -234,6 +250,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case IMPORT_NAME: return 2; + case INSTRUMENTED_ASYNC_CALL: + return 2 + oparg; + case INSTRUMENTED_ASYNC_CALL_KW: + return 3 + oparg; case INSTRUMENTED_CALL: return 2 + oparg; case INSTRUMENTED_CALL_FUNCTION_EX: @@ -533,6 +553,22 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { switch(opcode) { case ANNOTATIONS_PLACEHOLDER: return 0; + case ASYNC_CALL: + return 1; + case ASYNC_CALL_BOUND_METHOD_EXACT_ARGS: + return 2; + case ASYNC_CALL_BOUND_METHOD_GENERAL: + return 2; + case ASYNC_CALL_KW: + return 1; + case ASYNC_CALL_KW_BOUND_METHOD: + return 2; + case ASYNC_CALL_KW_PY: + return 2; + case ASYNC_CALL_PY_EXACT_ARGS: + return 2; + case ASYNC_CALL_PY_GENERAL: + return 2; case BINARY_OP: return 1; case BINARY_OP_ADD_FLOAT: @@ -729,6 +765,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2; case IMPORT_NAME: return 1; + case INSTRUMENTED_ASYNC_CALL: + return 1; + case INSTRUMENTED_ASYNC_CALL_KW: + return 1; case INSTRUMENTED_CALL: return 1; case INSTRUMENTED_CALL_FUNCTION_EX: @@ -1100,6 +1140,14 @@ struct opcode_metadata { PyAPI_DATA(const struct opcode_metadata) _PyOpcode_opcode_metadata[267]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { + [ASYNC_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [ASYNC_CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, @@ -1198,6 +1246,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_ASYNC_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_ASYNC_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, @@ -1345,7 +1395,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { }; #endif -#define MAX_UOP_PER_EXPANSION 11 +#define MAX_UOP_PER_EXPANSION 12 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1355,6 +1405,12 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 12, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = { .nuops = 9, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_KW_BOUND_METHOD] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_KW_PY] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_PY_EXACT_ARGS] = { .nuops = 10, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_PY_GENERAL] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [BINARY_OP] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _BINARY_OP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [BINARY_OP_ADD_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_ADD_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, @@ -1553,6 +1609,14 @@ PyAPI_DATA(const char) *_PyOpcode_OpName[267]; #ifdef NEED_OPCODE_METADATA const char *_PyOpcode_OpName[267] = { [ANNOTATIONS_PLACEHOLDER] = "ANNOTATIONS_PLACEHOLDER", + [ASYNC_CALL] = "ASYNC_CALL", + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = "ASYNC_CALL_BOUND_METHOD_EXACT_ARGS", + [ASYNC_CALL_BOUND_METHOD_GENERAL] = "ASYNC_CALL_BOUND_METHOD_GENERAL", + [ASYNC_CALL_KW] = "ASYNC_CALL_KW", + [ASYNC_CALL_KW_BOUND_METHOD] = "ASYNC_CALL_KW_BOUND_METHOD", + [ASYNC_CALL_KW_PY] = "ASYNC_CALL_KW_PY", + [ASYNC_CALL_PY_EXACT_ARGS] = "ASYNC_CALL_PY_EXACT_ARGS", + [ASYNC_CALL_PY_GENERAL] = "ASYNC_CALL_PY_GENERAL", [BINARY_OP] = "BINARY_OP", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", @@ -1651,6 +1715,8 @@ const char *_PyOpcode_OpName[267] = { [GET_LEN] = "GET_LEN", [IMPORT_FROM] = "IMPORT_FROM", [IMPORT_NAME] = "IMPORT_NAME", + [INSTRUMENTED_ASYNC_CALL] = "INSTRUMENTED_ASYNC_CALL", + [INSTRUMENTED_ASYNC_CALL_KW] = "INSTRUMENTED_ASYNC_CALL_KW", [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", [INSTRUMENTED_CALL_KW] = "INSTRUMENTED_CALL_KW", @@ -1819,7 +1885,9 @@ const uint8_t _PyOpcode_Caches[256] = { [POP_JUMP_IF_NOT_NONE] = 1, [GET_ITER] = 1, [CALL] = 3, + [ASYNC_CALL] = 3, [CALL_KW] = 3, + [ASYNC_CALL_KW] = 3, [CALL_FUNCTION_EX] = 1, [BINARY_OP] = 5, }; @@ -1828,8 +1896,6 @@ const uint8_t _PyOpcode_Caches[256] = { PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { - [117] = 117, - [118] = 118, [119] = 119, [120] = 120, [121] = 121, @@ -1839,20 +1905,20 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [219] = 219, - [220] = 220, - [221] = 221, - [222] = 222, - [223] = 223, - [224] = 224, [225] = 225, [226] = 226, [227] = 227, [228] = 228, [229] = 229, [230] = 230, - [231] = 231, - [232] = 232, + [ASYNC_CALL] = ASYNC_CALL, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = ASYNC_CALL, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = ASYNC_CALL, + [ASYNC_CALL_KW] = ASYNC_CALL_KW, + [ASYNC_CALL_KW_BOUND_METHOD] = ASYNC_CALL_KW, + [ASYNC_CALL_KW_PY] = ASYNC_CALL_KW, + [ASYNC_CALL_PY_EXACT_ARGS] = ASYNC_CALL, + [ASYNC_CALL_PY_GENERAL] = ASYNC_CALL, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -1951,6 +2017,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [GET_LEN] = GET_LEN, [IMPORT_FROM] = IMPORT_FROM, [IMPORT_NAME] = IMPORT_NAME, + [INSTRUMENTED_ASYNC_CALL] = INSTRUMENTED_ASYNC_CALL, + [INSTRUMENTED_ASYNC_CALL_KW] = INSTRUMENTED_ASYNC_CALL_KW, [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_CALL_KW] = INSTRUMENTED_CALL_KW, @@ -2089,8 +2157,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 117: \ - case 118: \ case 119: \ case 120: \ case 121: \ @@ -2100,20 +2166,12 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 219: \ - case 220: \ - case 221: \ - case 222: \ - case 223: \ - case 224: \ case 225: \ case 226: \ case 227: \ case 228: \ case 229: \ case 230: \ - case 231: \ - case 232: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ba6e0252024a6df..a04629e5dd0ed2f 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -13,6 +13,7 @@ enum { _EXIT_TRACE = 300, _SET_IP, _ALLOCATE_OBJECT, + _ASYNC_CALL_GEN_FRAME, _BINARY_OP, _BINARY_OP_ADD_FLOAT, _BINARY_OP_ADD_FLOAT_INPLACE, @@ -128,6 +129,7 @@ enum { _GET_ITER, _GET_ITER_TRAD, _GUARD_3OS_ASYNC_GEN_ASEND, + _GUARD_ASYNC_CALL_TAIL, _GUARD_BINARY_OP_EXTEND, _GUARD_BINARY_OP_EXTEND_LHS, _GUARD_BINARY_OP_EXTEND_RHS, @@ -366,7 +368,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 654 +#define MAX_UOP_ID 656 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -443,6 +445,7 @@ enum { enum { _ALLOCATE_OBJECT_r00 = MAX_UOP_ID + 1, + _ASYNC_CALL_GEN_FRAME_r13, _BINARY_OP_r23, _BINARY_OP_ADD_FLOAT_r03, _BINARY_OP_ADD_FLOAT_r13, @@ -698,6 +701,10 @@ enum { _GUARD_3OS_ASYNC_GEN_ASEND_r13, _GUARD_3OS_ASYNC_GEN_ASEND_r23, _GUARD_3OS_ASYNC_GEN_ASEND_r33, + _GUARD_ASYNC_CALL_TAIL_r00, + _GUARD_ASYNC_CALL_TAIL_r11, + _GUARD_ASYNC_CALL_TAIL_r22, + _GUARD_ASYNC_CALL_TAIL_r33, _GUARD_BINARY_OP_EXTEND_r22, _GUARD_BINARY_OP_EXTEND_LHS_r02, _GUARD_BINARY_OP_EXTEND_LHS_r12, @@ -1309,6 +1316,9 @@ enum { _SAVE_RETURN_OFFSET_r33, _SEND_ASYNC_GEN_r33, _SEND_ASYNC_GEN_TIER_TWO_r33, + _SEND_GEN_FRAME_r03, + _SEND_GEN_FRAME_r13, + _SEND_GEN_FRAME_r23, _SEND_GEN_FRAME_r33, _SEND_VIRTUAL_r33, _SEND_VIRTUAL_TIER_TWO_r03, @@ -1444,7 +1454,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1655 +#define MAX_UOP_REGS_ID 1665 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0cd82c53aa932ae..53cf9ed65fa47cf 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -352,6 +352,8 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_ASYNC_CALL_TAIL] = HAS_DEOPT_FLAG, + [_ASYNC_CALL_GEN_FRAME] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -1645,11 +1647,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_SEND_GEN_FRAME] = { - .best = { 3, 3, 3, 3 }, + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 3, 0, _SEND_GEN_FRAME_r03 }, + { 3, 1, _SEND_GEN_FRAME_r13 }, + { 3, 2, _SEND_GEN_FRAME_r23 }, { 3, 3, _SEND_GEN_FRAME_r33 }, }, }, @@ -3300,6 +3302,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_ASYNC_CALL_TAIL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_ASYNC_CALL_TAIL_r00 }, + { 1, 1, _GUARD_ASYNC_CALL_TAIL_r11 }, + { 2, 2, _GUARD_ASYNC_CALL_TAIL_r22 }, + { 3, 3, _GUARD_ASYNC_CALL_TAIL_r33 }, + }, + }, + [_ASYNC_CALL_GEN_FRAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 3, 3, _ASYNC_CALL_GEN_FRAME_r13 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_MAYBE_EXPAND_METHOD_KW] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -4285,6 +4305,9 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GET_AITER_r11] = _GET_AITER, [_GET_ANEXT_r12] = _GET_ANEXT, [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_SEND_GEN_FRAME_r03] = _SEND_GEN_FRAME, + [_SEND_GEN_FRAME_r13] = _SEND_GEN_FRAME, + [_SEND_GEN_FRAME_r23] = _SEND_GEN_FRAME, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, @@ -4642,6 +4665,11 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, [_CALL_METHOD_DESCRIPTOR_FAST_r00] = _CALL_METHOD_DESCRIPTOR_FAST, [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_INLINE, + [_GUARD_ASYNC_CALL_TAIL_r00] = _GUARD_ASYNC_CALL_TAIL, + [_GUARD_ASYNC_CALL_TAIL_r11] = _GUARD_ASYNC_CALL_TAIL, + [_GUARD_ASYNC_CALL_TAIL_r22] = _GUARD_ASYNC_CALL_TAIL, + [_GUARD_ASYNC_CALL_TAIL_r33] = _GUARD_ASYNC_CALL_TAIL, + [_ASYNC_CALL_GEN_FRAME_r13] = _ASYNC_CALL_GEN_FRAME, [_MAYBE_EXPAND_METHOD_KW_r11] = _MAYBE_EXPAND_METHOD_KW, [_PY_FRAME_KW_r11] = _PY_FRAME_KW, [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, @@ -4867,6 +4895,8 @@ const uint16_t _PyUop_SpillsAndReloads[4][4] = { const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_ALLOCATE_OBJECT] = "_ALLOCATE_OBJECT", [_ALLOCATE_OBJECT_r00] = "_ALLOCATE_OBJECT_r00", + [_ASYNC_CALL_GEN_FRAME] = "_ASYNC_CALL_GEN_FRAME", + [_ASYNC_CALL_GEN_FRAME_r13] = "_ASYNC_CALL_GEN_FRAME_r13", [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_r23] = "_BINARY_OP_r23", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", @@ -5250,6 +5280,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = "_GUARD_3OS_ASYNC_GEN_ASEND_r13", [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = "_GUARD_3OS_ASYNC_GEN_ASEND_r23", [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = "_GUARD_3OS_ASYNC_GEN_ASEND_r33", + [_GUARD_ASYNC_CALL_TAIL] = "_GUARD_ASYNC_CALL_TAIL", + [_GUARD_ASYNC_CALL_TAIL_r00] = "_GUARD_ASYNC_CALL_TAIL_r00", + [_GUARD_ASYNC_CALL_TAIL_r11] = "_GUARD_ASYNC_CALL_TAIL_r11", + [_GUARD_ASYNC_CALL_TAIL_r22] = "_GUARD_ASYNC_CALL_TAIL_r22", + [_GUARD_ASYNC_CALL_TAIL_r33] = "_GUARD_ASYNC_CALL_TAIL_r33", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", [_GUARD_BINARY_OP_EXTEND_LHS] = "_GUARD_BINARY_OP_EXTEND_LHS", @@ -6011,6 +6046,9 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_SEND_ASYNC_GEN_TIER_TWO] = "_SEND_ASYNC_GEN_TIER_TWO", [_SEND_ASYNC_GEN_TIER_TWO_r33] = "_SEND_ASYNC_GEN_TIER_TWO_r33", [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", + [_SEND_GEN_FRAME_r03] = "_SEND_GEN_FRAME_r03", + [_SEND_GEN_FRAME_r13] = "_SEND_GEN_FRAME_r13", + [_SEND_GEN_FRAME_r23] = "_SEND_GEN_FRAME_r23", [_SEND_GEN_FRAME_r33] = "_SEND_GEN_FRAME_r33", [_SEND_VIRTUAL_TIER_TWO] = "_SEND_VIRTUAL_TIER_TWO", [_SEND_VIRTUAL_TIER_TWO_r03] = "_SEND_VIRTUAL_TIER_TWO_r03", @@ -6832,6 +6870,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_METHOD_DESCRIPTOR_FAST_INLINE: return 0; + case _GUARD_ASYNC_CALL_TAIL: + return 0; + case _ASYNC_CALL_GEN_FRAME: + return 1; case _MAYBE_EXPAND_METHOD_KW: return 0; case _PY_FRAME_KW: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 11342ae451b9f6c..ab0fb90b56703e0 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -52,190 +52,200 @@ extern "C" { #define UNARY_NEGATIVE 39 #define UNARY_NOT 40 #define WITH_EXCEPT_START 41 -#define BINARY_OP 42 -#define BUILD_INTERPOLATION 43 -#define BUILD_LIST 44 -#define BUILD_MAP 45 -#define BUILD_SET 46 -#define BUILD_SLICE 47 -#define BUILD_STRING 48 -#define BUILD_TUPLE 49 -#define CALL 50 -#define CALL_INTRINSIC_1 51 -#define CALL_INTRINSIC_2 52 -#define CALL_KW 53 -#define COMPARE_OP 54 -#define CONTAINS_OP 55 -#define CONVERT_VALUE 56 -#define COPY 57 -#define COPY_FREE_VARS 58 -#define DELETE_DEREF 59 -#define DELETE_FAST 60 -#define DICT_MERGE 61 -#define DICT_UPDATE 62 -#define END_ASYNC_FOR 63 -#define EXTENDED_ARG 64 -#define FOR_ITER 65 -#define GET_AWAITABLE 66 -#define GET_ITER 67 -#define IMPORT_FROM 68 -#define IMPORT_NAME 69 -#define IS_OP 70 -#define JUMP_BACKWARD 71 -#define JUMP_BACKWARD_NO_INTERRUPT 72 -#define JUMP_FORWARD 73 -#define LIST_APPEND 74 -#define LIST_EXTEND 75 -#define LOAD_ATTR 76 -#define LOAD_COMMON_CONSTANT 77 -#define LOAD_CONST 78 -#define LOAD_DEREF 79 -#define LOAD_FAST 80 -#define LOAD_FAST_AND_CLEAR 81 -#define LOAD_FAST_BORROW 82 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 83 -#define LOAD_FAST_CHECK 84 -#define LOAD_FAST_LOAD_FAST 85 -#define LOAD_FROM_DICT_OR_DEREF 86 -#define LOAD_FROM_DICT_OR_GLOBALS 87 -#define LOAD_GLOBAL 88 -#define LOAD_NAME 89 -#define LOAD_SMALL_INT 90 -#define LOAD_SPECIAL 91 -#define LOAD_SUPER_ATTR 92 -#define MAKE_CELL 93 -#define MAP_ADD 94 -#define MATCH_CLASS 95 -#define POP_JUMP_IF_FALSE 96 -#define POP_JUMP_IF_NONE 97 -#define POP_JUMP_IF_NOT_NONE 98 -#define POP_JUMP_IF_TRUE 99 -#define RAISE_VARARGS 100 -#define RERAISE 101 -#define SEND 102 -#define SET_ADD 103 -#define SET_FUNCTION_ATTRIBUTE 104 -#define SET_UPDATE 105 -#define STORE_ATTR 106 -#define STORE_DEREF 107 -#define STORE_FAST 108 -#define STORE_FAST_LOAD_FAST 109 -#define STORE_FAST_STORE_FAST 110 -#define STORE_GLOBAL 111 -#define STORE_NAME 112 -#define SWAP 113 -#define UNPACK_EX 114 -#define UNPACK_SEQUENCE 115 -#define YIELD_VALUE 116 +#define ASYNC_CALL 42 +#define ASYNC_CALL_KW 43 +#define BINARY_OP 44 +#define BUILD_INTERPOLATION 45 +#define BUILD_LIST 46 +#define BUILD_MAP 47 +#define BUILD_SET 48 +#define BUILD_SLICE 49 +#define BUILD_STRING 50 +#define BUILD_TUPLE 51 +#define CALL 52 +#define CALL_INTRINSIC_1 53 +#define CALL_INTRINSIC_2 54 +#define CALL_KW 55 +#define COMPARE_OP 56 +#define CONTAINS_OP 57 +#define CONVERT_VALUE 58 +#define COPY 59 +#define COPY_FREE_VARS 60 +#define DELETE_DEREF 61 +#define DELETE_FAST 62 +#define DICT_MERGE 63 +#define DICT_UPDATE 64 +#define END_ASYNC_FOR 65 +#define EXTENDED_ARG 66 +#define FOR_ITER 67 +#define GET_AWAITABLE 68 +#define GET_ITER 69 +#define IMPORT_FROM 70 +#define IMPORT_NAME 71 +#define IS_OP 72 +#define JUMP_BACKWARD 73 +#define JUMP_BACKWARD_NO_INTERRUPT 74 +#define JUMP_FORWARD 75 +#define LIST_APPEND 76 +#define LIST_EXTEND 77 +#define LOAD_ATTR 78 +#define LOAD_COMMON_CONSTANT 79 +#define LOAD_CONST 80 +#define LOAD_DEREF 81 +#define LOAD_FAST 82 +#define LOAD_FAST_AND_CLEAR 83 +#define LOAD_FAST_BORROW 84 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 85 +#define LOAD_FAST_CHECK 86 +#define LOAD_FAST_LOAD_FAST 87 +#define LOAD_FROM_DICT_OR_DEREF 88 +#define LOAD_FROM_DICT_OR_GLOBALS 89 +#define LOAD_GLOBAL 90 +#define LOAD_NAME 91 +#define LOAD_SMALL_INT 92 +#define LOAD_SPECIAL 93 +#define LOAD_SUPER_ATTR 94 +#define MAKE_CELL 95 +#define MAP_ADD 96 +#define MATCH_CLASS 97 +#define POP_JUMP_IF_FALSE 98 +#define POP_JUMP_IF_NONE 99 +#define POP_JUMP_IF_NOT_NONE 100 +#define POP_JUMP_IF_TRUE 101 +#define RAISE_VARARGS 102 +#define RERAISE 103 +#define SEND 104 +#define SET_ADD 105 +#define SET_FUNCTION_ATTRIBUTE 106 +#define SET_UPDATE 107 +#define STORE_ATTR 108 +#define STORE_DEREF 109 +#define STORE_FAST 110 +#define STORE_FAST_LOAD_FAST 111 +#define STORE_FAST_STORE_FAST 112 +#define STORE_GLOBAL 113 +#define STORE_NAME 114 +#define SWAP 115 +#define UNPACK_EX 116 +#define UNPACK_SEQUENCE 117 +#define YIELD_VALUE 118 #define RESUME 128 -#define BINARY_OP_ADD_FLOAT 129 -#define BINARY_OP_ADD_INT 130 -#define BINARY_OP_ADD_UNICODE 131 -#define BINARY_OP_EXTEND 132 -#define BINARY_OP_MULTIPLY_FLOAT 133 -#define BINARY_OP_MULTIPLY_INT 134 -#define BINARY_OP_SUBSCR_DICT 135 -#define BINARY_OP_SUBSCR_GETITEM 136 -#define BINARY_OP_SUBSCR_LIST_INT 137 -#define BINARY_OP_SUBSCR_LIST_SLICE 138 -#define BINARY_OP_SUBSCR_STR_INT 139 -#define BINARY_OP_SUBSCR_TUPLE_INT 140 -#define BINARY_OP_SUBSCR_USTR_INT 141 -#define BINARY_OP_SUBTRACT_FLOAT 142 -#define BINARY_OP_SUBTRACT_INT 143 -#define CALL_ALLOC_AND_ENTER_INIT 144 -#define CALL_BOUND_METHOD_EXACT_ARGS 145 -#define CALL_BOUND_METHOD_GENERAL 146 -#define CALL_BUILTIN_CLASS 147 -#define CALL_BUILTIN_FAST 148 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 149 -#define CALL_BUILTIN_O 150 -#define CALL_EX_NON_PY_GENERAL 151 -#define CALL_EX_PY 152 -#define CALL_ISINSTANCE 153 -#define CALL_KW_BOUND_METHOD 154 -#define CALL_KW_NON_PY 155 -#define CALL_KW_PY 156 -#define CALL_LEN 157 -#define CALL_LIST_APPEND 158 -#define CALL_METHOD_DESCRIPTOR_FAST 159 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 160 -#define CALL_METHOD_DESCRIPTOR_NOARGS 161 -#define CALL_METHOD_DESCRIPTOR_O 162 -#define CALL_NON_PY_GENERAL 163 -#define CALL_PY_EXACT_ARGS 164 -#define CALL_PY_GENERAL 165 -#define CALL_STR_1 166 -#define CALL_TUPLE_1 167 -#define CALL_TYPE_1 168 -#define COMPARE_OP_FLOAT 169 -#define COMPARE_OP_INT 170 -#define COMPARE_OP_STR 171 -#define CONTAINS_OP_DICT 172 -#define CONTAINS_OP_SET 173 -#define FOR_ITER_GEN 174 -#define FOR_ITER_LIST 175 -#define FOR_ITER_RANGE 176 -#define FOR_ITER_TUPLE 177 -#define FOR_ITER_VIRTUAL 178 -#define GET_ITER_SELF 179 -#define GET_ITER_VIRTUAL 180 -#define JUMP_BACKWARD_JIT 181 -#define JUMP_BACKWARD_NO_JIT 182 -#define LOAD_ATTR_CLASS 183 -#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 184 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 185 -#define LOAD_ATTR_INSTANCE_VALUE 186 -#define LOAD_ATTR_METHOD_LAZY_DICT 187 -#define LOAD_ATTR_METHOD_NO_DICT 188 -#define LOAD_ATTR_METHOD_WITH_VALUES 189 -#define LOAD_ATTR_MODULE 190 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 191 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 192 -#define LOAD_ATTR_PROPERTY 193 -#define LOAD_ATTR_SLOT 194 -#define LOAD_ATTR_WITH_HINT 195 -#define LOAD_GLOBAL_BUILTIN 196 -#define LOAD_GLOBAL_MODULE 197 -#define LOAD_SUPER_ATTR_ATTR 198 -#define LOAD_SUPER_ATTR_METHOD 199 -#define RESUME_CHECK 200 -#define RESUME_CHECK_JIT 201 -#define SEND_ASYNC_GEN 202 -#define SEND_GEN 203 -#define SEND_VIRTUAL 204 -#define STORE_ATTR_INSTANCE_VALUE 205 -#define STORE_ATTR_SLOT 206 -#define STORE_ATTR_WITH_HINT 207 -#define STORE_SUBSCR_DICT 208 -#define STORE_SUBSCR_LIST_INT 209 -#define TO_BOOL_ALWAYS_TRUE 210 -#define TO_BOOL_BOOL 211 -#define TO_BOOL_INT 212 -#define TO_BOOL_LIST 213 -#define TO_BOOL_NONE 214 -#define TO_BOOL_STR 215 -#define UNPACK_SEQUENCE_LIST 216 -#define UNPACK_SEQUENCE_TUPLE 217 -#define UNPACK_SEQUENCE_TWO_TUPLE 218 -#define INSTRUMENTED_END_FOR 233 -#define INSTRUMENTED_POP_ITER 234 -#define INSTRUMENTED_END_SEND 235 -#define INSTRUMENTED_FOR_ITER 236 -#define INSTRUMENTED_INSTRUCTION 237 -#define INSTRUMENTED_JUMP_FORWARD 238 -#define INSTRUMENTED_NOT_TAKEN 239 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 240 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 241 -#define INSTRUMENTED_POP_JUMP_IF_NONE 242 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 243 -#define INSTRUMENTED_RESUME 244 -#define INSTRUMENTED_RETURN_VALUE 245 -#define INSTRUMENTED_YIELD_VALUE 246 -#define INSTRUMENTED_END_ASYNC_FOR 247 -#define INSTRUMENTED_LOAD_SUPER_ATTR 248 -#define INSTRUMENTED_CALL 249 -#define INSTRUMENTED_CALL_KW 250 +#define ASYNC_CALL_BOUND_METHOD_EXACT_ARGS 129 +#define ASYNC_CALL_BOUND_METHOD_GENERAL 130 +#define ASYNC_CALL_KW_BOUND_METHOD 131 +#define ASYNC_CALL_KW_PY 132 +#define ASYNC_CALL_PY_EXACT_ARGS 133 +#define ASYNC_CALL_PY_GENERAL 134 +#define BINARY_OP_ADD_FLOAT 135 +#define BINARY_OP_ADD_INT 136 +#define BINARY_OP_ADD_UNICODE 137 +#define BINARY_OP_EXTEND 138 +#define BINARY_OP_MULTIPLY_FLOAT 139 +#define BINARY_OP_MULTIPLY_INT 140 +#define BINARY_OP_SUBSCR_DICT 141 +#define BINARY_OP_SUBSCR_GETITEM 142 +#define BINARY_OP_SUBSCR_LIST_INT 143 +#define BINARY_OP_SUBSCR_LIST_SLICE 144 +#define BINARY_OP_SUBSCR_STR_INT 145 +#define BINARY_OP_SUBSCR_TUPLE_INT 146 +#define BINARY_OP_SUBSCR_USTR_INT 147 +#define BINARY_OP_SUBTRACT_FLOAT 148 +#define BINARY_OP_SUBTRACT_INT 149 +#define CALL_ALLOC_AND_ENTER_INIT 150 +#define CALL_BOUND_METHOD_EXACT_ARGS 151 +#define CALL_BOUND_METHOD_GENERAL 152 +#define CALL_BUILTIN_CLASS 153 +#define CALL_BUILTIN_FAST 154 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 155 +#define CALL_BUILTIN_O 156 +#define CALL_EX_NON_PY_GENERAL 157 +#define CALL_EX_PY 158 +#define CALL_ISINSTANCE 159 +#define CALL_KW_BOUND_METHOD 160 +#define CALL_KW_NON_PY 161 +#define CALL_KW_PY 162 +#define CALL_LEN 163 +#define CALL_LIST_APPEND 164 +#define CALL_METHOD_DESCRIPTOR_FAST 165 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 166 +#define CALL_METHOD_DESCRIPTOR_NOARGS 167 +#define CALL_METHOD_DESCRIPTOR_O 168 +#define CALL_NON_PY_GENERAL 169 +#define CALL_PY_EXACT_ARGS 170 +#define CALL_PY_GENERAL 171 +#define CALL_STR_1 172 +#define CALL_TUPLE_1 173 +#define CALL_TYPE_1 174 +#define COMPARE_OP_FLOAT 175 +#define COMPARE_OP_INT 176 +#define COMPARE_OP_STR 177 +#define CONTAINS_OP_DICT 178 +#define CONTAINS_OP_SET 179 +#define FOR_ITER_GEN 180 +#define FOR_ITER_LIST 181 +#define FOR_ITER_RANGE 182 +#define FOR_ITER_TUPLE 183 +#define FOR_ITER_VIRTUAL 184 +#define GET_ITER_SELF 185 +#define GET_ITER_VIRTUAL 186 +#define JUMP_BACKWARD_JIT 187 +#define JUMP_BACKWARD_NO_JIT 188 +#define LOAD_ATTR_CLASS 189 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 190 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 191 +#define LOAD_ATTR_INSTANCE_VALUE 192 +#define LOAD_ATTR_METHOD_LAZY_DICT 193 +#define LOAD_ATTR_METHOD_NO_DICT 194 +#define LOAD_ATTR_METHOD_WITH_VALUES 195 +#define LOAD_ATTR_MODULE 196 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 197 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 198 +#define LOAD_ATTR_PROPERTY 199 +#define LOAD_ATTR_SLOT 200 +#define LOAD_ATTR_WITH_HINT 201 +#define LOAD_GLOBAL_BUILTIN 202 +#define LOAD_GLOBAL_MODULE 203 +#define LOAD_SUPER_ATTR_ATTR 204 +#define LOAD_SUPER_ATTR_METHOD 205 +#define RESUME_CHECK 206 +#define RESUME_CHECK_JIT 207 +#define SEND_ASYNC_GEN 208 +#define SEND_GEN 209 +#define SEND_VIRTUAL 210 +#define STORE_ATTR_INSTANCE_VALUE 211 +#define STORE_ATTR_SLOT 212 +#define STORE_ATTR_WITH_HINT 213 +#define STORE_SUBSCR_DICT 214 +#define STORE_SUBSCR_LIST_INT 215 +#define TO_BOOL_ALWAYS_TRUE 216 +#define TO_BOOL_BOOL 217 +#define TO_BOOL_INT 218 +#define TO_BOOL_LIST 219 +#define TO_BOOL_NONE 220 +#define TO_BOOL_STR 221 +#define UNPACK_SEQUENCE_LIST 222 +#define UNPACK_SEQUENCE_TUPLE 223 +#define UNPACK_SEQUENCE_TWO_TUPLE 224 +#define INSTRUMENTED_END_FOR 231 +#define INSTRUMENTED_POP_ITER 232 +#define INSTRUMENTED_END_SEND 233 +#define INSTRUMENTED_FOR_ITER 234 +#define INSTRUMENTED_INSTRUCTION 235 +#define INSTRUMENTED_JUMP_FORWARD 236 +#define INSTRUMENTED_NOT_TAKEN 237 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 238 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 239 +#define INSTRUMENTED_POP_JUMP_IF_NONE 240 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 241 +#define INSTRUMENTED_RESUME 242 +#define INSTRUMENTED_RETURN_VALUE 243 +#define INSTRUMENTED_YIELD_VALUE 244 +#define INSTRUMENTED_END_ASYNC_FOR 245 +#define INSTRUMENTED_LOAD_SUPER_ATTR 246 +#define INSTRUMENTED_CALL 247 +#define INSTRUMENTED_ASYNC_CALL 248 +#define INSTRUMENTED_CALL_KW 249 +#define INSTRUMENTED_ASYNC_CALL_KW 250 #define INSTRUMENTED_CALL_FUNCTION_EX 251 #define INSTRUMENTED_JUMP_BACKWARD 252 #define INSTRUMENTED_LINE 253 @@ -255,7 +265,7 @@ extern "C" { #define HAVE_ARGUMENT 41 #define MIN_SPECIALIZED_OPCODE 129 -#define MIN_INSTRUMENTED_OPCODE 233 +#define MIN_INSTRUMENTED_OPCODE 231 #ifdef __cplusplus } diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index fa652cd231cfa95..e77d64bde71739a 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -104,3 +104,19 @@ so this data flow works along the chain (see `gen_send_ex2()` in Recall that a generator's `__next__` function simply calls `self.send(None)`, so all this works the same in generators and coroutines, but only coroutines use the value of the argument to `send`. + +An `await` expression is compiled like `yield from`, with +[`GET_AWAITABLE`](https://docs.python.org/dev/library/dis.html#opcode-GET_AWAITABLE) +in place of `GET_ITER`. When the awaited object is the result of a call, as in +`await f()`, the compiler emits `ASYNC_CALL` (or `ASYNC_CALL_KW`) instead of +`CALL`. Unspecialized, it does the same as `CALL`. When the called function is +a coroutine function, it is [specialized](interpreter.md) to one of the +`ASYNC_CALL_*` instructions, such as `ASYNC_CALL_PY_EXACT_ARGS`. These merge +the call with the first execution of the `GET_AWAITABLE`, `PUSH_NULL`, +`LOAD_COMMON_CONSTANT` and `SEND` instructions that follow it. + +`ASYNC_CALL_*` create the frame with the micro-ops of the matching `CALL_*` +or `CALL_KW_*` specialization. They then turn it into the frame of a new +coroutine and push it at once, as `RETURN_GENERATOR` and `SEND_GEN` would do. +The coroutine is not returned to the caller first and the frame of the caller +is left as if it were executing the `SEND`. diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index df92eae151d2488..fe760d8c3d9cdb1 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -121,11 +121,21 @@ "CALL_BOUND_METHOD_GENERAL", "CALL_NON_PY_GENERAL", ), + ASYNC_CALL=( + "ASYNC_CALL_PY_EXACT_ARGS", + "ASYNC_CALL_BOUND_METHOD_EXACT_ARGS", + "ASYNC_CALL_PY_GENERAL", + "ASYNC_CALL_BOUND_METHOD_GENERAL", + ), CALL_KW=( "CALL_KW_BOUND_METHOD", "CALL_KW_PY", "CALL_KW_NON_PY", ), + ASYNC_CALL_KW=( + "ASYNC_CALL_KW_PY", + "ASYNC_CALL_KW_BOUND_METHOD", + ), CALL_FUNCTION_EX=( "CALL_EX_PY", "CALL_EX_NON_PY_GENERAL", @@ -133,97 +143,103 @@ ) _specialized_opmap = frozendict( - BINARY_OP_ADD_FLOAT=129, - BINARY_OP_ADD_INT=130, - BINARY_OP_ADD_UNICODE=131, - BINARY_OP_EXTEND=132, + ASYNC_CALL_BOUND_METHOD_EXACT_ARGS=129, + ASYNC_CALL_BOUND_METHOD_GENERAL=130, + ASYNC_CALL_KW_BOUND_METHOD=131, + ASYNC_CALL_KW_PY=132, + ASYNC_CALL_PY_EXACT_ARGS=133, + ASYNC_CALL_PY_GENERAL=134, + BINARY_OP_ADD_FLOAT=135, + BINARY_OP_ADD_INT=136, + BINARY_OP_ADD_UNICODE=137, + BINARY_OP_EXTEND=138, BINARY_OP_INPLACE_ADD_UNICODE=3, - BINARY_OP_MULTIPLY_FLOAT=133, - BINARY_OP_MULTIPLY_INT=134, - BINARY_OP_SUBSCR_DICT=135, - BINARY_OP_SUBSCR_GETITEM=136, - BINARY_OP_SUBSCR_LIST_INT=137, - BINARY_OP_SUBSCR_LIST_SLICE=138, - BINARY_OP_SUBSCR_STR_INT=139, - BINARY_OP_SUBSCR_TUPLE_INT=140, - BINARY_OP_SUBSCR_USTR_INT=141, - BINARY_OP_SUBTRACT_FLOAT=142, - BINARY_OP_SUBTRACT_INT=143, - CALL_ALLOC_AND_ENTER_INIT=144, - CALL_BOUND_METHOD_EXACT_ARGS=145, - CALL_BOUND_METHOD_GENERAL=146, - CALL_BUILTIN_CLASS=147, - CALL_BUILTIN_FAST=148, - CALL_BUILTIN_FAST_WITH_KEYWORDS=149, - CALL_BUILTIN_O=150, - CALL_EX_NON_PY_GENERAL=151, - CALL_EX_PY=152, - CALL_ISINSTANCE=153, - CALL_KW_BOUND_METHOD=154, - CALL_KW_NON_PY=155, - CALL_KW_PY=156, - CALL_LEN=157, - CALL_LIST_APPEND=158, - CALL_METHOD_DESCRIPTOR_FAST=159, - CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=160, - CALL_METHOD_DESCRIPTOR_NOARGS=161, - CALL_METHOD_DESCRIPTOR_O=162, - CALL_NON_PY_GENERAL=163, - CALL_PY_EXACT_ARGS=164, - CALL_PY_GENERAL=165, - CALL_STR_1=166, - CALL_TUPLE_1=167, - CALL_TYPE_1=168, - COMPARE_OP_FLOAT=169, - COMPARE_OP_INT=170, - COMPARE_OP_STR=171, - CONTAINS_OP_DICT=172, - CONTAINS_OP_SET=173, - FOR_ITER_GEN=174, - FOR_ITER_LIST=175, - FOR_ITER_RANGE=176, - FOR_ITER_TUPLE=177, - FOR_ITER_VIRTUAL=178, - GET_ITER_SELF=179, - GET_ITER_VIRTUAL=180, - JUMP_BACKWARD_JIT=181, - JUMP_BACKWARD_NO_JIT=182, - LOAD_ATTR_CLASS=183, - LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=184, - LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=185, - LOAD_ATTR_INSTANCE_VALUE=186, - LOAD_ATTR_METHOD_LAZY_DICT=187, - LOAD_ATTR_METHOD_NO_DICT=188, - LOAD_ATTR_METHOD_WITH_VALUES=189, - LOAD_ATTR_MODULE=190, - LOAD_ATTR_NONDESCRIPTOR_NO_DICT=191, - LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=192, - LOAD_ATTR_PROPERTY=193, - LOAD_ATTR_SLOT=194, - LOAD_ATTR_WITH_HINT=195, - LOAD_GLOBAL_BUILTIN=196, - LOAD_GLOBAL_MODULE=197, - LOAD_SUPER_ATTR_ATTR=198, - LOAD_SUPER_ATTR_METHOD=199, - RESUME_CHECK=200, - RESUME_CHECK_JIT=201, - SEND_ASYNC_GEN=202, - SEND_GEN=203, - SEND_VIRTUAL=204, - STORE_ATTR_INSTANCE_VALUE=205, - STORE_ATTR_SLOT=206, - STORE_ATTR_WITH_HINT=207, - STORE_SUBSCR_DICT=208, - STORE_SUBSCR_LIST_INT=209, - TO_BOOL_ALWAYS_TRUE=210, - TO_BOOL_BOOL=211, - TO_BOOL_INT=212, - TO_BOOL_LIST=213, - TO_BOOL_NONE=214, - TO_BOOL_STR=215, - UNPACK_SEQUENCE_LIST=216, - UNPACK_SEQUENCE_TUPLE=217, - UNPACK_SEQUENCE_TWO_TUPLE=218, + BINARY_OP_MULTIPLY_FLOAT=139, + BINARY_OP_MULTIPLY_INT=140, + BINARY_OP_SUBSCR_DICT=141, + BINARY_OP_SUBSCR_GETITEM=142, + BINARY_OP_SUBSCR_LIST_INT=143, + BINARY_OP_SUBSCR_LIST_SLICE=144, + BINARY_OP_SUBSCR_STR_INT=145, + BINARY_OP_SUBSCR_TUPLE_INT=146, + BINARY_OP_SUBSCR_USTR_INT=147, + BINARY_OP_SUBTRACT_FLOAT=148, + BINARY_OP_SUBTRACT_INT=149, + CALL_ALLOC_AND_ENTER_INIT=150, + CALL_BOUND_METHOD_EXACT_ARGS=151, + CALL_BOUND_METHOD_GENERAL=152, + CALL_BUILTIN_CLASS=153, + CALL_BUILTIN_FAST=154, + CALL_BUILTIN_FAST_WITH_KEYWORDS=155, + CALL_BUILTIN_O=156, + CALL_EX_NON_PY_GENERAL=157, + CALL_EX_PY=158, + CALL_ISINSTANCE=159, + CALL_KW_BOUND_METHOD=160, + CALL_KW_NON_PY=161, + CALL_KW_PY=162, + CALL_LEN=163, + CALL_LIST_APPEND=164, + CALL_METHOD_DESCRIPTOR_FAST=165, + CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=166, + CALL_METHOD_DESCRIPTOR_NOARGS=167, + CALL_METHOD_DESCRIPTOR_O=168, + CALL_NON_PY_GENERAL=169, + CALL_PY_EXACT_ARGS=170, + CALL_PY_GENERAL=171, + CALL_STR_1=172, + CALL_TUPLE_1=173, + CALL_TYPE_1=174, + COMPARE_OP_FLOAT=175, + COMPARE_OP_INT=176, + COMPARE_OP_STR=177, + CONTAINS_OP_DICT=178, + CONTAINS_OP_SET=179, + FOR_ITER_GEN=180, + FOR_ITER_LIST=181, + FOR_ITER_RANGE=182, + FOR_ITER_TUPLE=183, + FOR_ITER_VIRTUAL=184, + GET_ITER_SELF=185, + GET_ITER_VIRTUAL=186, + JUMP_BACKWARD_JIT=187, + JUMP_BACKWARD_NO_JIT=188, + LOAD_ATTR_CLASS=189, + LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=190, + LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=191, + LOAD_ATTR_INSTANCE_VALUE=192, + LOAD_ATTR_METHOD_LAZY_DICT=193, + LOAD_ATTR_METHOD_NO_DICT=194, + LOAD_ATTR_METHOD_WITH_VALUES=195, + LOAD_ATTR_MODULE=196, + LOAD_ATTR_NONDESCRIPTOR_NO_DICT=197, + LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=198, + LOAD_ATTR_PROPERTY=199, + LOAD_ATTR_SLOT=200, + LOAD_ATTR_WITH_HINT=201, + LOAD_GLOBAL_BUILTIN=202, + LOAD_GLOBAL_MODULE=203, + LOAD_SUPER_ATTR_ATTR=204, + LOAD_SUPER_ATTR_METHOD=205, + RESUME_CHECK=206, + RESUME_CHECK_JIT=207, + SEND_ASYNC_GEN=208, + SEND_GEN=209, + SEND_VIRTUAL=210, + STORE_ATTR_INSTANCE_VALUE=211, + STORE_ATTR_SLOT=212, + STORE_ATTR_WITH_HINT=213, + STORE_SUBSCR_DICT=214, + STORE_SUBSCR_LIST_INT=215, + TO_BOOL_ALWAYS_TRUE=216, + TO_BOOL_BOOL=217, + TO_BOOL_INT=218, + TO_BOOL_LIST=219, + TO_BOOL_NONE=220, + TO_BOOL_STR=221, + UNPACK_SEQUENCE_LIST=222, + UNPACK_SEQUENCE_TUPLE=223, + UNPACK_SEQUENCE_TWO_TUPLE=224, ) opmap = frozendict( @@ -272,99 +288,103 @@ UNARY_NEGATIVE=39, UNARY_NOT=40, WITH_EXCEPT_START=41, - BINARY_OP=42, - BUILD_INTERPOLATION=43, - BUILD_LIST=44, - BUILD_MAP=45, - BUILD_SET=46, - BUILD_SLICE=47, - BUILD_STRING=48, - BUILD_TUPLE=49, - CALL=50, - CALL_INTRINSIC_1=51, - CALL_INTRINSIC_2=52, - CALL_KW=53, - COMPARE_OP=54, - CONTAINS_OP=55, - CONVERT_VALUE=56, - COPY=57, - COPY_FREE_VARS=58, - DELETE_DEREF=59, - DELETE_FAST=60, - DICT_MERGE=61, - DICT_UPDATE=62, - END_ASYNC_FOR=63, - EXTENDED_ARG=64, - FOR_ITER=65, - GET_AWAITABLE=66, - GET_ITER=67, - IMPORT_FROM=68, - IMPORT_NAME=69, - IS_OP=70, - JUMP_BACKWARD=71, - JUMP_BACKWARD_NO_INTERRUPT=72, - JUMP_FORWARD=73, - LIST_APPEND=74, - LIST_EXTEND=75, - LOAD_ATTR=76, - LOAD_COMMON_CONSTANT=77, - LOAD_CONST=78, - LOAD_DEREF=79, - LOAD_FAST=80, - LOAD_FAST_AND_CLEAR=81, - LOAD_FAST_BORROW=82, - LOAD_FAST_BORROW_LOAD_FAST_BORROW=83, - LOAD_FAST_CHECK=84, - LOAD_FAST_LOAD_FAST=85, - LOAD_FROM_DICT_OR_DEREF=86, - LOAD_FROM_DICT_OR_GLOBALS=87, - LOAD_GLOBAL=88, - LOAD_NAME=89, - LOAD_SMALL_INT=90, - LOAD_SPECIAL=91, - LOAD_SUPER_ATTR=92, - MAKE_CELL=93, - MAP_ADD=94, - MATCH_CLASS=95, - POP_JUMP_IF_FALSE=96, - POP_JUMP_IF_NONE=97, - POP_JUMP_IF_NOT_NONE=98, - POP_JUMP_IF_TRUE=99, - RAISE_VARARGS=100, - RERAISE=101, - SEND=102, - SET_ADD=103, - SET_FUNCTION_ATTRIBUTE=104, - SET_UPDATE=105, - STORE_ATTR=106, - STORE_DEREF=107, - STORE_FAST=108, - STORE_FAST_LOAD_FAST=109, - STORE_FAST_STORE_FAST=110, - STORE_GLOBAL=111, - STORE_NAME=112, - SWAP=113, - UNPACK_EX=114, - UNPACK_SEQUENCE=115, - YIELD_VALUE=116, - INSTRUMENTED_END_FOR=233, - INSTRUMENTED_POP_ITER=234, - INSTRUMENTED_END_SEND=235, - INSTRUMENTED_FOR_ITER=236, - INSTRUMENTED_INSTRUCTION=237, - INSTRUMENTED_JUMP_FORWARD=238, - INSTRUMENTED_NOT_TAKEN=239, - INSTRUMENTED_POP_JUMP_IF_TRUE=240, - INSTRUMENTED_POP_JUMP_IF_FALSE=241, - INSTRUMENTED_POP_JUMP_IF_NONE=242, - INSTRUMENTED_POP_JUMP_IF_NOT_NONE=243, - INSTRUMENTED_RESUME=244, - INSTRUMENTED_RETURN_VALUE=245, - INSTRUMENTED_YIELD_VALUE=246, - INSTRUMENTED_END_ASYNC_FOR=247, - INSTRUMENTED_LOAD_SUPER_ATTR=248, - INSTRUMENTED_CALL=249, - INSTRUMENTED_CALL_KW=250, + ASYNC_CALL=42, + ASYNC_CALL_KW=43, + BINARY_OP=44, + BUILD_INTERPOLATION=45, + BUILD_LIST=46, + BUILD_MAP=47, + BUILD_SET=48, + BUILD_SLICE=49, + BUILD_STRING=50, + BUILD_TUPLE=51, + CALL=52, + CALL_INTRINSIC_1=53, + CALL_INTRINSIC_2=54, + CALL_KW=55, + COMPARE_OP=56, + CONTAINS_OP=57, + CONVERT_VALUE=58, + COPY=59, + COPY_FREE_VARS=60, + DELETE_DEREF=61, + DELETE_FAST=62, + DICT_MERGE=63, + DICT_UPDATE=64, + END_ASYNC_FOR=65, + EXTENDED_ARG=66, + FOR_ITER=67, + GET_AWAITABLE=68, + GET_ITER=69, + IMPORT_FROM=70, + IMPORT_NAME=71, + IS_OP=72, + JUMP_BACKWARD=73, + JUMP_BACKWARD_NO_INTERRUPT=74, + JUMP_FORWARD=75, + LIST_APPEND=76, + LIST_EXTEND=77, + LOAD_ATTR=78, + LOAD_COMMON_CONSTANT=79, + LOAD_CONST=80, + LOAD_DEREF=81, + LOAD_FAST=82, + LOAD_FAST_AND_CLEAR=83, + LOAD_FAST_BORROW=84, + LOAD_FAST_BORROW_LOAD_FAST_BORROW=85, + LOAD_FAST_CHECK=86, + LOAD_FAST_LOAD_FAST=87, + LOAD_FROM_DICT_OR_DEREF=88, + LOAD_FROM_DICT_OR_GLOBALS=89, + LOAD_GLOBAL=90, + LOAD_NAME=91, + LOAD_SMALL_INT=92, + LOAD_SPECIAL=93, + LOAD_SUPER_ATTR=94, + MAKE_CELL=95, + MAP_ADD=96, + MATCH_CLASS=97, + POP_JUMP_IF_FALSE=98, + POP_JUMP_IF_NONE=99, + POP_JUMP_IF_NOT_NONE=100, + POP_JUMP_IF_TRUE=101, + RAISE_VARARGS=102, + RERAISE=103, + SEND=104, + SET_ADD=105, + SET_FUNCTION_ATTRIBUTE=106, + SET_UPDATE=107, + STORE_ATTR=108, + STORE_DEREF=109, + STORE_FAST=110, + STORE_FAST_LOAD_FAST=111, + STORE_FAST_STORE_FAST=112, + STORE_GLOBAL=113, + STORE_NAME=114, + SWAP=115, + UNPACK_EX=116, + UNPACK_SEQUENCE=117, + YIELD_VALUE=118, + INSTRUMENTED_END_FOR=231, + INSTRUMENTED_POP_ITER=232, + INSTRUMENTED_END_SEND=233, + INSTRUMENTED_FOR_ITER=234, + INSTRUMENTED_INSTRUCTION=235, + INSTRUMENTED_JUMP_FORWARD=236, + INSTRUMENTED_NOT_TAKEN=237, + INSTRUMENTED_POP_JUMP_IF_TRUE=238, + INSTRUMENTED_POP_JUMP_IF_FALSE=239, + INSTRUMENTED_POP_JUMP_IF_NONE=240, + INSTRUMENTED_POP_JUMP_IF_NOT_NONE=241, + INSTRUMENTED_RESUME=242, + INSTRUMENTED_RETURN_VALUE=243, + INSTRUMENTED_YIELD_VALUE=244, + INSTRUMENTED_END_ASYNC_FOR=245, + INSTRUMENTED_LOAD_SUPER_ATTR=246, + INSTRUMENTED_CALL=247, + INSTRUMENTED_ASYNC_CALL=248, + INSTRUMENTED_CALL_KW=249, + INSTRUMENTED_ASYNC_CALL_KW=250, INSTRUMENTED_CALL_FUNCTION_EX=251, INSTRUMENTED_JUMP_BACKWARD=252, ANNOTATIONS_PLACEHOLDER=256, @@ -381,4 +401,4 @@ ) HAVE_ARGUMENT = 41 -MIN_INSTRUMENTED_OPCODE = 233 +MIN_INSTRUMENTED_OPCODE = 231 diff --git a/Lib/opcode.py b/Lib/opcode.py index 750d83b2c87af58..348e6154b648616 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -97,6 +97,14 @@ counter=1, func_version=2, ), + ASYNC_CALL=frozendict( + counter=1, + func_version=2, + ), + ASYNC_CALL_KW=frozendict( + counter=1, + func_version=2, + ), CALL_FUNCTION_EX=frozendict( counter=1, ), diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 88077aacecbf5b1..320af175affe0f2 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -5917,6 +5917,49 @@ def testfunc(n): # _POP_TOP_NOP is a sign the optimizer ran and didn't hit bottom. self.assertGreaterEqual(count_ops(ex, "_POP_TOP_NOP"), 1) + def test_async_call_gen_frame(self): + + async def inner(i): + return i + 1 + def make_closure(v): + async def inner(i): + return i + v + return inner + def make_outer(callee): + ns = {"callee": callee} + exec(textwrap.dedent(""" + async def outer(n): + total = 0 + for i in range(n): + total += await callee(i) + return total + """), ns) + return ns["outer"] + def run(coro): + try: + coro.send(None) + except StopIteration as ex: + return ex.value + + n = TIER2_THRESHOLD * 2 + for callee in inner, make_closure(1): + with self.subTest(callee=callee): + outer = make_outer(callee) + self.assertEqual(run(outer(n)), sum(range(1, n + 1))) + ex = get_first_executor(outer) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + # The coroutine is started without being returned to outer() + # first, and runs to its return within the trace. + self.assertIn("_ASYNC_CALL_GEN_FRAME", uops) + self.assertNotIn("_RETURN_GENERATOR", uops) + self.assertNotIn("_SEND_GEN_FRAME", uops) + self.assertIn("_RETURN_VALUE", uops) + self.assertIn("_END_SEND", uops) + # The guard is only needed in tier 1. + self.assertNotIn("_GUARD_ASYNC_CALL_TAIL", uops) + def test_send_virtual(self): def send_list(n): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index df766387dd4c72d..cbcba0d50f101d9 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1380,6 +1380,37 @@ def test_folding_type_param(self): self.assertIn('LOAD_SMALL_INT', instructions) self.assertIn(45, args) + def test_awaited_call_opcodes(self): + # A call whose result is awaited at once compiles to ASYNC_CALL or + # ASYNC_CALL_KW, always followed by the instructions of the await. + async def f(c, o, x): + await c(1) + await o.m(1) + await c(1, k=2) + await o.m(1, k=2) + async with o: + pass + await c(*x) + await x + c(1) + o.m(1, k=2) + tail = ['GET_AWAITABLE', 'PUSH_NULL', 'LOAD_COMMON_CONSTANT', 'SEND'] + instrs = list(dis.get_instructions(f)) + opnames = [i.opname for i in instrs] + awaited = [i for i, name in enumerate(opnames) + if name in ('ASYNC_CALL', 'ASYNC_CALL_KW')] + for i in awaited: + self.assertEqual(opnames[i + 1:i + 5], tail, instrs[i]) + self.assertEqual([opnames[i] for i in awaited], + ['ASYNC_CALL', 'ASYNC_CALL', 'ASYNC_CALL_KW', + 'ASYNC_CALL_KW', 'ASYNC_CALL', 'ASYNC_CALL']) + self.assertEqual(opnames.count('CALL'), 1) + self.assertEqual(opnames.count('CALL_KW'), 1) + self.assertEqual(opnames.count('CALL_FUNCTION_EX'), 1) + for i, name in enumerate(opnames): + if name in ('CALL', 'CALL_KW'): + self.assertNotEqual(opnames[i + 1], 'GET_AWAITABLE', instrs[i]) + def test_lineno_procedure_call(self): def call(): ( diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index c5900da09501b36..2e740066060bf93 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2397,6 +2397,360 @@ async def coro(): self.assertEqual(yielded, []) self.assertEqual(result, 42) +class AwaitCallTest(unittest.TestCase): + # Run each call often enough to exercise the specialized forms too. + N = 100 + + def test_function(self): + async def add(a, b=10): + return a + b + + async def caller(): + total = 0 + for i in range(self.N): + total += await add(i, 1) + total += await add(i) + return total + + expected = sum(i + 1 + i + 10 for i in range(self.N)) + self.assertEqual(run_async(caller()), ([], expected)) + + def test_method(self): + class A: + async def add(self, a, b=10): + return a + b + + async def caller(): + obj = A() + bound = obj.add + total = 0 + for i in range(self.N): + total += await obj.add(i, 1) + total += await obj.add(i) + total += await bound(i, 1) + total += await bound(i) + return total + + expected = sum(2 * (i + 1 + i + 10) for i in range(self.N)) + self.assertEqual(run_async(caller()), ([], expected)) + + def test_keyword_arguments(self): + async def add(a, b=10): + return a + b + + class A: + async def add(self, a, b=10): + return a + b + + async def caller(): + obj = A() + bound = obj.add + total = 0 + for i in range(self.N): + total += await add(i, b=1) + total += await add(a=i) + total += await obj.add(i, b=1) + total += await bound(b=1, a=i) + return total + + expected = sum(i + 1 + i + 10 + 2 * (i + 1) for i in range(self.N)) + self.assertEqual(run_async(caller()), ([], expected)) + + def test_closure(self): + # COPY_FREE_VARS and MAKE_CELL precede RETURN_GENERATOR + def outer(free): + async def inner(arg): + cell = arg + return (lambda: (cell, free, __class__))() + return inner + + async def caller(): + inner = outer("free") + for i in range(self.N): + self.assertEqual(await inner(i), (i, "free", AwaitCallTest)) + + self.assertEqual(run_async(caller()), ([], None)) + + def test_suspend(self): + class Suspend: + def __init__(self, value): + self.value = value + + def __await__(self): + return (yield self.value) + + async def inner(i): + sent = await Suspend(i) + return sent * 2 + + async def middle(i): + return await inner(i) + 1 + + async def outer(): + return [await middle(i) for i in range(self.N)] + + coro = outer() + value = coro.send(None) + for i in range(self.N): + self.assertEqual(value, i) + self.assertEqual(inspect.getcoroutinestate(coro), "CORO_SUSPENDED") + # outer() -> middle() -> inner() -> Suspend().__await__() + self.assertEqual(coro.cr_await.cr_await.cr_code, inner.__code__) + try: + value = coro.send(i + 100) + except StopIteration as ex: + self.assertEqual(i, self.N - 1) + result = ex.value + self.assertEqual(result, [(i + 100) * 2 + 1 for i in range(self.N)]) + + def test_async_with(self): + class CM: + async def __aenter__(self): + return "entered" + + async def __aexit__(self, *exc_info): + return exc_info[0] is ZeroDivisionError + + async def caller(): + cm = CM() + for i in range(self.N): + async with cm as value: + self.assertEqual(value, "entered") + 1 / 0 + + self.assertEqual(run_async(caller()), ([], None)) + + def test_generator_based_coroutine(self): + @types.coroutine + def gen_coro(value): + sent = yield value + return sent + 1 + + async def caller(): + return [await gen_coro(i) for i in range(self.N)] + + coro = caller() + value = coro.send(None) + for i in range(self.N): + self.assertEqual(value, i) + self.assertIs(coro.cr_await.gi_code, gen_coro.__code__) + try: + value = coro.send(i * 2) + except StopIteration as ex: + self.assertEqual(i, self.N - 1) + result = ex.value + self.assertEqual(result, [i * 2 + 1 for i in range(self.N)]) + + def test_throw_and_close(self): + async def inner(log): + try: + await AsyncYield("suspended") + except ZeroDivisionError: + log.append("caught") + return "handled" + finally: + log.append("finally") + + async def outer(log): + return await inner(log) + + for _ in range(self.N): + log = [] + coro = outer(log) + self.assertEqual(coro.send(None), "suspended") + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertEqual(cm.exception.value, "handled") + self.assertEqual(log, ["caught", "finally"]) + + log = [] + coro = outer(log) + self.assertEqual(coro.send(None), "suspended") + coro.close() + self.assertEqual(log, ["finally"]) + + def test_exception(self): + async def fail(i): + raise ZeroDivisionError(i) + + async def caller(): + for i in range(self.N): + try: + await fail(i) + except ZeroDivisionError as ex: + self.assertEqual(ex.args, (i,)) + tb = traceback.extract_tb(ex.__traceback__) + self.assertEqual([entry.name for entry in tb], + ["caller", "fail"]) + self.assertEqual(tb[0].line, "await fail(i)") + else: + self.fail("ZeroDivisionError not raised") + + self.assertEqual(run_async(caller()), ([], None)) + + def test_wrong_arguments(self): + async def func(a, b=1): + return a + + async def caller(): + for i in range(self.N): + try: + await func() + except TypeError as ex: + self.assertIn("missing 1 required positional", str(ex)) + else: + self.fail("TypeError not raised") + + self.assertEqual(run_async(caller()), ([], None)) + + def test_not_a_coroutine_function(self): + async def coro(i): + return i + + def function(i): + return coro(i) + + class Awaitable: + def __init__(self, i): + self.i = i + + def __await__(self): + return self.i + yield + + @types.coroutine + def gen_coro(i): + return i + yield + + def gen(i): + yield + + async def agen(i): + yield + + async def caller(): + for i in range(self.N): + self.assertEqual(await function(i), i) + self.assertEqual(await Awaitable(i), i) + self.assertEqual(await gen_coro(i), i) + for func in int, gen, agen: + with self.assertRaisesRegex(TypeError, "can't be awaited"): + await func(i) + + self.assertEqual(run_async(caller()), ([], None)) + + def test_change_callable(self): + class Awaitable: + def __await__(self): + return 5 + yield + + async def coro(a=1): + return a + + async def coro2(a): + return a + 1 + + def function(a): + return coro(a + 2) + + class A: + async def method(self, a): + return a + 3 + + async def caller(func, expected): + self.assertEqual(await func(1), expected) + + obj = A() + callables = [(coro, 1), (coro2, 2), (function, 3), (obj.method, 4), + (lambda a: Awaitable(), 5)] + for _ in range(self.N): + for func, expected in callables: + with self.subTest(func=func): + self.assertEqual(run_async(caller(func, expected)), + ([], None)) + + def test_already_awaited(self): + async def inner(): + await AsyncYield(None) + + async def outer(coro): + await coro + + async def caller(coro): + return await outer(coro) + + for _ in range(self.N): + awaited = inner() + first = caller(awaited) + first.send(None) + second = caller(awaited) + with self.assertRaisesRegex(RuntimeError, "already"): + second.send(None) + first.close() + + def test_recursion(self): + async def recurse(n): + if n: + return await recurse(n - 1) + 1 + return 0 + + self.assertEqual(run_async(recurse(self.N)), ([], self.N)) + + async def infinite(): + await infinite() + + with support.infinite_recursion(), self.assertRaises(RecursionError): + run_async(infinite()) + + def test_origin_tracking(self): + async def get_origin(): + coro = sys._getframe().f_generator + return coro.cr_origin + + async def caller(): + origins = set() + for _ in range(self.N): + origins.add(await get_origin()) + return origins + + orig_depth = sys.get_coroutine_origin_tracking_depth() + self.addCleanup(sys.set_coroutine_origin_tracking_depth, orig_depth) + sys.set_coroutine_origin_tracking_depth(1) + lineno = caller.__code__.co_firstlineno + 3 + self.assertEqual(run_async(caller()), + ([], {((__file__, lineno, "caller"),)})) + sys.set_coroutine_origin_tracking_depth(0) + self.assertEqual(run_async(caller()), ([], {None})) + + def test_line_events(self): + # The GET_AWAITABLE that follows the CALL starts a new line here, + # so it must not be skipped when tracing lines. + async def inner(): + pass + + async def caller(): + for _ in range(self.N): + await ( + inner() + ) + + def trace(frame, event, arg): + if frame.f_code is caller.__code__ and event == "line": + lines.append(frame.f_lineno - first) + return trace + + first = caller.__code__.co_firstlineno + run_async(caller()) + lines = [] + self.addCleanup(sys.settrace, sys.gettrace()) + sys.settrace(trace) + run_async(caller()) + sys.settrace(None) + self.assertEqual(lines, [1] + [3, 2, 1] * self.N) + + @unittest.skipIf( support.is_emscripten or support.is_wasi, "asyncio does not work under Emscripten/WASI yet." diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 709fa374704f1f5..226349bf64bd91c 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -613,7 +613,7 @@ async def _asyncwith(c): SWAP 2 SWAP 3 LOAD_SPECIAL 2 (__aenter__) - CALL 0 + ASYNC_CALL 0 GET_AWAITABLE 1 PUSH_NULL LOAD_COMMON_CONSTANT 7 (None) @@ -630,7 +630,7 @@ async def _asyncwith(c): %4d L7: LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) LOAD_COMMON_CONSTANT 7 (None) - CALL 3 + ASYNC_CALL 3 GET_AWAITABLE 2 PUSH_NULL LOAD_COMMON_CONSTANT 7 (None) diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 24cc51f4b4053ab..c57e46bd404df3c 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -2573,6 +2573,90 @@ def f(a=1, b=2): self.assertEqual(call_data[0], (f, 1)) self.assertEqual(call_data[1], (f, sys.monitoring.MISSING)) + def test_async_call(self): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + + async def coro(a): + return a + + async def caller(n): + for i in range(n): + await coro(i) + + def run(coro): + with self.assertRaises(StopIteration): + coro.send(None) + + def call(code, offset, callable, arg0): + if code is caller.__code__: + calls.append((callable, arg0)) + + def start(code, offset): + if code is coro.__code__: + starts.append(code) + + n = _testinternalcapi.SPECIALIZATION_THRESHOLD + 1 + expected = [(range, n)] + [(coro, i) for i in range(n)] + # Specialize the ASYNC_CALL before it is instrumented. + run(caller(n)) + + calls = [] + starts = [] + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.CALL, call) + sys.monitoring.register_callback(0, E.PY_START, start) + sys.monitoring.set_events(0, E.CALL | E.PY_START) + run(caller(n)) + sys.monitoring.set_events(0, 0) + self.assertEqual(calls, expected) + self.assertEqual(len(starts), n) + + run(caller(n)) + self.assertEqual(calls, expected) + + def test_async_call_instrumented(self): + # ASYNC_CALL and ASYNC_CALL_KW instrumented before they are + # specialized; a C callee gets a C_RETURN event too. + async def coro(a, b=0): + return a + b + + class Done: + def __await__(self): + return iter(()) + + async def caller(): + await coro(1) + await coro(1, b=2) + await Done() + + def run(coro): + with self.assertRaises(StopIteration): + coro.send(None) + + calls = [] + def call(code, offset, callable, arg0): + if code is caller.__code__: + calls.append(("call", callable)) + + def c_return(code, offset, callable, arg0): + if code is caller.__code__: + calls.append(("c_return", callable)) + + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.CALL, call) + sys.monitoring.register_callback(0, E.C_RETURN, c_return) + sys.monitoring.set_events(0, E.CALL | E.C_RETURN | E.C_RAISE) + run(caller()) + sys.monitoring.set_events(0, 0) + self.assertEqual(calls, [ + ("call", coro), ("call", coro), + ("call", Done), ("c_return", Done), + ]) + def test_instruction_explicit_callback(self): # gh-122247 # Calling the instruction event callback explicitly should not diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 5f7c15f93a8332a..685f5e00eaa0667 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1630,8 +1630,15 @@ def run_async(coro): except StopIteration: break + @types.coroutine + def suspend(): + yield + class CM: async def __aenter__(self): + # ASYNC_CALL_* starts the coroutine in place of the SEND + # in send_with(), which is only executed to resume it. + await suspend() return self async def __aexit__(self, *exc): @@ -1693,6 +1700,158 @@ async def send_for(): self.assert_specialized(send_for, "SEND_ASYNC_GEN") self.assert_no_opcode(send_for, "SEND") + @cpython_only + @requires_specialization + def test_async_call(self): + def run_async(coro): + try: + coro.send(None) + except StopIteration as ex: + return ex.value + self.fail("coroutine did not finish") + + async def coro(a, b=1): + return a + b + + class C: + async def method(self, a, b=1): + return a + b + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + pass + + def function(a): + return coro(a) + + obj = C() + + async def py_exact_args(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await coro(i, 2), i + 2) + self.assertEqual(await obj.method(i, 2), i + 2) + + run_async(py_exact_args()) + self.assert_specialized(py_exact_args, "ASYNC_CALL_PY_EXACT_ARGS") + self.assert_no_opcode(py_exact_args, "CALL_PY_EXACT_ARGS") + + async def py_general(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await coro(i), i + 1) + + run_async(py_general()) + self.assert_specialized(py_general, "ASYNC_CALL_PY_GENERAL") + + async def bound_method_exact_args(): + bound = obj.method + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await bound(i, 2), i + 2) + + run_async(bound_method_exact_args()) + self.assert_specialized(bound_method_exact_args, + "ASYNC_CALL_BOUND_METHOD_EXACT_ARGS") + self.assert_no_opcode(bound_method_exact_args, + "CALL_BOUND_METHOD_EXACT_ARGS") + + async def bound_method_general(): + bound = obj.method + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await bound(i), i + 1) + + run_async(bound_method_general()) + self.assert_specialized(bound_method_general, + "ASYNC_CALL_BOUND_METHOD_GENERAL") + + async def kw_py(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await coro(i, b=2), i + 2) + self.assertEqual(await obj.method(i, b=2), i + 2) + + run_async(kw_py()) + self.assert_specialized(kw_py, "ASYNC_CALL_KW_PY") + self.assert_no_opcode(kw_py, "CALL_KW_PY") + + async def kw_bound_method(): + bound = obj.method + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await bound(i, b=2), i + 2) + + run_async(kw_bound_method()) + self.assert_specialized(kw_bound_method, "ASYNC_CALL_KW_BOUND_METHOD") + self.assert_no_opcode(kw_bound_method, "CALL_KW_BOUND_METHOD") + self.assert_no_opcode(bound_method_general, + "CALL_BOUND_METHOD_GENERAL") + + async def async_with(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + async with obj as value: + self.assertIs(value, obj) + + run_async(async_with()) + self.assert_specialized(async_with, "ASYNC_CALL_PY_EXACT_ARGS") + self.assert_specialized(async_with, "ASYNC_CALL_PY_GENERAL") + + async def not_a_coroutine_function(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await function(i), i + 1) + + run_async(not_a_coroutine_function()) + # Only calls to coroutine functions are specialized. + self.assert_specialized(not_a_coroutine_function, "ASYNC_CALL") + self.assert_no_opcode(not_a_coroutine_function, + "ASYNC_CALL_PY_EXACT_ARGS") + self.assert_no_opcode(not_a_coroutine_function, "CALL_PY_EXACT_ARGS") + + async def not_awaited(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + c = coro(i) + self.assertEqual(await c, i + 1) + + run_async(not_awaited()) + self.assert_specialized(not_awaited, "CALL_PY_GENERAL") + self.assert_no_opcode(not_awaited, "ASYNC_CALL_PY_GENERAL") + self.assert_specialized(not_awaited, "SEND_GEN") + + @cpython_only + @requires_specialization + def test_async_call_deopt(self): + def run_async(coro): + try: + coro.send(None) + except StopIteration as ex: + return ex.value + self.fail("coroutine did not finish") + + async def coro(a): + return a + 1 + + async def other(a): + return a + 2 + + def function(a): + return coro(a + 2) + + async def caller(func, n): + return [await func(i) for i in range(n)] + + n = _testinternalcapi.SPECIALIZATION_THRESHOLD + self.assertEqual(run_async(caller(coro, n)), list(range(1, n + 1))) + self.assert_specialized(caller, "ASYNC_CALL_PY_EXACT_ARGS") + + n = _testinternalcapi.SPECIALIZATION_COOLDOWN + self.assertEqual(run_async(caller(function, n)), list(range(3, n + 3))) + self.assert_no_opcode(caller, "ASYNC_CALL_PY_EXACT_ARGS") + + self.assertEqual(run_async(caller(other, n)), list(range(2, n + 2))) + self.assert_specialized(caller, "ASYNC_CALL_PY_EXACT_ARGS") + + # Changing the code of the function invalidates the specialization. + other.__code__ = (lambda a: a).__code__ + with self.assertRaisesRegex(TypeError, "can't be awaited"): + run_async(caller(other, n)) + @cpython_only @requires_specialization def test_store_attr_slot(self): diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a5e7bbbfad5ffca..672f1a333e9a9d0 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2994,6 +2994,27 @@ def test_jump_or_pop(output): output.append(1) _ = output.append(2) and "Spam" output.append(3) + def test_no_jump_to_first_line_of_generator(self): + # The "def" line only owns the prologue of a generator frame + # (RETURN_GENERATOR), which must not run again. + def gen(): + output.append(1) + yield 1 + + output = [] + def trace(frame, event, arg): + if event == 'line' and frame.f_code is gen.__code__: + frame.f_lineno = gen.__code__.co_firstlineno + return trace + + sys.settrace(trace) + try: + with self.assertRaisesRegex(ValueError, "cannot find bytecode"): + list(gen()) + finally: + sys.settrace(None) + self.assertEqual(output, []) + class TestExtendedArgs(unittest.TestCase): diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 8e7ba1f69553dda..219cc271738a43f 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -18,6 +18,1299 @@ /* BEGIN INSTRUCTIONS */ + TARGET(ASYNC_CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL); + PREDICTED_ASYNC_CALL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = ASYNC_CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + // _SPECIALIZE_ASYNC_CALL + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_AsyncCall(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(ASYNC_CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL, + opcode == INSTRUMENTED_CALL || opcode == INSTRUMENTED_ASYNC_CALL, + frame, + this_instr, + tstate); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_BOUND_METHOD_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW); + PREDICTED_ASYNC_CALL_KW:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = ASYNC_CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + // _SPECIALIZE_ASYNC_CALL_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_AsyncCallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(ASYNC_CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + kwnames, + opcode == INSTRUMENTED_CALL_KW || opcode == INSTRUMENTED_ASYNC_CALL_KW, + frame, + this_instr, + tstate); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW_BOUND_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(callable_s); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(BINARY_OP) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP; @@ -1908,7 +3201,7 @@ arguments, total_args, PyStackRef_NULL, - opcode == INSTRUMENTED_CALL, + opcode == INSTRUMENTED_CALL || opcode == INSTRUMENTED_ASYNC_CALL, frame, this_instr, tstate); @@ -3479,7 +4772,7 @@ arguments, total_args, kwnames, - opcode == INSTRUMENTED_CALL_KW, + opcode == INSTRUMENTED_CALL_KW || opcode == INSTRUMENTED_ASYNC_CALL_KW, frame, this_instr, tstate); @@ -6431,10 +7724,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME @@ -7010,83 +8300,346 @@ if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - res_o = _PyEval_LazyImportFrom( - tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); + res_o = _PyEval_LazyImportFrom( + tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); + _PyFrame_StackPointerInvalidate(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyEval_ImportFrom( + tstate, PyStackRef_AsPyObjectBorrow(from), name); + _PyFrame_StackPointerInvalidate(frame); + } + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(IMPORT_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_NAME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyObject *res_o; + if (!(oparg & 0x02)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), + LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level), + oparg & 0x01); + _PyFrame_StackPointerInvalidate(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), + LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + _PyFrame_StackPointerInvalidate(frame); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_ASYNC_CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_ASYNC_CALL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_ASYNC_CALL); + opcode = INSTRUMENTED_ASYNC_CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef func; + _PyStackRef maybe_self; + _PyStackRef *args; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = self_or_null; + func = callable; + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL, + opcode == INSTRUMENTED_CALL || opcode == INSTRUMENTED_ASYNC_CALL, + frame, + this_instr, + tstate); _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - else { + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - res_o = _PyEval_ImportFrom( - tstate, PyStackRef_AsPyObjectBorrow(from), name); + int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } } - if (res_o == NULL) { - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } - TARGET(IMPORT_NAME) { + TARGET(INSTRUMENTED_ASYNC_CALL_KW) { #if _Py_TAIL_CALL_INTERP - int opcode = IMPORT_NAME; + int opcode = INSTRUMENTED_ASYNC_CALL_KW; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - _PyStackRef level; - _PyStackRef fromlist; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_ASYNC_CALL_KW); + opcode = INSTRUMENTED_ASYNC_CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; _PyStackRef res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyObject *res_o; - if (!(oparg & 0x02)) { + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _MONITOR_CALL_KW + { + args = &stack_pointer[-1 - oparg]; + int is_meth = !PyStackRef_IsNull(self_or_null); + PyObject *arg; + if (is_meth) { + arg = PyStackRef_AsPyObjectBorrow(self_or_null); + } + else if (args) { + arg = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg = &_PyInstrumentation_MISSING; + } + PyObject *function = PyStackRef_AsPyObjectBorrow(callable); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), - LOCALS(), name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level), - oparg & 0x01); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), - LOCALS(), name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + kwnames, + opcode == INSTRUMENTED_CALL_KW || opcode == INSTRUMENTED_ASYNC_CALL_KW, + frame, + this_instr, + tstate); _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyStackRef tmp = fromlist; - fromlist = PyStackRef_NULL; - stack_pointer[-1] = fromlist; - PyStackRef_CLOSE(tmp); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - tmp = level; - level = PyStackRef_NULL; - stack_pointer[-2] = level; - PyStackRef_CLOSE(tmp); - _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (res_o == NULL) { - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7198,7 +8751,7 @@ arguments, total_args, PyStackRef_NULL, - opcode == INSTRUMENTED_CALL, + opcode == INSTRUMENTED_CALL || opcode == INSTRUMENTED_ASYNC_CALL, frame, this_instr, tstate); @@ -7518,7 +9071,7 @@ arguments, total_args, kwnames, - opcode == INSTRUMENTED_CALL_KW, + opcode == INSTRUMENTED_CALL_KW || opcode == INSTRUMENTED_ASYNC_CALL_KW, frame, this_instr, tstate); @@ -11440,34 +12993,34 @@ next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + frame->instr_ptr++; + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); + gen->gi_frame_state = FRAME_CREATED; + _Py_LeaveRecursiveCallPy(tstate); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -11721,11 +13274,8 @@ STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4b..fdfce2e514702f6 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -42,6 +42,8 @@ static void *opcode_targets_table[256] = { &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, &&TARGET_WITH_EXCEPT_START, + &&TARGET_ASYNC_CALL, + &&TARGET_ASYNC_CALL_KW, &&TARGET_BINARY_OP, &&TARGET_BUILD_INTERPOLATION, &&TARGET_BUILD_LIST, @@ -126,9 +128,13 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, + &&TARGET_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_ASYNC_CALL_BOUND_METHOD_GENERAL, + &&TARGET_ASYNC_CALL_KW_BOUND_METHOD, + &&TARGET_ASYNC_CALL_KW_PY, + &&TARGET_ASYNC_CALL_PY_EXACT_ARGS, + &&TARGET_ASYNC_CALL_PY_GENERAL, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, @@ -225,14 +231,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -250,7 +248,9 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_END_ASYNC_FOR, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, &&TARGET_INSTRUMENTED_CALL, + &&TARGET_INSTRUMENTED_ASYNC_CALL, &&TARGET_INSTRUMENTED_CALL_KW, + &&TARGET_INSTRUMENTED_ASYNC_CALL_KW, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_LINE, @@ -376,8 +376,8 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -478,20 +478,20 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, @@ -531,6 +531,14 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PA static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_PY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_GENERAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); @@ -629,6 +637,8 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAM static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_ASYNC_CALL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_ASYNC_CALL_KW(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); @@ -774,6 +784,14 @@ JUMP_TO_LABEL(error); } static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { + [ASYNC_CALL] = _TAIL_CALL_ASYNC_CALL, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL, + [ASYNC_CALL_KW] = _TAIL_CALL_ASYNC_CALL_KW, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_ASYNC_CALL_KW_PY, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_ASYNC_CALL_PY_GENERAL, [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -872,6 +890,8 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [GET_LEN] = _TAIL_CALL_GET_LEN, [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, + [INSTRUMENTED_ASYNC_CALL] = _TAIL_CALL_INSTRUMENTED_ASYNC_CALL, + [INSTRUMENTED_ASYNC_CALL_KW] = _TAIL_CALL_INSTRUMENTED_ASYNC_CALL_KW, [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_CALL_KW] = _TAIL_CALL_INSTRUMENTED_CALL_KW, @@ -1005,8 +1025,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, - [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1016,22 +1034,22 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, [228] = _TAIL_CALL_UNKNOWN_OPCODE, [229] = _TAIL_CALL_UNKNOWN_OPCODE, [230] = _TAIL_CALL_UNKNOWN_OPCODE, - [231] = _TAIL_CALL_UNKNOWN_OPCODE, - [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [ASYNC_CALL] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, @@ -1130,6 +1148,8 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [GET_LEN] = _TAIL_CALL_TRACE_RECORD, [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_ASYNC_CALL] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_ASYNC_CALL_KW] = _TAIL_CALL_TRACE_RECORD, [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACE_RECORD, @@ -1263,8 +1283,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, - [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1274,19 +1292,11 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, [228] = _TAIL_CALL_UNKNOWN_OPCODE, [229] = _TAIL_CALL_UNKNOWN_OPCODE, [230] = _TAIL_CALL_UNKNOWN_OPCODE, - [231] = _TAIL_CALL_UNKNOWN_OPCODE, - [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 475be76dc3da21b..fdef52383cbd897 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1840,7 +1840,9 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value) int64_t start_stack = stacks[_PyInterpreterFrame_LASTI(self->f_frame)]; int err = -1; const char *msg = "cannot find bytecode for specified line"; - for (int i = 0; i < len; i++) { + /* The instructions before _co_firsttraceable set up a generator + * frame and must not run again once the frame belongs to one. */ + for (int i = code->_co_firsttraceable; i < len; i++) { if (lines[i] == new_lineno) { int64_t target_stack = stacks[i]; if (compatible_stack(start_stack, target_stack)) { diff --git a/Objects/genobject.c b/Objects/genobject.c index 82a1bb1ff420682..f5dd4501b701804 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1128,7 +1128,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) } PyObject * -_Py_MakeCoro(PyFunctionObject *func) +_Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame) { int coro_flags = ((PyCodeObject *)func->func_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); @@ -1160,7 +1160,6 @@ _Py_MakeCoro(PyFunctionObject *func) if (origin_depth == 0) { ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; } else { - _PyInterpreterFrame *frame = tstate->current_frame; assert(frame); assert(_PyFrame_IsIncomplete(frame)); frame = _PyFrame_GetFirstComplete(frame->previous); @@ -1174,6 +1173,12 @@ _Py_MakeCoro(PyFunctionObject *func) return coro; } +PyObject * +_Py_MakeCoro(PyFunctionObject *func) +{ + return _Py_MakeCoroWithFrame(func, _PyThreadState_GET()->current_frame); +} + static PyObject * gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, PyObject *name, PyObject *qualname) diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index ebcb38b18dec90c..ec288ca382f9568 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,19 +1,19 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, - 0,0,0,0,0,115,188,0,0,0,128,0,0,0,90,0, - 77,7,69,0,112,0,90,0,77,7,69,4,112,1,89,2, - 31,0,78,1,50,1,0,0,0,0,0,0,29,0,89,2, - 31,0,78,2,89,0,76,6,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,50,2,0,0,0,0, - 0,0,29,0,89,1,76,8,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,31,0,50,0,0,0, - 0,0,0,0,78,3,42,26,0,0,0,0,0,0,0,0, - 0,0,112,5,78,6,67,0,0,0,65,24,0,0,112,6, - 89,2,31,0,78,4,89,6,12,0,78,5,89,5,89,6, - 42,26,0,0,0,0,0,0,0,0,0,0,12,0,48,4, - 50,1,0,0,0,0,0,0,29,0,71,26,0,0,9,0, - 28,0,77,7,33,0,41,7,105,0,0,0,0,122,18,70, + 0,0,0,0,0,115,188,0,0,0,128,0,0,0,92,0, + 79,7,71,0,114,0,92,0,79,7,71,4,114,1,91,2, + 31,0,80,1,52,1,0,0,0,0,0,0,29,0,91,2, + 31,0,80,2,91,0,78,6,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,52,2,0,0,0,0, + 0,0,29,0,91,1,78,8,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,31,0,52,0,0,0, + 0,0,0,0,80,3,44,26,0,0,0,0,0,0,0,0, + 0,0,114,5,80,6,69,0,0,0,67,24,0,0,114,6, + 91,2,31,0,80,4,91,6,12,0,80,5,91,5,91,6, + 44,26,0,0,0,0,0,0,0,0,0,0,12,0,50,4, + 52,1,0,0,0,0,0,0,29,0,73,26,0,0,9,0, + 28,0,79,7,33,0,41,7,105,0,0,0,0,122,18,70, 114,111,122,101,110,32,72,101,108,108,111,32,87,111,114,108, 100,122,8,115,121,115,46,97,114,103,118,218,6,99,111,110, 102,105,103,122,7,99,111,110,102,105,103,32,122,2,58,32, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 42ad83daa1ea461..5e89a48af7b8977 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1730,11 +1730,8 @@ dummy_func( _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); DEAD(v); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)(INSTRUCTION_SIZE + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } @@ -4078,11 +4075,8 @@ dummy_func( STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; // oparg is the return offset from the next instruction. - frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)(INSTRUCTION_SIZE + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } @@ -4380,7 +4374,7 @@ dummy_func( arguments, total_args, PyStackRef_NULL, - opcode == INSTRUMENTED_CALL, + opcode == INSTRUMENTED_CALL || opcode == INSTRUMENTED_ASYNC_CALL, frame, this_instr, tstate); @@ -5345,6 +5339,141 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; + // Cache layout: counter/1, func_version/2 + // The compiler emits ASYNC_CALL instead of CALL when the result of + // the call is awaited at once, as in `await f(...)`, `async with` + // and async comprehensions. It is always followed by + // GET_AWAITABLE; PUSH_NULL; LOAD_COMMON_CONSTANT None; SEND + // Unspecialized, it does the same as CALL. Its specializations, for + // calls to coroutine functions, merge the call with the first + // execution of the instructions that follow it: the new coroutine is + // started without being returned to the caller first. They are built + // from the uops of the matching CALL_* specialization, with + // _ASYNC_CALL_GEN_FRAME in place of _SAVE_RETURN_OFFSET. + family(ASYNC_CALL, INLINE_CACHE_ENTRIES_ASYNC_CALL) = { + ASYNC_CALL_PY_EXACT_ARGS, + ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + ASYNC_CALL_PY_GENERAL, + ASYNC_CALL_BOUND_METHOD_GENERAL, + }; + + specializing op(_SPECIALIZE_ASYNC_CALL, (counter/1, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_AsyncCall(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(ASYNC_CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + + macro(ASYNC_CALL) = _SPECIALIZE_ASYNC_CALL + unused/2 + _MAYBE_EXPAND_METHOD + _DO_CALL + _CHECK_PERIODIC_AT_END; + macro(INSTRUMENTED_ASYNC_CALL) = unused/3 + _MAYBE_EXPAND_METHOD + _MONITOR_CALL + _DO_CALL + _CHECK_PERIODIC_AT_END; + + + // ASYNC_CALL_* skip the instructions up to and including SEND, so + // none of them may be instrumented. Only the first and the last of + // them can start a line. + // Executors are invalidated when the code is instrumented. + op(_GUARD_ASYNC_CALL_TAIL, (--)) { + #if TIER_ONE + DEOPT_IF(next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE); + DEOPT_IF(next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE); + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + + // Turns the frame of a call to a coroutine function into a running + // coroutine: the equivalent of executing RETURN_GENERATOR in that + // frame, followed by GET_AWAITABLE, PUSH_NULL, LOAD_COMMON_CONSTANT + // None and SEND_GEN in this one. + op(_ASYNC_CALL_GEN_FRAME, (new_frame -- coro, null, gen_frame)) { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + DEAD(new_frame); + SYNC_SP(); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + if (gen == NULL) { + _PyEval_FrameClearAndPop(tstate, callee); + ERROR_NO_POP(); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + // Otherwise RETURN_GENERATOR is preceded by COPY_FREE_VARS or + // MAKE_CELL. They run in the frame of the coroutine, where + // RETURN_GENERATOR only pushes the None that POP_TOP discards. + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + // The coroutine yields and returns to the SEND of the await. + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + INSTRUCTION_SIZE); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + + macro(ASYNC_CALL_PY_EXACT_ARGS) = + _RECORD_CALLABLE + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_FUNCTION_VERSION + + _CHECK_FUNCTION_EXACT_ARGS + + _CHECK_STACK_SPACE + + _CHECK_RECURSION_REMAINING + + _INIT_CALL_PY_EXACT_ARGS + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS) = + _RECORD_BOUND_METHOD + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + + _INIT_CALL_BOUND_METHOD_EXACT_ARGS + + flush + // In case the following deopt + _CHECK_FUNCTION_VERSION + + _CHECK_FUNCTION_EXACT_ARGS + + _CHECK_STACK_SPACE + + _CHECK_RECURSION_REMAINING + + _INIT_CALL_PY_EXACT_ARGS + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_PY_GENERAL) = + _RECORD_CALLABLE + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_FUNCTION_VERSION + + _CHECK_RECURSION_REMAINING + + _PY_FRAME_GENERAL + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_BOUND_METHOD_GENERAL) = + _RECORD_BOUND_METHOD + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_METHOD_VERSION + + _EXPAND_METHOD + + flush + // so that self is in the argument array + _CHECK_RECURSION_REMAINING + + _PY_FRAME_GENERAL + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + // Cache layout: counter/1, func_version/2 family(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW) = { CALL_KW_BOUND_METHOD, @@ -5426,7 +5555,7 @@ dummy_func( arguments, total_args, kwnames, - opcode == INSTRUMENTED_CALL_KW, + opcode == INSTRUMENTED_CALL_KW || opcode == INSTRUMENTED_ASYNC_CALL_KW, frame, this_instr, tstate); @@ -5542,6 +5671,61 @@ dummy_func( _MONITOR_CALL_KW + _DO_CALL_KW; + // Cache layout: counter/1, func_version/2 + // ASYNC_CALL_KW is to CALL_KW what ASYNC_CALL is to CALL. + family(ASYNC_CALL_KW, INLINE_CACHE_ENTRIES_ASYNC_CALL_KW) = { + ASYNC_CALL_KW_PY, + ASYNC_CALL_KW_BOUND_METHOD, + }; + + specializing op(_SPECIALIZE_ASYNC_CALL_KW, (counter/1, callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _Py_Specialize_AsyncCallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(ASYNC_CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + + macro(ASYNC_CALL_KW) = + _SPECIALIZE_ASYNC_CALL_KW + + unused/2 + + _MAYBE_EXPAND_METHOD_KW + + _DO_CALL_KW; + + macro(INSTRUMENTED_ASYNC_CALL_KW) = + counter/1 + + unused/2 + + _MAYBE_EXPAND_METHOD_KW + + _MONITOR_CALL_KW + + _DO_CALL_KW; + + macro(ASYNC_CALL_KW_PY) = + _RECORD_CALLABLE_KW + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_FUNCTION_VERSION_KW + + _CHECK_RECURSION_REMAINING + + _PY_FRAME_KW + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_KW_BOUND_METHOD) = + _RECORD_CALLABLE_KW + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_METHOD_VERSION_KW + + _EXPAND_METHOD_KW + + flush + // so that self is in the argument array + _PY_FRAME_KW + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); EXIT_IF(PyFunction_Check(callable_o)); @@ -5804,27 +5988,28 @@ dummy_func( } inst(RETURN_GENERATOR, (-- res)) { - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - ERROR_IF(gen == NULL); - assert(STACK_LEVEL() <= 2); - SAVE_STACK(); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - RELOAD_STACK(); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + // Handled by ASYNC_CALL_* already. + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + ERROR_IF(gen == NULL); + assert(STACK_LEVEL() <= 2); + SAVE_STACK(); + frame->instr_ptr++; + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); + gen->gi_frame_state = FRAME_CREATED; + _Py_LeaveRecursiveCallPy(tstate); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + RELOAD_STACK(); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + } } inst(BUILD_SLICE, (args[oparg] -- slice)) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 36921e35533433d..f36a53ad614ea11 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -594,6 +594,32 @@ gen_try_set_executing(PyGenObject *gen) return false; } +// Move the frame of a call to a generator function into the generator +// and pop it from the thread's data stack. `prev` is the frame that +// takes its place. +static inline void +gen_take_frame(PyThreadState *tstate, PyGenObject *gen, + _PyInterpreterFrame *frame, _PyInterpreterFrame *prev) +{ + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); +} + +// Link the frame of a generator to the frame that is about to enter it. +static inline void +gen_link_frame(PyThreadState *tstate, PyGenObject *gen, + _PyInterpreterFrame *frame, uint16_t return_offset) +{ + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + gen->gi_iframe.previous = frame; + frame->return_offset = return_offset; +} + // Macro for inplace float binary ops (tier 2 only). // Mutates the uniquely-referenced TARGET operand in place. // TARGET must be either left or right. diff --git a/Python/codegen.c b/Python/codegen.c index a975705576a167c..c18014cf1688895 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -224,7 +224,8 @@ static int codegen_call_helper_impl(compiler *c, location loc, int n, /* Args already pushed */ asdl_expr_seq *args, PyObject *injected_arg, - asdl_keyword_seq *keywords); + asdl_keyword_seq *keywords, + bool awaited); static int codegen_call_helper(compiler *c, location loc, int n, asdl_expr_seq *args, asdl_keyword_seq *keywords); @@ -487,12 +488,12 @@ codegen_addop_j(instr_sequence *seq, location loc, RETURN_IF_ERROR(codegen_visit_unused_ ## TYPE((C), (V))) static int -codegen_call_exit_with_nones(compiler *c, location loc) +codegen_call_exit_with_nones(compiler *c, location loc, bool awaited) { ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); ADDOP_LOAD_CONST(c, loc, Py_None); - ADDOP_I(c, loc, CALL, 3); + ADDOP_I(c, loc, awaited ? ASYNC_CALL : CALL, 3); return SUCCESS; } @@ -615,8 +616,9 @@ codegen_unwind_fblock(compiler *c, location *ploc, ADDOP_I(c, *ploc, SWAP, 3); ADDOP_I(c, *ploc, SWAP, 2); } - RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc)); - if (info->fb_type == COMPILE_FBLOCK_ASYNC_WITH) { + bool is_async = info->fb_type == COMPILE_FBLOCK_ASYNC_WITH; + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc, is_async)); + if (is_async) { ADDOP_I(c, *ploc, GET_AWAITABLE, 2); ADDOP(c, *ploc, PUSH_NULL); ADDOP_LOAD_CONST(c, *ploc, Py_None); @@ -1758,7 +1760,8 @@ codegen_class(compiler *c, stmt_ty s) RETURN_IF_ERROR_IN_SCOPE(c, codegen_call_helper_impl(c, loc, 2, s->v.ClassDef.bases, &_Py_STR(generic_base), - s->v.ClassDef.keywords)); + s->v.ClassDef.keywords, + false)); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 0); @@ -4152,7 +4155,7 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) // Return 1 if the method call was optimized, 0 if not, and -1 on error. static int -maybe_optimize_method_call(compiler *c, expr_ty e) +maybe_optimize_method_call(compiler *c, expr_ty e, bool awaited) { Py_ssize_t argsl, i, kwdsl; expr_ty meth = e->v.Call.func; @@ -4217,11 +4220,11 @@ maybe_optimize_method_call(compiler *c, expr_ty e) RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, kwds, kwdsl)); loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL_KW, argsl + kwdsl); + ADDOP_I(c, loc, awaited ? ASYNC_CALL_KW : CALL_KW, argsl + kwdsl); } else { loc = update_start_location_to_match_attr(c, LOC(e), meth); - ADDOP_I(c, loc, CALL, argsl); + ADDOP_I(c, loc, awaited ? ASYNC_CALL : CALL, argsl); } return 1; } @@ -4245,11 +4248,13 @@ codegen_validate_keywords(compiler *c, asdl_keyword_seq *keywords) return SUCCESS; } +/* awaited: the result of the call is the operand of an await, as in + * `await f(...)`, so it is emitted as ASYNC_CALL or ASYNC_CALL_KW. */ static int -codegen_call(compiler *c, expr_ty e) +codegen_call(compiler *c, expr_ty e, bool awaited) { RETURN_IF_ERROR(codegen_validate_keywords(c, e->v.Call.keywords)); - int ret = maybe_optimize_method_call(c, e); + int ret = maybe_optimize_method_call(c, e, awaited); if (ret < 0) { return ERROR; } @@ -4263,9 +4268,9 @@ codegen_call(compiler *c, expr_ty e) location loc = LOC(e->v.Call.func); ADDOP(c, loc, PUSH_NULL); loc = LOC(e); - ret = codegen_call_helper(c, loc, 0, - e->v.Call.args, - e->v.Call.keywords); + ret = codegen_call_helper_impl(c, loc, 0, + e->v.Call.args, NULL, + e->v.Call.keywords, awaited); USE_LABEL(c, skip_normal_call); return ret; } @@ -4458,7 +4463,8 @@ codegen_call_helper_impl(compiler *c, location loc, int n, /* Args already pushed */ asdl_expr_seq *args, PyObject *injected_arg, - asdl_keyword_seq *keywords) + asdl_keyword_seq *keywords, + bool awaited) { Py_ssize_t i, nseen, nelts, nkwelts; @@ -4497,10 +4503,10 @@ codegen_call_helper_impl(compiler *c, location loc, VISIT_SEQ(c, keyword, keywords); RETURN_IF_ERROR( codegen_call_simple_kw_helper(c, loc, keywords, nkwelts)); - ADDOP_I(c, loc, CALL_KW, n + nelts + nkwelts); + ADDOP_I(c, loc, awaited ? ASYNC_CALL_KW : CALL_KW, n + nelts + nkwelts); } else { - ADDOP_I(c, loc, CALL, n + nelts); + ADDOP_I(c, loc, awaited ? ASYNC_CALL : CALL, n + nelts); } return SUCCESS; @@ -4566,7 +4572,7 @@ codegen_call_helper(compiler *c, location loc, asdl_expr_seq *args, asdl_keyword_seq *keywords) { - return codegen_call_helper_impl(c, loc, n, args, NULL, keywords); + return codegen_call_helper_impl(c, loc, n, args, NULL, keywords, false); } /* List and set comprehensions work by being inlined at the location where @@ -5163,9 +5169,10 @@ codegen_comprehension(compiler *c, expr_ty e, int type, Py_CLEAR(co); VISIT(c, expr, outermost->iter); - ADDOP_I(c, loc, CALL, 0); + bool awaited = is_async_comprehension && type != COMP_GENEXP; + ADDOP_I(c, loc, awaited ? ASYNC_CALL : CALL, 0); - if (is_async_comprehension && type != COMP_GENEXP) { + if (awaited) { ADDOP_I(c, loc, GET_AWAITABLE, 0); ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -5310,7 +5317,7 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, SWAP, 2); ADDOP_I(c, loc, SWAP, 3); ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AENTER__); - ADDOP_I(c, loc, CALL, 0); + ADDOP_I(c, loc, ASYNC_CALL, 0); ADDOP_I(c, loc, GET_AWAITABLE, 1); ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -5347,7 +5354,7 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) /* For successful outcome: * call __exit__(None, None, None) */ - RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc, true)); ADDOP_I(c, loc, GET_AWAITABLE, 2); ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -5454,7 +5461,7 @@ codegen_with_inner(compiler *c, stmt_ty s, int pos) /* For successful outcome: * call __exit__(None, None, None) */ - RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); + RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc, false)); ADDOP(c, loc, POP_TOP); ADDOP_JUMP(c, loc, JUMP, exit); @@ -5550,7 +5557,12 @@ codegen_visit_expr_impl(compiler *c, expr_ty e, bool result_is_unused) ADD_YIELD_FROM(c, loc, 0); break; case Await_kind: - VISIT(c, expr, e->v.Await.value); + if (e->v.Await.value->kind == Call_kind) { + RETURN_IF_ERROR(codegen_call(c, e->v.Await.value, true)); + } + else { + VISIT(c, expr, e->v.Await.value); + } ADDOP_I(c, loc, GET_AWAITABLE, 0); ADDOP(c, loc, PUSH_NULL); ADDOP_LOAD_CONST(c, loc, Py_None); @@ -5559,7 +5571,7 @@ codegen_visit_expr_impl(compiler *c, expr_ty e, bool result_is_unused) case Compare_kind: return codegen_compare(c, e); case Call_kind: - return codegen_call(c, e); + return codegen_call(c, e, false); case Constant_kind: ADDOP_LOAD_CONST(c, loc, e->v.Constant.value); break; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bc85ccd9ddb4fc1..1ee676c07903c49 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8940,6 +8940,123 @@ break; } + case _SEND_GEN_FRAME_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + _PyStackRef receiver; + _PyStackRef gen_frame; + oparg = CURRENT_OPARG(); + v = stack_pointer[-1]; + receiver = stack_pointer[-3]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + assert( 2u + oparg <= UINT16_MAX); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _SEND_GEN_FRAME_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + _PyStackRef receiver; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = v; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = v; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + assert( 2u + oparg <= UINT16_MAX); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _SEND_GEN_FRAME_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + _PyStackRef receiver; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + v = _stack_item_1; + receiver = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = v; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = v; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + assert( 2u + oparg <= UINT16_MAX); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = _stack_item_0; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _SEND_GEN_FRAME_r33: { CHECK_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -8972,11 +9089,8 @@ STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = _stack_item_1; @@ -16203,10 +16317,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = stack_pointer[-1]; @@ -16242,10 +16353,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = _stack_item_0; @@ -16284,10 +16392,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = _stack_item_1; @@ -19918,6 +20023,161 @@ break; } + case _GUARD_ASYNC_CALL_TAIL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + SET_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_ASYNC_CALL_TAIL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_ASYNC_CALL_TAIL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_ASYNC_CALL_TAIL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _ASYNC_CALL_GEN_FRAME_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + new_frame = _stack_item_0; + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + _tos_cache2 = gen_frame; + _tos_cache1 = null; + _tos_cache0 = coro; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _MAYBE_EXPAND_METHOD_KW_r11: { @@ -20718,35 +20978,35 @@ CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + frame->instr_ptr++; + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); + gen->gi_frame_state = FRAME_CREATED; + _Py_LeaveRecursiveCallPy(tstate); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); _tos_cache0 = res; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ee9b64107bb9f24..dd631d4c1921f6c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -18,6 +18,1299 @@ /* BEGIN INSTRUCTIONS */ + TARGET(ASYNC_CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL); + PREDICTED_ASYNC_CALL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = ASYNC_CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + // _SPECIALIZE_ASYNC_CALL + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_AsyncCall(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(ASYNC_CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL, + opcode == INSTRUMENTED_CALL || opcode == INSTRUMENTED_ASYNC_CALL, + frame, + this_instr, + tstate); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = check_periodics_at_end(tstate, frame); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_BOUND_METHOD_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW); + PREDICTED_ASYNC_CALL_KW:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = ASYNC_CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + // _SPECIALIZE_ASYNC_CALL_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _Py_Specialize_AsyncCallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(ASYNC_CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + kwnames, + opcode == INSTRUMENTED_CALL_KW || opcode == INSTRUMENTED_ASYNC_CALL_KW, + frame, + this_instr, + tstate); + _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW_BOUND_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(callable_s); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_ASYNC_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(BINARY_OP) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP; @@ -1908,7 +3201,7 @@ arguments, total_args, PyStackRef_NULL, - opcode == INSTRUMENTED_CALL, + opcode == INSTRUMENTED_CALL || opcode == INSTRUMENTED_ASYNC_CALL, frame, this_instr, tstate); @@ -3479,7 +4772,7 @@ arguments, total_args, kwnames, - opcode == INSTRUMENTED_CALL_KW, + opcode == INSTRUMENTED_CALL_KW || opcode == INSTRUMENTED_ASYNC_CALL_KW, frame, this_instr, tstate); @@ -6431,10 +7724,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME @@ -7010,83 +8300,346 @@ if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - res_o = _PyEval_LazyImportFrom( - tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); + res_o = _PyEval_LazyImportFrom( + tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); + _PyFrame_StackPointerInvalidate(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyEval_ImportFrom( + tstate, PyStackRef_AsPyObjectBorrow(from), name); + _PyFrame_StackPointerInvalidate(frame); + } + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(IMPORT_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_NAME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyObject *res_o; + if (!(oparg & 0x02)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), + LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level), + oparg & 0x01); + _PyFrame_StackPointerInvalidate(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), + LOCALS(), name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + _PyFrame_StackPointerInvalidate(frame); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + DISPATCH(); + } + + TARGET(INSTRUMENTED_ASYNC_CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_ASYNC_CALL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_ASYNC_CALL); + opcode = INSTRUMENTED_ASYNC_CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef func; + _PyStackRef maybe_self; + _PyStackRef *args; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = self_or_null; + func = callable; + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL, + opcode == INSTRUMENTED_CALL || opcode == INSTRUMENTED_ASYNC_CALL, + frame, + this_instr, + tstate); _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - else { + // _CHECK_PERIODIC_AT_END + { + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - res_o = _PyEval_ImportFrom( - tstate, PyStackRef_AsPyObjectBorrow(from), name); + int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } } - if (res_o == NULL) { - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } - TARGET(IMPORT_NAME) { + TARGET(INSTRUMENTED_ASYNC_CALL_KW) { #if _Py_TAIL_CALL_INTERP - int opcode = IMPORT_NAME; + int opcode = INSTRUMENTED_ASYNC_CALL_KW; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - _PyStackRef level; - _PyStackRef fromlist; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_ASYNC_CALL_KW); + opcode = INSTRUMENTED_ASYNC_CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; _PyStackRef res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyObject *res_o; - if (!(oparg & 0x02)) { + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + } + // _MONITOR_CALL_KW + { + args = &stack_pointer[-1 - oparg]; + int is_meth = !PyStackRef_IsNull(self_or_null); + PyObject *arg; + if (is_meth) { + arg = PyStackRef_AsPyObjectBorrow(self_or_null); + } + else if (args) { + arg = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg = &_PyInstrumentation_MISSING; + } + PyObject *function = PyStackRef_AsPyObjectBorrow(callable); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), - LOCALS(), name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level), - oparg & 0x01); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); _PyFrame_StackPointerInvalidate(frame); + if (err) { + JUMP_TO_LABEL(error); + } } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + !IS_PEP523_HOOKED(tstate) && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), - LOCALS(), name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); + PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( + callable, + arguments, + total_args, + kwnames, + opcode == INSTRUMENTED_CALL_KW || opcode == INSTRUMENTED_ASYNC_CALL_KW, + frame, + this_instr, + tstate); _PyFrame_StackPointerInvalidate(frame); + if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyStackRef tmp = fromlist; - fromlist = PyStackRef_NULL; - stack_pointer[-1] = fromlist; - PyStackRef_CLOSE(tmp); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - tmp = level; - level = PyStackRef_NULL; - stack_pointer[-2] = level; - PyStackRef_CLOSE(tmp); - _PyFrame_StackPointerInvalidate(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - if (res_o == NULL) { - JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7198,7 +8751,7 @@ arguments, total_args, PyStackRef_NULL, - opcode == INSTRUMENTED_CALL, + opcode == INSTRUMENTED_CALL || opcode == INSTRUMENTED_ASYNC_CALL, frame, this_instr, tstate); @@ -7518,7 +9071,7 @@ arguments, total_args, kwnames, - opcode == INSTRUMENTED_CALL_KW, + opcode == INSTRUMENTED_CALL_KW || opcode == INSTRUMENTED_ASYNC_CALL_KW, frame, this_instr, tstate); @@ -11437,34 +12990,34 @@ next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + frame->instr_ptr++; + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); + gen->gi_frame_state = FRAME_CREATED; + _Py_LeaveRecursiveCallPy(tstate); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -11718,11 +13271,8 @@ STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 3c9b2afcde100f3..f0ef4b7b66d373a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -78,6 +78,10 @@ static const int8_t EVENT_FOR_OPCODE[256] = { [INSTRUMENTED_CALL] = PY_MONITORING_EVENT_CALL, [CALL_KW] = PY_MONITORING_EVENT_CALL, [INSTRUMENTED_CALL_KW] = PY_MONITORING_EVENT_CALL, + [ASYNC_CALL] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_ASYNC_CALL] = PY_MONITORING_EVENT_CALL, + [ASYNC_CALL_KW] = PY_MONITORING_EVENT_CALL, + [INSTRUMENTED_ASYNC_CALL_KW] = PY_MONITORING_EVENT_CALL, [CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = PY_MONITORING_EVENT_CALL, [LOAD_SUPER_ATTR] = PY_MONITORING_EVENT_CALL, @@ -115,6 +119,8 @@ static const uint8_t DE_INSTRUMENT[256] = { [INSTRUMENTED_RETURN_VALUE] = RETURN_VALUE, [INSTRUMENTED_CALL] = CALL, [INSTRUMENTED_CALL_KW] = CALL_KW, + [INSTRUMENTED_ASYNC_CALL] = ASYNC_CALL, + [INSTRUMENTED_ASYNC_CALL_KW] = ASYNC_CALL_KW, [INSTRUMENTED_CALL_FUNCTION_EX] = CALL_FUNCTION_EX, [INSTRUMENTED_YIELD_VALUE] = YIELD_VALUE, [INSTRUMENTED_JUMP_FORWARD] = JUMP_FORWARD, @@ -139,6 +145,10 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, [CALL_KW] = INSTRUMENTED_CALL_KW, [INSTRUMENTED_CALL_KW] = INSTRUMENTED_CALL_KW, + [ASYNC_CALL] = INSTRUMENTED_ASYNC_CALL, + [INSTRUMENTED_ASYNC_CALL] = INSTRUMENTED_ASYNC_CALL, + [ASYNC_CALL_KW] = INSTRUMENTED_ASYNC_CALL_KW, + [INSTRUMENTED_ASYNC_CALL_KW] = INSTRUMENTED_ASYNC_CALL_KW, [CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, [YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4b..fdfce2e514702f6 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -42,6 +42,8 @@ static void *opcode_targets_table[256] = { &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, &&TARGET_WITH_EXCEPT_START, + &&TARGET_ASYNC_CALL, + &&TARGET_ASYNC_CALL_KW, &&TARGET_BINARY_OP, &&TARGET_BUILD_INTERPOLATION, &&TARGET_BUILD_LIST, @@ -126,9 +128,13 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, + &&TARGET_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_ASYNC_CALL_BOUND_METHOD_GENERAL, + &&TARGET_ASYNC_CALL_KW_BOUND_METHOD, + &&TARGET_ASYNC_CALL_KW_PY, + &&TARGET_ASYNC_CALL_PY_EXACT_ARGS, + &&TARGET_ASYNC_CALL_PY_GENERAL, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, @@ -225,14 +231,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -250,7 +248,9 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_END_ASYNC_FOR, &&TARGET_INSTRUMENTED_LOAD_SUPER_ATTR, &&TARGET_INSTRUMENTED_CALL, + &&TARGET_INSTRUMENTED_ASYNC_CALL, &&TARGET_INSTRUMENTED_CALL_KW, + &&TARGET_INSTRUMENTED_ASYNC_CALL_KW, &&TARGET_INSTRUMENTED_CALL_FUNCTION_EX, &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_LINE, @@ -376,8 +376,8 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -478,20 +478,20 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, @@ -531,6 +531,14 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PA static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_PY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_GENERAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); @@ -629,6 +637,8 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_ITER_VIRTUAL(TAIL_CALL_PARAM static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_ASYNC_CALL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_ASYNC_CALL_KW(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); @@ -774,6 +784,14 @@ JUMP_TO_LABEL(error); } static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { + [ASYNC_CALL] = _TAIL_CALL_ASYNC_CALL, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL, + [ASYNC_CALL_KW] = _TAIL_CALL_ASYNC_CALL_KW, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_ASYNC_CALL_KW_PY, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_ASYNC_CALL_PY_GENERAL, [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -872,6 +890,8 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [GET_LEN] = _TAIL_CALL_GET_LEN, [IMPORT_FROM] = _TAIL_CALL_IMPORT_FROM, [IMPORT_NAME] = _TAIL_CALL_IMPORT_NAME, + [INSTRUMENTED_ASYNC_CALL] = _TAIL_CALL_INSTRUMENTED_ASYNC_CALL, + [INSTRUMENTED_ASYNC_CALL_KW] = _TAIL_CALL_INSTRUMENTED_ASYNC_CALL_KW, [INSTRUMENTED_CALL] = _TAIL_CALL_INSTRUMENTED_CALL, [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX, [INSTRUMENTED_CALL_KW] = _TAIL_CALL_INSTRUMENTED_CALL_KW, @@ -1005,8 +1025,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, - [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1016,22 +1034,22 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, [228] = _TAIL_CALL_UNKNOWN_OPCODE, [229] = _TAIL_CALL_UNKNOWN_OPCODE, [230] = _TAIL_CALL_UNKNOWN_OPCODE, - [231] = _TAIL_CALL_UNKNOWN_OPCODE, - [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [ASYNC_CALL] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, @@ -1130,6 +1148,8 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [GET_LEN] = _TAIL_CALL_TRACE_RECORD, [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_ASYNC_CALL] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_ASYNC_CALL_KW] = _TAIL_CALL_TRACE_RECORD, [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACE_RECORD, @@ -1263,8 +1283,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, - [117] = _TAIL_CALL_UNKNOWN_OPCODE, - [118] = _TAIL_CALL_UNKNOWN_OPCODE, [119] = _TAIL_CALL_UNKNOWN_OPCODE, [120] = _TAIL_CALL_UNKNOWN_OPCODE, [121] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1274,19 +1292,11 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, [228] = _TAIL_CALL_UNKNOWN_OPCODE, [229] = _TAIL_CALL_UNKNOWN_OPCODE, [230] = _TAIL_CALL_UNKNOWN_OPCODE, - [231] = _TAIL_CALL_UNKNOWN_OPCODE, - [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/optimizer.c b/Python/optimizer.c index e05adb344c8d06d..55675b7ef4acabe 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -744,7 +744,11 @@ _PyJit_translate_single_bytecode_to_trace( assert(!OPCODE_HAS_DEOPT(opcode)); } - bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode); + // RETURN_GENERATOR in a coroutine that ASYNC_CALL_* already started + // stays in its frame: it only pushes None (see _ASYNC_CALL_GEN_FRAME). + bool coro_running = opcode == RETURN_GENERATOR && + frame == tracer->prev_state.instr_frame; + bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode) && !coro_running; if (has_dynamic_jump_taken && !needs_guard_ip) { DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]); goto unsupported; @@ -908,6 +912,12 @@ _PyJit_translate_single_bytecode_to_trace( * start with RESUME_CHECK */ ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); break; + case RETURN_GENERATOR: + if (coro_running) { + ADD_TO_TRACE(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_None, target); + break; + } + _Py_FALLTHROUGH; default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6284885c428832b..9bc41b022189d71 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1528,6 +1528,18 @@ dummy_func(void) { gen_frame = PyJitRef_WrapInvalid(new_frame); } + op(_GUARD_ASYNC_CALL_TAIL, (--)) { + // Only needed in tier 1: executors are invalidated + // when the code is instrumented. + ADD_OP(_NOP, 0, 0); + } + + op(_ASYNC_CALL_GEN_FRAME, (new_frame -- coro, null, gen_frame)) { + coro = sym_new_type(ctx, &PyCoro_Type); + null = sym_new_null(ctx); + gen_frame = new_frame; + } + op(_SEND_GEN_FRAME, (receiver, null, v -- receiver, null, gen_frame)) { _Py_UOpsAbstractFrame *new_frame = frame_new_from_symbol(ctx, receiver, NULL, 0); if (new_frame == NULL) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9cd20c495db4418..0d36e5ac09cc8f1 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -5062,6 +5062,29 @@ break; } + case _GUARD_ASYNC_CALL_TAIL: { + ADD_OP(_NOP, 0, 0); + break; + } + + case _ASYNC_CALL_GEN_FRAME: { + JitOptRef new_frame; + JitOptRef coro; + JitOptRef null; + JitOptRef gen_frame; + new_frame = stack_pointer[-1]; + coro = sym_new_type(ctx, &PyCoro_Type); + null = sym_new_null(ctx); + gen_frame = new_frame; + CHECK_STACK_BOUNDS(2); + stack_pointer[-1] = coro; + stack_pointer[0] = null; + stack_pointer[1] = gen_frame; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 */ case _MAYBE_EXPAND_METHOD_KW: { diff --git a/Python/pystats.c b/Python/pystats.c index 6862521f659f202..f2e64d033c6911e 100644 --- a/Python/pystats.c +++ b/Python/pystats.c @@ -127,6 +127,8 @@ _Py_GetSpecializationStats(void) { err += add_stat_dict(src, stats, JUMP_BACKWARD, "jump_backward"); err += add_stat_dict(src, stats, CALL, "call"); err += add_stat_dict(src, stats, CALL_KW, "call_kw"); + err += add_stat_dict(src, stats, ASYNC_CALL, "async_call"); + err += add_stat_dict(src, stats, ASYNC_CALL_KW, "async_call_kw"); err += add_stat_dict(src, stats, BINARY_OP, "binary_op"); err += add_stat_dict(src, stats, COMPARE_OP, "compare_op"); err += add_stat_dict(src, stats, UNPACK_SEQUENCE, "unpack_sequence"); diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 98abe3d0505e202..56a01460b69fb22 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -190,9 +190,17 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_METHOD_DESCRIPTOR_NOARGS] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_METHOD_DESCRIPTOR_FAST] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_PY_EXACT_ARGS] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_PY_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_KW_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW_BOUND_METHOD] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [ASYNC_CALL_KW] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [ASYNC_CALL_KW_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [ASYNC_CALL_KW_BOUND_METHOD] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW_NON_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_FUNCTION_EX] = {1, {_RECORD_4OS_INDEX}}, [CALL_EX_PY] = {1, {_RECORD_4OS_INDEX}}, @@ -241,8 +249,14 @@ const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = { [CALL_METHOD_DESCRIPTOR_O] = {1, 0, {0}}, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = {1, 0, {0}}, [CALL_METHOD_DESCRIPTOR_NOARGS] = {1, 0, {0}}, + [ASYNC_CALL_PY_EXACT_ARGS] = {1, 0, {0}}, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = {1, 1, {0}}, + [ASYNC_CALL_PY_GENERAL] = {1, 0, {0}}, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = {1, 1, {0}}, [CALL_KW_PY] = {1, 0, {0}}, [CALL_KW_BOUND_METHOD] = {1, 0, {0}}, + [ASYNC_CALL_KW_PY] = {1, 0, {0}}, + [ASYNC_CALL_KW_BOUND_METHOD] = {1, 0, {0}}, [CALL_EX_PY] = {1, 0, {0}}, [BINARY_OP] = {2, 2, {1, 0}}, }; diff --git a/Python/specialize.c b/Python/specialize.c index 067bae59f6073b7..ffb1fb15780924f 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -262,6 +262,7 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters #define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30 #define SPEC_FAIL_CALL_METACLASS 31 #define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32 +#define SPEC_FAIL_CALL_NOT_COROUTINE 33 /* COMPARE_OP */ #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12 @@ -1751,40 +1752,70 @@ specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, return 0; } +/* Can the ASYNC_CALL or ASYNC_CALL_KW at "instr", a call to a function + * with the code "code", be specialized to one of the ASYNC_CALL_* + * instructions? These start the new coroutine in place of the SEND that + * awaits it. GET_AWAITABLE never fails for a new coroutine. */ +static bool +can_start_coroutine(PyCodeObject *code, _Py_CODEUNIT *instr) +{ + int coro_flags = code->co_flags & + (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); + if (coro_flags != CO_COROUTINE) { + return false; + } + _Py_CODEUNIT *next = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[instr->op.code]]; + return _PyCode_AwaitedCallSend(next) != NULL; +} + static int specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, - bool bound_method) + bool bound_method, bool async_call) { _PyCallCache *cache = (_PyCallCache *)(instr + 1); PyCodeObject *code = (PyCodeObject *)func->func_code; int kind = function_kind(code); /* Don't specialize if PEP 523 is active */ if (!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); + SPECIALIZATION_FAIL(_PyOpcode_Deopt[instr->op.code], SPEC_FAIL_CALL_PEP_523); return -1; } if (func->vectorcall != _PyFunction_Vectorcall) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_VECTORCALL); + SPECIALIZATION_FAIL(_PyOpcode_Deopt[instr->op.code], SPEC_FAIL_CALL_VECTORCALL); return -1; } int argcount = -1; if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); + SPECIALIZATION_FAIL(_PyOpcode_Deopt[instr->op.code], SPEC_FAIL_CODE_NOT_OPTIMIZED); return -1; } if (kind == SIMPLE_FUNCTION) { argcount = code->co_argcount; } + if (async_call && !can_start_coroutine(code, instr)) { + SPECIALIZATION_FAIL(_PyOpcode_Deopt[instr->op.code], SPEC_FAIL_CALL_NOT_COROUTINE); + return -1; + } int version = _PyFunction_GetVersionForCurrentState(func); if (!_PyFunction_IsVersionValid(version)) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); + SPECIALIZATION_FAIL(_PyOpcode_Deopt[instr->op.code], SPEC_FAIL_OUT_OF_VERSIONS); return -1; } write_u32(cache->func_version, version); uint8_t opcode; if (argcount == nargs + bound_method) { - opcode = - bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; + if (async_call) { + opcode = bound_method ? + ASYNC_CALL_BOUND_METHOD_EXACT_ARGS : ASYNC_CALL_PY_EXACT_ARGS; + } + else { + opcode = bound_method ? + CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; + } + } + else if (async_call) { + opcode = bound_method ? + ASYNC_CALL_BOUND_METHOD_GENERAL : ASYNC_CALL_PY_GENERAL; } else { opcode = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL; @@ -1793,34 +1824,44 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return 0; } - static int specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, - bool bound_method) + bool bound_method, bool async_call) { _PyCallCache *cache = (_PyCallCache *)(instr + 1); PyCodeObject *code = (PyCodeObject *)func->func_code; int kind = function_kind(code); /* Don't specialize if PEP 523 is active */ if (!_PyInterpreterState_IsSpecializationEnabled(_PyInterpreterState_GET())) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); + SPECIALIZATION_FAIL(_PyOpcode_Deopt[instr->op.code], SPEC_FAIL_CALL_PEP_523); return -1; } if (func->vectorcall != _PyFunction_Vectorcall) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_VECTORCALL); + SPECIALIZATION_FAIL(_PyOpcode_Deopt[instr->op.code], SPEC_FAIL_CALL_VECTORCALL); return -1; } if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); + SPECIALIZATION_FAIL(_PyOpcode_Deopt[instr->op.code], SPEC_FAIL_CODE_NOT_OPTIMIZED); + return -1; + } + if (async_call && !can_start_coroutine(code, instr)) { + SPECIALIZATION_FAIL(_PyOpcode_Deopt[instr->op.code], SPEC_FAIL_CALL_NOT_COROUTINE); return -1; } int version = _PyFunction_GetVersionForCurrentState(func); if (!_PyFunction_IsVersionValid(version)) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); + SPECIALIZATION_FAIL(_PyOpcode_Deopt[instr->op.code], SPEC_FAIL_OUT_OF_VERSIONS); return -1; } write_u32(cache->func_version, version); - specialize(instr, bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY); + uint8_t opcode; + if (async_call) { + opcode = bound_method ? ASYNC_CALL_KW_BOUND_METHOD : ASYNC_CALL_KW_PY; + } + else { + opcode = bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY; + } + specialize(instr, opcode); return 0; } @@ -1881,7 +1922,7 @@ _Py_Specialize_Call(_PyStackRef callable_st, _PyStackRef self_or_null_st, _Py_CO fail = specialize_c_call(callable, instr, nargs); } else if (PyFunction_Check(callable)) { - fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, false); + fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, false, false); } else if (PyType_Check(callable)) { fail = specialize_class_call(callable, instr, nargs); @@ -1894,7 +1935,7 @@ _Py_Specialize_Call(_PyStackRef callable_st, _PyStackRef self_or_null_st, _Py_CO else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; if (PyFunction_Check(func)) { - fail = specialize_py_call((PyFunctionObject *)func, instr, nargs, true); + fail = specialize_py_call((PyFunctionObject *)func, instr, nargs, true, false); } else { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); @@ -1920,12 +1961,12 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL_KW); int fail; if (PyFunction_Check(callable)) { - fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false); + fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false, false); } else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; if (PyFunction_Check(func)) { - fail = specialize_py_call_kw((PyFunctionObject *)func, instr, nargs, true); + fail = specialize_py_call_kw((PyFunctionObject *)func, instr, nargs, true, false); } else { SPECIALIZATION_FAIL(CALL_KW, SPEC_FAIL_CALL_BOUND_METHOD); @@ -1941,6 +1982,70 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) } } +/* Only calls to coroutine functions are specialized: the result of any + * other call is awaited by the instructions following ASYNC_CALL. */ +void +_Py_Specialize_AsyncCall(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) +{ + PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); + + assert(ENABLE_SPECIALIZATION); + assert(_PyOpcode_Caches[ASYNC_CALL] == INLINE_CACHE_ENTRIES_ASYNC_CALL); + assert(_Py_OPCODE(*instr) != INSTRUMENTED_ASYNC_CALL); + int fail; + if (PyFunction_Check(callable)) { + fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, false, true); + } + else if (PyMethod_Check(callable)) { + PyObject *func = ((PyMethodObject *)callable)->im_func; + if (PyFunction_Check(func)) { + fail = specialize_py_call((PyFunctionObject *)func, instr, nargs, true, true); + } + else { + SPECIALIZATION_FAIL(ASYNC_CALL, SPEC_FAIL_CALL_BOUND_METHOD); + fail = -1; + } + } + else { + SPECIALIZATION_FAIL(ASYNC_CALL, SPEC_FAIL_CALL_NOT_COROUTINE); + fail = -1; + } + if (fail) { + unspecialize(instr); + } +} + +void +_Py_Specialize_AsyncCallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) +{ + PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); + + assert(ENABLE_SPECIALIZATION); + assert(_PyOpcode_Caches[ASYNC_CALL_KW] == INLINE_CACHE_ENTRIES_ASYNC_CALL_KW); + assert(_Py_OPCODE(*instr) != INSTRUMENTED_ASYNC_CALL_KW); + int fail; + if (PyFunction_Check(callable)) { + fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false, true); + } + else if (PyMethod_Check(callable)) { + PyObject *func = ((PyMethodObject *)callable)->im_func; + if (PyFunction_Check(func)) { + fail = specialize_py_call_kw((PyFunctionObject *)func, instr, nargs, true, true); + } + else { + SPECIALIZATION_FAIL(ASYNC_CALL_KW, SPEC_FAIL_CALL_BOUND_METHOD); + fail = -1; + } + } + else { + SPECIALIZATION_FAIL(ASYNC_CALL_KW, SPEC_FAIL_CALL_NOT_COROUTINE); + fail = -1; + } + if (fail) { + unspecialize(instr); + } +} + #ifdef Py_STATS static int binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index c3e0b0dbff21993..01d0ecfb5a47abf 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -603,6 +603,9 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyFunction_SetVersion", "_PyGen_GetGeneratorFromFrame", "gen_try_set_executing", + "gen_link_frame", + "_PyCode_AwaitedCallSend", + "gen_take_frame", "_PyInterpreterState_GET", "_PyList_AppendTakeRef", "_PyList_ITEMS", From 27d92561256063cf207879bc5b566e0b1a0b3e14 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Wed, 18 Nov 2026 15:48:12 +0100 Subject: [PATCH 2/2] Create the coroutine object of a specialized ASYNC_CALL lazily The ASYNC_CALL_* specializations no longer create the coroutine object when they start the body of the coroutine function. The body runs in the frame of the call, marked lazy_coro, with its own exception state pushed on the data stack right after the frame, while the caller is left as if SEND_GEN had pushed the frame of a coroutine. A body that returns or raises before it suspends never gets a coroutine object. The first YIELD_VALUE in such a frame creates the coroutine, moves the frame into it and replaces the None that stood in for it on the stack of the caller; from then on it is an ordinary coroutine. RETURN_GENERATOR in a lazy frame only pushes the None that the following POP_TOP discards. While coroutine origin tracking is on, the object has to exist at creation, so ASYNC_CALL is not specialized and the guard of the specializations deoptimizes. Until it first suspends, frame.f_generator is None for such a coroutine. asyncio.graph classifies frames by their code flags instead. This replaces the look-ahead at the caller's next instruction of RETURN_GENERATOR proposed in gh-161171 with the ASYNC_CALL opcode. Co-Authored-By: Claude Fable 5.1 --- Doc/c-api/frame.rst | 4 + Doc/library/inspect.rst | 4 +- Doc/reference/datamodel.rst | 14 +- Doc/whatsnew/3.16.rst | 31 + Include/internal/pycore_ceval.h | 7 +- Include/internal/pycore_interpframe.h | 4 + Include/internal/pycore_interpframe_structs.h | 6 +- Include/internal/pycore_opcode_metadata.h | 14 +- Include/internal/pycore_uop_ids.h | 4 +- Include/internal/pycore_uop_metadata.h | 16 +- InternalDocs/generators.md | 22 +- Lib/asyncio/graph.py | 63 +- Lib/test/test_asyncio/test_graph.py | 47 +- Lib/test/test_capi/test_opt.py | 51 +- Lib/test/test_coroutines.py | 559 +++++++++++++++++- Lib/test/test_opcache.py | 35 ++ Modules/_testinternalcapi/test_cases.c.h | 325 ++++++---- Objects/genobject.c | 82 ++- Python/bytecodes.c | 121 ++-- Python/ceval.c | 48 ++ Python/executor_cases.c.h | 103 +++- Python/generated_cases.c.h | 325 ++++++---- Python/intrinsics.c | 6 +- Python/optimizer.c | 5 +- Python/optimizer_bytecodes.c | 14 +- Python/optimizer_cases.c.h | 15 +- Python/specialize.c | 9 +- 27 files changed, 1481 insertions(+), 453 deletions(-) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index ea6846306273552..7ded5326861f6dd 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -85,6 +85,10 @@ See also :ref:`Reflection `. .. versionadded:: 3.11 + .. versionchanged:: 3.16 + May return ``NULL`` for a coroutine frame before its first suspension; + see :attr:`frame.f_generator`. + .. c:function:: PyObject* PyFrame_GetGlobals(PyFrameObject *frame) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 51c465e8f00b5ee..328bfea8a372abd 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -151,7 +151,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | coroutine object that | | | | owns this frame, or | | | | ``None`` if the frame is | -| | | of a regular function | +| | | of a regular function or | +| | | the generator is not | +| | | materialized yet | +-----------------+-------------------+---------------------------+ | | f_trace | tracing function for this | | | | frame, or ``None`` | diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 02663ed465f424e..07fa9cf5441e643 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1736,10 +1736,22 @@ Special read-only attributes * - .. attribute:: frame.f_generator - The :term:`generator` or :term:`coroutine` object that owns this frame, - or ``None`` if the frame is a normal function. + or ``None`` if the frame is a normal function or if the generator + is not materialized yet. + + .. impl-detail:: + + A coroutine that is awaited as soon as it is called + (``await coro_func()``) may run without a coroutine object until + it first suspends, so this attribute is ``None`` before that. + Use :attr:`~codeobject.co_flags` of :attr:`frame.f_code` to tell + coroutine frames apart from normal function frames. .. versionadded:: 3.14 + .. versionchanged:: 3.16 + May be ``None`` for a coroutine frame before its first suspension. + .. index:: single: f_trace (frame attribute) single: f_trace_lines (frame attribute) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 4af5d3c0bd072e2..d4030b4d50867a5 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1015,6 +1015,17 @@ zlib Optimizations ============= +Interpreter +----------- + +* A coroutine function that is awaited as soon as it is called + (``await coro_func()``, ``async with``) no longer allocates a coroutine + object unless its body suspends, once the call has been specialized. + Until it first suspends, :attr:`frame.f_generator` is ``None`` for such a + coroutine; see :ref:`whatsnew316-porting`. + This does not apply while coroutine origin tracking is enabled. + (Contributed by Andrew Svetlov in :gh:`161170`.) + datetime and time ----------------- @@ -1273,12 +1284,23 @@ New deprecations .. include:: ../deprecations/pending-removal-in-future.rst +.. _whatsnew316-porting: + Porting to Python 3.16 ====================== This section lists previously described changes and other bugfixes that may require changes to your code. +* :attr:`frame.f_generator` and :c:func:`PyFrame_GetGenerator` may now + return ``None``/``NULL`` for a coroutine frame before its first + suspension, because a coroutine that is awaited as soon as it is called + (``await coro_func()``) may run without a coroutine object until then. + Code that used ``frame.f_generator is None`` to detect frames of normal + functions should check ``frame.f_code.co_flags`` for + :data:`~inspect.CO_COROUTINE` instead. + (Contributed by Andrew Svetlov in :gh:`161170`.) + * In :mod:`tkinter`, the *name* parameter of the :meth:`~tkinter.Misc.wait_variable`, :meth:`~tkinter.Misc.setvar` and :meth:`~tkinter.Misc.getvar` methods and the *value* parameter of @@ -1334,6 +1356,15 @@ that may require changes to your code. (Contributed by An Long in :gh:`86768`.) +CPython bytecode changes +======================== + +* Added :opcode:`ASYNC_CALL` and :opcode:`ASYNC_CALL_KW`, which the compiler + emits instead of :opcode:`CALL` and :opcode:`CALL_KW` for a call whose + result is awaited at once. + (Contributed by Andrew Svetlov in :gh:`161170`.) + + Build changes ============= diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index f60605e609c3bee..6d91c625c27212b 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -267,7 +267,12 @@ extern int _PyEval_EnsureBuiltinsWithModule( PyObject **p_builtins); PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); -PyAPI_FUNC(PyObject *)_Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame); +// Lazily created coroutines (see _ASYNC_CALL_LAZY_FRAME in bytecodes.c). +// Exported for _testinternalcapi and external JIT support. +PyAPI_FUNC(PyObject *) _PyCoro_MaterializeLazy(_PyInterpreterFrame *frame); +PyAPI_FUNC(_PyInterpreterFrame *) _PyEval_MoveLazyCoroFrame(PyThreadState *tstate, + PyGenObject *gen, + _PyInterpreterFrame *frame); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index a85fa9bd32853c8..ea280a324e758a8 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -152,6 +152,8 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * dest->stackpointer = dest->localsplus + stacktop; // visited is GC bookkeeping for the current stack walk, not frame state. dest->visited = 0; + // A lazily started coroutine stops being lazy once its frame is copied. + dest->lazy_coro = 0; #ifdef Py_DEBUG dest->stackpointer_valid = src->stackpointer_valid; dest->lltrace = src->lltrace; @@ -208,6 +210,7 @@ _PyFrame_Initialize( frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->lazy_coro = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; @@ -441,6 +444,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int #endif frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; + frame->lazy_coro = 0; #ifdef Py_DEBUG frame->stackpointer_valid = 1; frame->lltrace = 0; diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 0d6a37cc2c1a17a..0b593623dc09a85 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -44,9 +44,13 @@ struct _PyInterpreterFrame { char owner; uint8_t visited:1; uint8_t external:1; /* Set when the frame was pushed by something other than the VM */ + /* Set for a thread-owned frame running a coroutine body whose coroutine + * object has not been created yet (see RETURN_GENERATOR in bytecodes.c). + * Its own _PyErr_StackItem lives on the data stack right after it. */ + uint8_t lazy_coro:1; #ifdef Py_DEBUG uint8_t stackpointer_valid:1; - uint8_t lltrace:5; + uint8_t lltrace:4; #endif /* Locals and stack */ _PyStackRef localsplus[1]; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e301dff3cc6fd14..48fee7ffecca508 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1380,7 +1380,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1405,12 +1405,12 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { - [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 12, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [ASYNC_CALL_BOUND_METHOD_GENERAL] = { .nuops = 9, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [ASYNC_CALL_KW_BOUND_METHOD] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [ASYNC_CALL_KW_PY] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [ASYNC_CALL_PY_EXACT_ARGS] = { .nuops = 10, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [ASYNC_CALL_PY_GENERAL] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 12, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_LAZY_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = { .nuops = 9, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_LAZY_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_KW_BOUND_METHOD] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_LAZY_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_KW_PY] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_LAZY_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_PY_EXACT_ARGS] = { .nuops = 10, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_LAZY_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_PY_GENERAL] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_LAZY_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [BINARY_OP] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _BINARY_OP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [BINARY_OP_ADD_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_ADD_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index a04629e5dd0ed2f..853ce04d6507922 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -13,7 +13,7 @@ enum { _EXIT_TRACE = 300, _SET_IP, _ALLOCATE_OBJECT, - _ASYNC_CALL_GEN_FRAME, + _ASYNC_CALL_LAZY_FRAME, _BINARY_OP, _BINARY_OP_ADD_FLOAT, _BINARY_OP_ADD_FLOAT_INPLACE, @@ -445,7 +445,7 @@ enum { enum { _ALLOCATE_OBJECT_r00 = MAX_UOP_ID + 1, - _ASYNC_CALL_GEN_FRAME_r13, + _ASYNC_CALL_LAZY_FRAME_r13, _BINARY_OP_r23, _BINARY_OP_ADD_FLOAT_r03, _BINARY_OP_ADD_FLOAT_r13, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 53cf9ed65fa47cf..5c358bfc046204b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -174,7 +174,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SEND_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_GUARD_3OS_ASYNC_GEN_ASEND] = HAS_EXIT_FLAG, [_SEND_ASYNC_GEN_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -353,7 +353,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_ASYNC_CALL_TAIL] = HAS_DEOPT_FLAG, - [_ASYNC_CALL_GEN_FRAME] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, + [_ASYNC_CALL_LAZY_FRAME] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -3311,11 +3311,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_ASYNC_CALL_TAIL_r33 }, }, }, - [_ASYNC_CALL_GEN_FRAME] = { + [_ASYNC_CALL_LAZY_FRAME] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 3, 3, _ASYNC_CALL_GEN_FRAME_r13 }, + { 3, 3, _ASYNC_CALL_LAZY_FRAME_r13 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -4669,7 +4669,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_ASYNC_CALL_TAIL_r11] = _GUARD_ASYNC_CALL_TAIL, [_GUARD_ASYNC_CALL_TAIL_r22] = _GUARD_ASYNC_CALL_TAIL, [_GUARD_ASYNC_CALL_TAIL_r33] = _GUARD_ASYNC_CALL_TAIL, - [_ASYNC_CALL_GEN_FRAME_r13] = _ASYNC_CALL_GEN_FRAME, + [_ASYNC_CALL_LAZY_FRAME_r13] = _ASYNC_CALL_LAZY_FRAME, [_MAYBE_EXPAND_METHOD_KW_r11] = _MAYBE_EXPAND_METHOD_KW, [_PY_FRAME_KW_r11] = _PY_FRAME_KW, [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, @@ -4895,8 +4895,8 @@ const uint16_t _PyUop_SpillsAndReloads[4][4] = { const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_ALLOCATE_OBJECT] = "_ALLOCATE_OBJECT", [_ALLOCATE_OBJECT_r00] = "_ALLOCATE_OBJECT_r00", - [_ASYNC_CALL_GEN_FRAME] = "_ASYNC_CALL_GEN_FRAME", - [_ASYNC_CALL_GEN_FRAME_r13] = "_ASYNC_CALL_GEN_FRAME_r13", + [_ASYNC_CALL_LAZY_FRAME] = "_ASYNC_CALL_LAZY_FRAME", + [_ASYNC_CALL_LAZY_FRAME_r13] = "_ASYNC_CALL_LAZY_FRAME_r13", [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_r23] = "_BINARY_OP_r23", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", @@ -6872,7 +6872,7 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_ASYNC_CALL_TAIL: return 0; - case _ASYNC_CALL_GEN_FRAME: + case _ASYNC_CALL_LAZY_FRAME: return 1; case _MAYBE_EXPAND_METHOD_KW: return 0; diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index e77d64bde71739a..df2ff1defdc46b2 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -116,7 +116,21 @@ the call with the first execution of the `GET_AWAITABLE`, `PUSH_NULL`, `LOAD_COMMON_CONSTANT` and `SEND` instructions that follow it. `ASYNC_CALL_*` create the frame with the micro-ops of the matching `CALL_*` -or `CALL_KW_*` specialization. They then turn it into the frame of a new -coroutine and push it at once, as `RETURN_GENERATOR` and `SEND_GEN` would do. -The coroutine is not returned to the caller first and the frame of the caller -is left as if it were executing the `SEND`. +or `CALL_KW_*` specialization and push it at once, as a plain call would, but +leave the frame of the caller as if `SEND_GEN` had pushed the frame of a +coroutine: `None` and `NULL` on its stack and the instruction pointer at the +`SEND`. The frame is marked `lazy_coro` and gets its own `_PyErr_StackItem`, +pushed on the data stack right after it. In it, `RETURN_GENERATOR` only pushes +the `None` that the following `POP_TOP` discards. + +No coroutine object exists yet. A body that returns or raises before it +suspends never creates one: `RETURN_VALUE` lands in the caller at the +`END_SEND`, and `clear_thread_frame()` pops the exception state. When the body +reaches `YIELD_VALUE` for the first time, `_PyCoro_MaterializeLazy()` creates +the coroutine, `_PyEval_MoveLazyCoroFrame()` moves the frame into it, and the +`None` in the caller's stack is replaced by the coroutine, which is then +suspended and resumed like any other. + +While coroutine origin tracking is on, the origin has to be recorded when the +coroutine is created, so `ASYNC_CALL` is not specialized and the coroutine is +created at once by `RETURN_GENERATOR`. diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index d76d2f6383343d5..3412f46ae2fd84b 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -1,6 +1,7 @@ """Introspection utils for tasks call graphs.""" import dataclasses +import inspect import io import operator import sys @@ -38,6 +39,26 @@ class FutureCallGraph: awaited_by: tuple["FutureCallGraph", ...] +# A coroutine that is awaited right away may run without a coroutine +# object until it first suspends, so ``f.f_generator`` may be None +# for a coroutine frame; classify frames by their code instead. +_CO_ASYNC_FLAGS = inspect.CO_COROUTINE | inspect.CO_ASYNC_GENERATOR +_CO_GEN_FLAGS = _CO_ASYNC_FLAGS | inspect.CO_GENERATOR +_FRAME_TAGS = { + inspect.CO_COROUTINE: 'async ', + inspect.CO_ASYNC_GENERATOR: 'async generator ', + inspect.CO_GENERATOR: 'generator ', +} + + +def _is_async_frame(f): + return bool(f.f_code.co_flags & _CO_ASYNC_FLAGS) + + +def _is_sync_frame(f): + return not (f.f_code.co_flags & _CO_GEN_FLAGS) + + def _build_graph_for_future( future: futures.Future, *, @@ -166,12 +187,10 @@ def capture_call_graph( try: while f is not None: # gh-156988: sync gen should not clear the call chain - is_async = isinstance( - f.f_generator, (types.CoroutineType, types.AsyncGeneratorType)) call_stack.append(FrameCallGraphEntry(f)) - if is_async: - if f.f_back is not None and f.f_back.f_generator is None: + if _is_async_frame(f): + if f.f_back is not None and _is_sync_frame(f.f_back): # We've reached the bottom of the coroutine stack, which # must be the Task that runs it. break @@ -228,36 +247,12 @@ def add_line(line: str) -> None: ) for ste in st.call_stack: f = ste.frame - - if f.f_generator is None: - f = ste.frame - add_line( - f' | File {f.f_code.co_filename!r},' - f' line {f.f_lineno}, in' - f' {f.f_code.co_qualname}()' - ) - else: - c = f.f_generator - - try: - f = c.cr_frame - code = c.cr_code - tag = 'async' - except AttributeError: - try: - f = c.ag_frame - code = c.ag_code - tag = 'async generator' - except AttributeError: - f = c.gi_frame - code = c.gi_code - tag = 'generator' - - add_line( - f' | File {f.f_code.co_filename!r},' - f' line {f.f_lineno}, in' - f' {tag} {code.co_qualname}()' - ) + tag = _FRAME_TAGS.get(f.f_code.co_flags & _CO_GEN_FLAGS, '') + add_line( + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag}{f.f_code.co_qualname}()' + ) if st.awaited_by: add_line( diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 8af42a851707a56..0f099d5b9b83fd0 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -1,4 +1,5 @@ import asyncio +import inspect import io import sys import types @@ -20,19 +21,23 @@ def walk(s): if isinstance(s.future, asyncio.Task) else 'F' ] + def tag(frame): + # A coroutine awaited right after it is called may run + # without a coroutine object until it first suspends, so + # classify frames by their code rather than f_generator. + flags = frame.f_code.co_flags + if flags & inspect.CO_COROUTINE: + return "a" + if flags & inspect.CO_ASYNC_GENERATOR: + return "ag" + if flags & inspect.CO_GENERATOR: + return "g" + return "s" + ret.append( [ - ( - f"s {entry.frame.f_code.co_name}" - if entry.frame.f_generator is None else - ( - f"a {entry.frame.f_generator.cr_code.co_name}" - if hasattr(entry.frame.f_generator, 'cr_code') else - f"ag {entry.frame.f_generator.ag_code.co_name}" - if hasattr(entry.frame.f_generator, 'ag_code') else - f"g {entry.frame.f_generator.gi_code.co_name}" - ) - ) for entry in s.call_stack + f"{tag(entry.frame)} {entry.frame.f_code.co_name}" + for entry in s.call_stack # Skip the pure Python Future.__await__ generator frame if entry.frame.f_code is not future_await_code ] @@ -969,6 +974,26 @@ async def main(): await main() self.assertEqual(stack[:3], ['gen', 'middle', 'main']) + async def test_capture_call_graph_through_generator_based_coroutine(self): + # A native coroutine driven by a generator-based coroutine + # (yield from) keeps the frames of its callers. + stack = None + + async def native(): + nonlocal stack + graph = asyncio.capture_call_graph() + stack = [entry.frame.f_code.co_name for entry in graph.call_stack] + + @types.coroutine + def legacy(): + return (yield from native()) + + async def main(): + await legacy() + + await main() + self.assertEqual(stack[:3], ['native', 'legacy', 'main']) + @unittest.skipIf( not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 320af175affe0f2..77e18249e12bbad 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -5935,30 +5935,53 @@ async def outer(n): return total """), ns) return ns["outer"] + class Suspend: + def __await__(self): + yield + # Not `await Suspend()`: a call to a class is an ASYNC_CALL that + # is never specialized, which would end the trace. + suspend = Suspend() + async def suspending(i): + await suspend + return i + 1 def run(coro): - try: - coro.send(None) - except StopIteration as ex: - return ex.value + yields = 0 + while True: + try: + coro.send(None) + except StopIteration as ex: + return ex.value, yields + yields += 1 + def drive(coro): + # A loop in C: a trace of a Python driver loop would block + # tracing outer() while it is being recorded. + return len(list(iter(lambda: coro.send(None), object()))) n = TIER2_THRESHOLD * 2 - for callee in inner, make_closure(1): + for callee in inner, make_closure(1), suspending: with self.subTest(callee=callee): outer = make_outer(callee) - self.assertEqual(run(outer(n)), sum(range(1, n + 1))) + yields = n if callee is suspending else 0 + self.assertEqual(run(outer(n)), (sum(range(1, n + 1)), yields)) + if callee is suspending: + outer = make_outer(callee) + self.assertEqual(drive(outer(n)), n) ex = get_first_executor(outer) self.assertIsNotNone(ex) uops = get_opnames(ex) - # The coroutine is started without being returned to outer() - # first, and runs to its return within the trace. - self.assertIn("_ASYNC_CALL_GEN_FRAME", uops) + # The body of the coroutine is started without a coroutine + # object, in the trace. + self.assertIn("_ASYNC_CALL_LAZY_FRAME", uops) self.assertNotIn("_RETURN_GENERATOR", uops) - self.assertNotIn("_SEND_GEN_FRAME", uops) - self.assertIn("_RETURN_VALUE", uops) - self.assertIn("_END_SEND", uops) - # The guard is only needed in tier 1. - self.assertNotIn("_GUARD_ASYNC_CALL_TAIL", uops) + if callee is suspending: + # The coroutine is created at the suspension, which + # the trace reaches through the awaited generator. + self.assertIn("_YIELD_VALUE", uops) + else: + self.assertNotIn("_SEND_GEN_FRAME", uops) + self.assertIn("_RETURN_VALUE", uops) + self.assertIn("_END_SEND", uops) def test_send_virtual(self): diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 2e740066060bf93..0e5bae448064628 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1,6 +1,8 @@ import ast +import builtins import contextlib import copy +import gc import inspect import pickle import sys @@ -16,6 +18,10 @@ import _testcapi except ImportError: _testcapi = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None class AsyncYieldFrom: @@ -2707,7 +2713,9 @@ async def infinite(): def test_origin_tracking(self): async def get_origin(): coro = sys._getframe().f_generator - return coro.cr_origin + # None once the call is specialized: the coroutine object is + # created only if the body suspends. + return None if coro is None else coro.cr_origin async def caller(): origins = set() @@ -2755,6 +2763,555 @@ def trace(frame, event, arg): support.is_emscripten or support.is_wasi, "asyncio does not work under Emscripten/WASI yet." ) + + +@support.cpython_only +@support.requires_specialization +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") +class LazyCoroutineTest(unittest.TestCase): + """Once an awaited call is specialized to ASYNC_CALL_*, the body of + the coroutine runs in the frame of the call and the coroutine object + is only created when it first suspends. This must not be observable, + except that no coroutine object exists before the first suspension.""" + + @staticmethod + def coroutines_for(code): + return [o for o in gc.get_objects() + if type(o) is types.CoroutineType and o.cr_code is code] + + @staticmethod + def warm_up(coro_fn, *args): + """Run until the awaited calls in coro_fn are specialized.""" + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + run_async(coro_fn(*args)) + + def test_no_object_before_first_suspension(self): + seen = [] + + async def inner(): + seen.append(len(self.coroutines_for(inner.__code__))) + await AsyncYield(1) + seen.append(len(self.coroutines_for(inner.__code__))) + return 42 + + async def outer(): + return await inner() + + self.warm_up(outer) + seen.clear() + self.assertEqual(run_async(outer()), ([1], 42)) + self.assertEqual(seen, [0, 1]) + + def test_return_without_suspending(self): + async def inner(x): + return x + 1 + + async def outer(): + total = 0 + for i in range(100): + total += await inner(i) + return total + + self.assertEqual(run_async(outer()), ([], sum(range(1, 101)))) + self.assertEqual(self.coroutines_for(inner.__code__), []) + + def test_nested_chain(self): + async def chain(n): + if n == 0: + await AsyncYield("bottom") + return 0 + return 1 + await chain(n - 1) + + self.assertEqual(run_async(chain(20)), (["bottom"], 20)) + + def test_exception_before_first_suspension(self): + async def inner(): + raise ValueError("boom") + + async def outer(): + try: + await inner() + except ValueError as exc: + tb = exc.__traceback__ + names = [] + while tb is not None: + names.append(tb.tb_frame.f_code.co_name) + tb = tb.tb_next + return names + + self.warm_up(outer) + self.assertEqual(run_async(outer()), ([], ["outer", "inner"])) + + def test_stopiteration_before_first_suspension(self): + async def inner(): + raise StopIteration + + async def outer(): + await inner() + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD + 1): + with self.assertRaisesRegex(RuntimeError, + "coroutine raised StopIteration"): + run_async(outer()) + + def test_frame_object(self): + async def inner(lazy): + x = 1 + frame = sys._getframe() + if lazy: + self.assertIsNone(frame.f_generator) + self.assertEqual(frame.f_locals["x"], 1) + self.assertEqual(frame.f_back.f_code.co_name, "outer") + await AsyncYield(1) + coro = frame.f_generator + self.assertIsInstance(coro, types.CoroutineType) + self.assertIs(coro.cr_frame, frame) + self.assertIs(coro.cr_code, inner.__code__) + self.assertEqual(frame.f_locals["x"], 1) + self.assertEqual(frame.f_back.f_code.co_name, "outer") + x = 2 + return frame + + async def outer(lazy): + return await inner(lazy) + + self.warm_up(outer, False) + _, frame = run_async(outer(True)) + self.assertIsNone(frame.f_generator) # finished + self.assertEqual(frame.f_locals["x"], 2) + + def test_exc_info_suspend_inside_handler(self): + async def inner(): + try: + raise KeyError + except KeyError: + await AsyncYield(1) + self.assertIsInstance(sys.exc_info()[1], KeyError) + await AsyncYield(2) + self.assertIsInstance(sys.exc_info()[1], KeyError) + # The caller's context is visible after the handler. + self.assertIsInstance(sys.exc_info()[1], TypeError) + await AsyncYield(3) + self.assertIsInstance(sys.exc_info()[1], TypeError) + return "ok" + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exc_info()[1], TypeError) + self.assertIsNone(sys.exc_info()[1]) + return result + + self.warm_up(outer) + self.assertEqual(run_async(outer()), ([1, 2, 3], "ok")) + + def test_exc_info_handler_without_suspending(self): + async def inner(): + try: + raise KeyError + except KeyError: + self.assertIsInstance(sys.exc_info()[1], KeyError) + self.assertIsInstance(sys.exc_info()[1], TypeError) + return "ok" + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exc_info()[1], TypeError) + self.assertIsNone(sys.exc_info()[1]) + return result + + self.warm_up(outer) + self.assertEqual(run_async(outer()), ([], "ok")) + + def test_throw_into_materialized_coroutine(self): + async def inner(): + try: + await AsyncYield(1) + except ZeroDivisionError: + return "caught" + return "not caught" + + async def outer(): + return await inner() + + self.warm_up(outer) + coro = outer() + self.assertEqual(coro.send(None), 1) + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertEqual(cm.exception.value, "caught") + + def test_close_materialized_coroutine(self): + closed = [] + + async def inner(): + try: + await AsyncYield(1) + finally: + closed.append(True) + + async def outer(): + await inner() + + self.warm_up(outer) + closed.clear() + coro = outer() + self.assertEqual(coro.send(None), 1) + coro.close() + self.assertEqual(closed, [True]) + + def test_async_with(self): + log = [] + + class CM: + async def __aenter__(self): + log.append("enter") + return self + + async def __aexit__(self, *args): + log.append("exit") + return False + + async def outer(): + async with CM() as cm: + log.append(type(cm).__name__) + await AsyncYield(1) + return "done" + + self.warm_up(outer) + log.clear() + self.assertEqual(run_async(outer()), ([1], "done")) + self.assertEqual(log, ["enter", "CM", "exit"]) + self.assertEqual(self.coroutines_for(CM.__aenter__.__code__), []) + + def test_async_comprehension(self): + async def inner(x): + return x * 2 + + async def outer(): + return [await inner(x) for x in range(5)] + + self.warm_up(outer) + self.assertEqual(run_async(outer()), ([], [0, 2, 4, 6, 8])) + + def test_origin_tracking(self): + orig_depth = sys.get_coroutine_origin_tracking_depth() + + async def inner(): + await AsyncYield(1) + return sys._getframe().f_generator.cr_origin + + async def outer(): + coro = inner() + eager = await coro + lazy = await inner() + if eager is None: # tracking off, warming up + return None + return eager[0], lazy[0] + + sys.set_coroutine_origin_tracking_depth(0) + try: + self.warm_up(outer) + sys.set_coroutine_origin_tracking_depth(2) + _, (eager, lazy) = run_async(outer()) + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + # Origin tracking disables the lazy path: the origin is the + # call, not the await, and is captured at creation. + self.assertEqual(eager[0], __file__) + self.assertEqual(eager[2], "outer") + self.assertEqual(lazy[0], __file__) + self.assertEqual(lazy[2], "outer") + self.assertEqual(lazy[1], eager[1] + 2) + + def test_origin_tracking_multiline_await(self): + orig_depth = sys.get_coroutine_origin_tracking_depth() + + async def inner(): + await AsyncYield(1) + return sys._getframe().f_generator.cr_origin + + async def outer(): + origin = await ( + inner() + ) + if origin is None: # tracking off, warming up + return None + return origin[0][1] - outer.__code__.co_firstlineno + + sys.set_coroutine_origin_tracking_depth(0) + try: + self.warm_up(outer) + sys.set_coroutine_origin_tracking_depth(1) + _, line = run_async(outer()) + finally: + sys.set_coroutine_origin_tracking_depth(orig_depth) + # The line of the call, not of the await. + self.assertEqual(line, 2) + + def test_settrace_events(self): + async def inner(): + await AsyncYield(1) + return 1 + + async def eager(): + coro = inner() + return await coro + + async def lazy(): + return await inner() + + def trace(coro_fn): + events = [] + + def tracer(frame, event, arg): + if frame.f_code is inner.__code__: + events.append((event, frame.f_lineno - frame.f_code.co_firstlineno)) + return tracer + + sys.settrace(tracer) + try: + result = run_async(coro_fn()) + finally: + sys.settrace(None) + return events, result + + self.warm_up(lazy) + self.assertEqual(trace(lazy), trace(eager)) + + def test_monitoring_events(self): + async def inner(): + await AsyncYield(1) + return 1 + + async def eager(): + coro = inner() + return await coro + + async def lazy(): + return await inner() + + E = sys.monitoring.events + # No LINE events: they instrument the caller's SEND, which + # makes the await take the regular path. + names = {E.PY_START: "start", E.PY_RESUME: "resume", + E.PY_RETURN: "return", E.PY_YIELD: "yield", + E.STOP_ITERATION: "stop_iteration"} + codes = {inner.__code__: "inner", eager.__code__: "outer", + lazy.__code__: "outer"} + + def monitor(coro_fn): + events = [] + tool = sys.monitoring.DEBUGGER_ID + sys.monitoring.use_tool_id(tool, "test") + try: + for event, name in names.items(): + def callback(code, offset, *args, name=name): + if code in codes: + events.append( + (codes[code], name, type(args[-1]) if args else None)) + sys.monitoring.register_callback(tool, event, callback) + sys.monitoring.set_events(tool, sum(names)) + result = run_async(coro_fn()) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + return events, result + + self.warm_up(lazy) + self.assertEqual(monitor(lazy), monitor(eager)) + self.assertIn(("outer", "stop_iteration", StopIteration), + monitor(lazy)[0]) + + def test_stop_iteration_event_without_suspending(self): + async def inner(): + return 42 + + async def outer(): + return await inner() + + self.warm_up(outer) + E = sys.monitoring.events + events = [] + tool = sys.monitoring.DEBUGGER_ID + sys.monitoring.use_tool_id(tool, "test") + try: + def callback(code, offset, exc): + if code is outer.__code__: + events.append(exc.value) + sys.monitoring.register_callback(tool, E.STOP_ITERATION, callback) + sys.monitoring.set_events(tool, E.STOP_ITERATION) + self.assertEqual(run_async(outer()), ([], 42)) + finally: + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + self.assertEqual(events, [42]) + + def test_exc_state_after_suspending_inside_handler(self): + # The coroutine's own exception state must not keep the + # caller's exception once its handler is done. + async def inner(): + try: + raise KeyError + except KeyError: + await AsyncYield(1) + await AsyncYield(2) + return sys.exc_info()[1] + + async def outer(): + try: + raise TypeError + except TypeError: + return await inner() + + self.warm_up(outer) + coro = outer() + coro.send(None) + inner_coro = coro.cr_await + inner_coro.send(None) + self.assertNotIn(TypeError, + map(type, gc.get_referents(inner_coro))) + # Resumed directly, not through outer: no handled exception. + with self.assertRaises(StopIteration) as cm: + inner_coro.send(None) + self.assertIsNone(cm.exception.value) + coro.close() + + def test_throw_into_coroutine_suspended_inside_handler(self): + async def inner(): + try: + raise KeyError + except KeyError: + try: + await AsyncYield(1) + except ZeroDivisionError: + pass + return sys.exception() + + async def outer(): + try: + raise TypeError + except TypeError: + result = await inner() + self.assertIsInstance(sys.exception(), TypeError) + return result + + self.warm_up(outer) + coro = outer() + self.assertEqual(coro.send(None), 1) + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertIsNone(cm.exception.value) + + def test_import_hook_locals(self): + seen = [] + orig_import = builtins.__import__ + + def hook(name, globals=None, locals=None, fromlist=(), level=0): + seen.append(locals) + return orig_import(name, globals, locals, fromlist, level) + + async def inner(): + import sys as _sys + return _sys + + async def outer(): + try: + raise TypeError + except TypeError: + return await inner() + + self.warm_up(outer) + with support.swap_attr(builtins, "__import__", hook): + self.assertEqual(run_async(outer()), ([], sys)) + self.assertEqual(seen, [None]) + + def test_traceback_frame_keeps_no_caller_exception(self): + async def inner(): + raise ValueError + + async def outer(): + try: + raise TypeError + except TypeError: + try: + await inner() + except ValueError as exc: + return exc.__traceback__.tb_next.tb_frame + + self.warm_up(outer) + _, frame = run_async(outer()) + self.assertIs(frame.f_code, inner.__code__) + self.assertNotIn(TypeError, map(type, gc.get_referents(frame))) + + def test_code_replaced_before_first_suspension(self): + # The coroutine object is created at the first suspension, so it + # must be sized for the code the frame runs, not func.__code__. + async def small(): + big, small # same free variables as big() + await AsyncYield(1) + + async def big(replace): + a = b = c = d = e = f = g = h = i = j = k = m = n = o = p = 1 + if replace: + big.__code__ = small.__code__ + await AsyncYield(1) + return (a, b, c, d, e, f, g, h, i, j, k, m, n, o, p) + + async def outer(replace): + return await big(replace) + + self.warm_up(outer, False) + self.assertEqual(run_async(outer(True)), ([1], (1,) * 15)) + + def test_finished_frame_has_no_f_back(self): + async def inner(): + return sys._getframe() + + async def outer(): + return await inner() + + self.warm_up(outer) + _, frame = run_async(outer()) + self.assertIs(frame.f_code, inner.__code__) + self.assertIsNone(frame.f_back) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_memory_error_at_first_suspension(self): + @types.coroutine + def suspend(): + yield 1 + + async def inner(awaitable, fail): + if fail: + _testcapi.set_nomemory(0, 1) + try: + await awaitable + finally: + _testcapi.remove_mem_hooks() + return "resumed" + + async def outer(awaitable, fail): + return await inner(awaitable, fail) + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + run_async(outer(suspend(), False)) + awaitable = suspend() + coro = outer(awaitable, True) + try: + with self.assertRaises(MemoryError): + coro.send(None) + finally: + _testcapi.remove_mem_hooks() + awaitable.close() + + + class CoroAsyncIOCompatTest(unittest.TestCase): def test_asyncio_1(self): diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 685f5e00eaa0667..0a2ee91f1810eea 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1814,6 +1814,41 @@ async def not_awaited(): self.assert_no_opcode(not_awaited, "ASYNC_CALL_PY_GENERAL") self.assert_specialized(not_awaited, "SEND_GEN") + @cpython_only + @requires_specialization + def test_async_call_origin_tracking(self): + # The origin of a coroutine is recorded when its object is + # created, so a call that is awaited at once keeps creating it + # while origin tracking is on. + def run_async(coro): + try: + coro.send(None) + except StopIteration as ex: + return ex.value + self.fail("coroutine did not finish") + + async def coro(a): + return sys._getframe().f_generator + + async def caller(n): + return [await coro(i) for i in range(n)] + + orig_depth = sys.get_coroutine_origin_tracking_depth() + self.addCleanup(sys.set_coroutine_origin_tracking_depth, orig_depth) + n = _testinternalcapi.SPECIALIZATION_THRESHOLD + sys.set_coroutine_origin_tracking_depth(1) + self.assertNotIn(None, run_async(caller(n))) + self.assert_no_opcode(caller, "ASYNC_CALL_PY_EXACT_ARGS") + + sys.set_coroutine_origin_tracking_depth(0) + n = _testinternalcapi.SPECIALIZATION_COOLDOWN + self.assertIn(None, run_async(caller(n))) + self.assert_specialized(caller, "ASYNC_CALL_PY_EXACT_ARGS") + + # Turned on again, the specialized instruction deoptimizes. + sys.set_coroutine_origin_tracking_depth(1) + self.assertNotIn(None, run_async(caller(n))) + @cpython_only @requires_specialization def test_async_call_deopt(self): diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 219cc271738a43f..5ed02676039a551 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -157,8 +157,8 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; - _PyStackRef coro; - _PyStackRef gen_frame; + _PyStackRef none; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -183,6 +183,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { @@ -276,46 +281,43 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -353,8 +355,8 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; - _PyStackRef coro; - _PyStackRef gen_frame; + _PyStackRef none; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -379,6 +381,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } } // _CHECK_METHOD_VERSION { @@ -460,44 +467,41 @@ } new_frame = PyStackRef_Wrap(temp); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -658,8 +662,8 @@ _PyStackRef *args; _PyStackRef kwnames; _PyStackRef new_frame; - _PyStackRef coro; - _PyStackRef gen_frame; + _PyStackRef none; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -684,6 +688,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } } // _CHECK_METHOD_VERSION_KW { @@ -767,44 +776,41 @@ } new_frame = PyStackRef_Wrap(temp); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -842,9 +848,9 @@ _PyStackRef *args; _PyStackRef kwnames; _PyStackRef new_frame; - _PyStackRef coro; + _PyStackRef none; _PyStackRef null; - _PyStackRef gen_frame; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -869,6 +875,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } } // _CHECK_FUNCTION_VERSION_KW { @@ -932,44 +943,41 @@ } new_frame = PyStackRef_Wrap(temp); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1006,9 +1014,9 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; - _PyStackRef coro; + _PyStackRef none; _PyStackRef null; - _PyStackRef gen_frame; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -1033,6 +1041,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } } // _CHECK_FUNCTION_VERSION { @@ -1096,46 +1109,43 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1172,9 +1182,9 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; - _PyStackRef coro; + _PyStackRef none; _PyStackRef null; - _PyStackRef gen_frame; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -1199,6 +1209,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } } // _CHECK_FUNCTION_VERSION { @@ -1252,44 +1267,41 @@ } new_frame = PyStackRef_Wrap(temp); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -6490,9 +6502,6 @@ null_in = stack_pointer[-3]; sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -9203,7 +9212,9 @@ index_or_null = stack_pointer[-2]; receiver = stack_pointer[-3]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o) || + Py_IsNone(receiver_o)) + { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -9868,8 +9879,30 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + stack_pointer[-1] = retval; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -9884,6 +9917,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -9899,6 +9935,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -12993,7 +13034,7 @@ next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; - if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + if (frame->lazy_coro) { res = PyStackRef_None; } else { @@ -15133,8 +15174,30 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + stack_pointer[-1] = retval; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -15149,6 +15212,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -15164,6 +15230,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); } diff --git a/Objects/genobject.c b/Objects/genobject.c index f5dd4501b701804..d9da788cffab0b3 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1104,9 +1104,8 @@ PyTypeObject PyGen_Type = { }; static PyObject * -make_gen(PyTypeObject *type, PyFunctionObject *func) +make_gen(PyTypeObject *type, PyFunctionObject *func, PyCodeObject *code) { - PyCodeObject *code = (PyCodeObject *)func->func_code; int slots = _PyFrame_NumSlotsForCodeObject(code); PyGenObject *gen = PyObject_GC_NewVar(PyGenObject, type, slots); if (gen == NULL) { @@ -1127,30 +1126,11 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) return (PyObject *)gen; } -PyObject * -_Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame) +static PyObject * +make_coro(PyFunctionObject *func, PyCodeObject *code, + _PyInterpreterFrame *caller) { - int coro_flags = ((PyCodeObject *)func->func_code)->co_flags & - (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); - assert(coro_flags); - if (coro_flags == CO_GENERATOR) { - return make_gen(&PyGen_Type, func); - } - if (coro_flags == CO_ASYNC_GENERATOR) { - PyAsyncGenObject *ag; - ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func); - if (ag == NULL) { - return NULL; - } - ag->ag_origin_or_finalizer = NULL; - ag->ag_closed = 0; - ag->ag_hooks_inited = 0; - ag->ag_running_async = 0; - return (PyObject*)ag; - } - - assert (coro_flags == CO_COROUTINE); - PyObject *coro = make_gen(&PyCoro_Type, func); + PyObject *coro = make_gen(&PyCoro_Type, func, code); if (!coro) { return NULL; } @@ -1160,9 +1140,7 @@ _Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame) if (origin_depth == 0) { ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; } else { - assert(frame); - assert(_PyFrame_IsIncomplete(frame)); - frame = _PyFrame_GetFirstComplete(frame->previous); + _PyInterpreterFrame *frame = _PyFrame_GetFirstComplete(caller); PyObject *cr_origin = _PyCoro_ComputeOrigin(origin_depth, frame); ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin; if (!cr_origin) { @@ -1176,7 +1154,53 @@ _Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame) PyObject * _Py_MakeCoro(PyFunctionObject *func) { - return _Py_MakeCoroWithFrame(func, _PyThreadState_GET()->current_frame); + PyCodeObject *code = (PyCodeObject *)func->func_code; + int coro_flags = code->co_flags & + (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); + assert(coro_flags); + if (coro_flags == CO_GENERATOR) { + return make_gen(&PyGen_Type, func, code); + } + if (coro_flags == CO_ASYNC_GENERATOR) { + PyAsyncGenObject *ag; + ag = (PyAsyncGenObject *)make_gen(&PyAsyncGen_Type, func, code); + if (ag == NULL) { + return NULL; + } + ag->ag_origin_or_finalizer = NULL; + ag->ag_closed = 0; + ag->ag_hooks_inited = 0; + ag->ag_running_async = 0; + return (PyObject*)ag; + } + + assert (coro_flags == CO_COROUTINE); + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *frame = tstate->current_frame; + assert(frame); + assert(_PyFrame_IsIncomplete(frame)); + return make_coro(func, code, frame->previous); +} + +/* Create the coroutine object for a coroutine whose body has been + * executing directly in `frame`, a frame owned by the thread, because + * its caller awaited the result immediately (see _ASYNC_CALL_LAZY_FRAME + * in bytecodes.c). Called when the body suspends for the first time. + * The frame itself is moved into the coroutine by the caller. + * The body may have replaced func.__code__ in the meantime, so the + * coroutine is sized for the code the frame is actually running. */ +PyObject * +_PyCoro_MaterializeLazy(_PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(frame->lazy_coro); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = + (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyCodeObject *code = _PyFrame_GetCode(frame); + assert((code->co_flags & + (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) == CO_COROUTINE); + return make_coro(func, code, frame->previous); } static PyObject * diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5e89a48af7b8977..20d571c05714daf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -437,7 +437,11 @@ dummy_func( tier1 inst(INSTRUMENTED_END_SEND, (receiver, index_or_null, value -- val)) { PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + /* None is the placeholder of a lazily started coroutine that + * returned before it was created (see _ASYNC_CALL_LAZY_FRAME). */ + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o) || + Py_IsNone(receiver_o)) + { int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); if (err) { ERROR_NO_POP(); @@ -1850,9 +1854,29 @@ dummy_func( // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. + // The one exception is a lazily created coroutine (see + // _ASYNC_CALL_LAZY_FRAME) that cannot be allocated: the await + // is then treated as having raised MemoryError. assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + if (gen == NULL) { + ERROR_NO_POP(); + } + /* Move the body's exception state (see _ASYNC_CALL_LAZY_FRAME) + * into the coroutine, where SEND_GEN would have put it. */ + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -1865,6 +1889,9 @@ dummy_func( _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -1880,6 +1907,12 @@ dummy_func( #endif RELOAD_STACK(); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + /* Fill in the coroutine slot of the caller's SEND. */ + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -1959,9 +1992,9 @@ dummy_func( tier1 inst(CLEANUP_THROW, (sub_iter, null_in, last_sent_val, exc_value_st -- none, null_out, value)) { PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif + /* Reached on throw()/close(), or when a lazily started + * coroutine could not be created at its first suspension + * (see _YIELD_VALUE). */ assert(exc_value && PyExceptionInstance_Check(exc_value)); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); if (matches) { @@ -5346,10 +5379,11 @@ dummy_func( // GET_AWAITABLE; PUSH_NULL; LOAD_COMMON_CONSTANT None; SEND // Unspecialized, it does the same as CALL. Its specializations, for // calls to coroutine functions, merge the call with the first - // execution of the instructions that follow it: the new coroutine is - // started without being returned to the caller first. They are built - // from the uops of the matching CALL_* specialization, with - // _ASYNC_CALL_GEN_FRAME in place of _SAVE_RETURN_OFFSET. + // execution of the instructions that follow it: the body of the + // coroutine is started in the frame of the call, and the coroutine + // object is only created if the body suspends. They are built from + // the uops of the matching CALL_* specialization, with + // _ASYNC_CALL_LAZY_FRAME in place of _SAVE_RETURN_OFFSET. family(ASYNC_CALL, INLINE_CACHE_ENTRIES_ASYNC_CALL) = { ASYNC_CALL_PY_EXACT_ARGS, ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, @@ -5377,48 +5411,53 @@ dummy_func( // none of them may be instrumented. Only the first and the last of // them can start a line. // Executors are invalidated when the code is instrumented. + // Coroutine origin tracking records the origin when the coroutine + // object is created, which the generic path does at once. op(_GUARD_ASYNC_CALL_TAIL, (--)) { #if TIER_ONE DEOPT_IF(next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE); DEOPT_IF(next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE); assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + DEOPT_IF(tstate->coroutine_origin_tracking_depth != 0); } - // Turns the frame of a call to a coroutine function into a running - // coroutine: the equivalent of executing RETURN_GENERATOR in that + // Starts the body of a call to a coroutine function in the frame + // that was just created for it, without creating the coroutine + // object: the equivalent of executing RETURN_GENERATOR in that // frame, followed by GET_AWAITABLE, PUSH_NULL, LOAD_COMMON_CONSTANT - // None and SEND_GEN in this one. - op(_ASYNC_CALL_GEN_FRAME, (new_frame -- coro, null, gen_frame)) { + // None and SEND_GEN in this one. The coroutine object is created + // when the body first suspends (see _YIELD_VALUE), so a body that + // returns or raises before that never needs one. + op(_ASYNC_CALL_LAZY_FRAME, (new_frame -- none, null, lazy_frame)) { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); DEAD(new_frame); SYNC_SP(); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); - if (gen == NULL) { + assert(callee->previous == frame); + // The body gets its own exception state, as SEND_GEN would + // give it. It lives on the data stack right after the frame; + // clear_thread_frame() and _YIELD_VALUE pop it. + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); + if (exc_info == NULL) { _PyEval_FrameClearAndPop(tstate, callee); ERROR_NO_POP(); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - // Otherwise RETURN_GENERATOR is preceded by COPY_FREE_VARS or - // MAKE_CELL. They run in the frame of the coroutine, where - // RETURN_GENERATOR only pushes the None that POP_TOP discards. - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; - // The coroutine yields and returns to the SEND of the await. + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; + // The body yields and returns to the SEND of the await, as if + // SEND_GEN had pushed its frame. None stands in for the + // coroutine on the stack until it is created. _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + INSTRUCTION_SIZE); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } macro(ASYNC_CALL_PY_EXACT_ARGS) = @@ -5431,7 +5470,7 @@ dummy_func( _CHECK_STACK_SPACE + _CHECK_RECURSION_REMAINING + _INIT_CALL_PY_EXACT_ARGS + - _ASYNC_CALL_GEN_FRAME + + _ASYNC_CALL_LAZY_FRAME + _PUSH_FRAME; macro(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS) = @@ -5447,7 +5486,7 @@ dummy_func( _CHECK_STACK_SPACE + _CHECK_RECURSION_REMAINING + _INIT_CALL_PY_EXACT_ARGS + - _ASYNC_CALL_GEN_FRAME + + _ASYNC_CALL_LAZY_FRAME + _PUSH_FRAME; macro(ASYNC_CALL_PY_GENERAL) = @@ -5458,7 +5497,7 @@ dummy_func( _CHECK_FUNCTION_VERSION + _CHECK_RECURSION_REMAINING + _PY_FRAME_GENERAL + - _ASYNC_CALL_GEN_FRAME + + _ASYNC_CALL_LAZY_FRAME + _PUSH_FRAME; macro(ASYNC_CALL_BOUND_METHOD_GENERAL) = @@ -5471,7 +5510,7 @@ dummy_func( flush + // so that self is in the argument array _CHECK_RECURSION_REMAINING + _PY_FRAME_GENERAL + - _ASYNC_CALL_GEN_FRAME + + _ASYNC_CALL_LAZY_FRAME + _PUSH_FRAME; // Cache layout: counter/1, func_version/2 @@ -5711,7 +5750,7 @@ dummy_func( _CHECK_FUNCTION_VERSION_KW + _CHECK_RECURSION_REMAINING + _PY_FRAME_KW + - _ASYNC_CALL_GEN_FRAME + + _ASYNC_CALL_LAZY_FRAME + _PUSH_FRAME; macro(ASYNC_CALL_KW_BOUND_METHOD) = @@ -5723,7 +5762,7 @@ dummy_func( _EXPAND_METHOD_KW + flush + // so that self is in the argument array _PY_FRAME_KW + - _ASYNC_CALL_GEN_FRAME + + _ASYNC_CALL_LAZY_FRAME + _PUSH_FRAME; op(_CHECK_IS_NOT_PY_CALLABLE_KW, (callable, unused, unused[oparg], unused -- callable, unused, unused[oparg], unused)) { @@ -5988,8 +6027,10 @@ dummy_func( } inst(RETURN_GENERATOR, (-- res)) { - if (frame->owner == FRAME_OWNED_BY_GENERATOR) { - // Handled by ASYNC_CALL_* already. + if (frame->lazy_coro) { + // The body of a coroutine started by ASYNC_CALL_*: the + // coroutine is created when it first suspends, if ever + // (see _ASYNC_CALL_LAZY_FRAME). res = PyStackRef_None; } else { diff --git a/Python/ceval.c b/Python/ceval.c index 0d053cdc7a63e34..c8f409d9a100800 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1999,10 +1999,19 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, return -1; } +static PyObject *pop_lazy_coro_exc_state(PyThreadState *, _PyInterpreterFrame *); + static void clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { assert(frame->owner == FRAME_OWNED_BY_THREAD); + PyObject *lazy_exc = NULL; + if (frame->lazy_coro) { + lazy_exc = pop_lazy_coro_exc_state(tstate, frame); + // Like clear_gen_frame(): a frame object that outlives the + // coroutine must not keep the caller's frame alive via f_back. + frame->previous = NULL; + } // Make sure that this is, indeed, the top frame. We can't check this in // _PyThreadState_PopFrame, since f_code is already cleared at that point: assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == @@ -2011,6 +2020,7 @@ clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) _PyFrame_ClearExceptCode(frame); PyStackRef_CLEAR(frame->f_executable); _PyThreadState_PopFrame(tstate, frame); + Py_XDECREF(lazy_exc); } static void @@ -3680,6 +3690,44 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name } } +/* Pop the exception state that _ASYNC_CALL_LAZY_FRAME gave a lazily + * started coroutine body, restoring the caller's. Returns the body's exception, + * if any (a new reference). */ +static PyObject * +pop_lazy_coro_exc_state(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + assert(frame->lazy_coro); + _PyErr_StackItem *exc_info = tstate->exc_info; + tstate->exc_info = exc_info->previous_item; + PyObject *exc_value = exc_info->exc_value; + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + frame->lazy_coro = 0; + return exc_value; +} + +/* YIELD_VALUE support: move the frame of a lazily created coroutine + * from the thread's data stack into `gen`. Returns the new frame. */ +_PyInterpreterFrame * +_PyEval_MoveLazyCoroFrame(PyThreadState *tstate, PyGenObject *gen, + _PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_THREAD); + assert(frame->lazy_coro); + assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == + tstate->datastack_top); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_Copy(frame, gen_frame); + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + PyFrameObject *f = gen_frame->frame_obj; + if (f != NULL) { + Py_BEGIN_CRITICAL_SECTION(f); + f->f_frame = gen_frame; + Py_END_CRITICAL_SECTION(); + } + _PyThreadState_PopFrame(tstate, frame); + return gen_frame; +} + PyObject * _PyEval_GetAwaitable(PyObject *iterable, int oparg) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1ee676c07903c49..17ac14e95f37bb7 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -9612,11 +9612,39 @@ oparg = CURRENT_OPARG(); retval = _stack_item_0; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + stack_pointer[0] = retval; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + stack_pointer += 1; + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); @@ -9626,6 +9654,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -9641,6 +9672,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); _tos_cache0 = value; @@ -20039,6 +20075,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } SET_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -20063,6 +20104,12 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -20091,6 +20138,13 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); @@ -20123,6 +20177,14 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } _tos_cache2 = _stack_item_2; _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; @@ -20131,23 +20193,23 @@ break; } - case _ASYNC_CALL_GEN_FRAME_r13: { + case _ASYNC_CALL_LAZY_FRAME_r13: { CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef new_frame; - _PyStackRef coro; + _PyStackRef none; _PyStackRef null; - _PyStackRef gen_frame; + _PyStackRef lazy_frame; _PyStackRef _stack_item_0 = _tos_cache0; new_frame = _stack_item_0; _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); @@ -20155,24 +20217,21 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); - _tos_cache2 = gen_frame; + lazy_frame = PyStackRef_Wrap(callee); + _tos_cache2 = lazy_frame; _tos_cache1 = null; - _tos_cache0 = coro; + _tos_cache0 = none; SET_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; @@ -20978,7 +21037,7 @@ CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef res; - if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + if (frame->lazy_coro) { res = PyStackRef_None; } else { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index dd631d4c1921f6c..5ae87105546e784 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -157,8 +157,8 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; - _PyStackRef coro; - _PyStackRef gen_frame; + _PyStackRef none; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -183,6 +183,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { @@ -276,46 +281,43 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -353,8 +355,8 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; - _PyStackRef coro; - _PyStackRef gen_frame; + _PyStackRef none; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -379,6 +381,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } } // _CHECK_METHOD_VERSION { @@ -460,44 +467,41 @@ } new_frame = PyStackRef_Wrap(temp); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -658,8 +662,8 @@ _PyStackRef *args; _PyStackRef kwnames; _PyStackRef new_frame; - _PyStackRef coro; - _PyStackRef gen_frame; + _PyStackRef none; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -684,6 +688,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } } // _CHECK_METHOD_VERSION_KW { @@ -767,44 +776,41 @@ } new_frame = PyStackRef_Wrap(temp); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -842,9 +848,9 @@ _PyStackRef *args; _PyStackRef kwnames; _PyStackRef new_frame; - _PyStackRef coro; + _PyStackRef none; _PyStackRef null; - _PyStackRef gen_frame; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -869,6 +875,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL_KW)); + JUMP_TO_PREDICTED(ASYNC_CALL_KW); + } } // _CHECK_FUNCTION_VERSION_KW { @@ -932,44 +943,41 @@ } new_frame = PyStackRef_Wrap(temp); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1006,9 +1014,9 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; - _PyStackRef coro; + _PyStackRef none; _PyStackRef null; - _PyStackRef gen_frame; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -1033,6 +1041,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } } // _CHECK_FUNCTION_VERSION { @@ -1096,46 +1109,43 @@ } new_frame = PyStackRef_Wrap(pushed_frame); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1172,9 +1182,9 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; - _PyStackRef coro; + _PyStackRef none; _PyStackRef null; - _PyStackRef gen_frame; + _PyStackRef lazy_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { @@ -1199,6 +1209,11 @@ } assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif + if (tstate->coroutine_origin_tracking_depth != 0) { + UPDATE_MISS_STATS(ASYNC_CALL); + assert(_PyOpcode_Deopt[opcode] == (ASYNC_CALL)); + JUMP_TO_PREDICTED(ASYNC_CALL); + } } // _CHECK_FUNCTION_VERSION { @@ -1252,44 +1267,41 @@ } new_frame = PyStackRef_Wrap(temp); } - // _ASYNC_CALL_GEN_FRAME + // _ASYNC_CALL_LAZY_FRAME { _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); - assert(PyStackRef_FunctionCheck(callee->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + assert(callee->previous == frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyErr_StackItem *exc_info = (_PyErr_StackItem *)_PyThreadState_PushFrame( + tstate, sizeof(_PyErr_StackItem) / sizeof(PyObject *)); _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { + if (exc_info == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, callee); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(Py_TYPE(gen) == &PyCoro_Type); - if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - assert(callee->instr_ptr[1].op.code == POP_TOP); - callee->instr_ptr += 2; - } - gen_take_frame(tstate, gen, callee, frame); - gen->gi_frame_state = FRAME_EXECUTING; + exc_info->exc_value = NULL; + exc_info->previous_item = tstate->exc_info; + tstate->exc_info = exc_info; + callee->lazy_coro = 1; _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); assert(send != NULL); frame->instr_ptr = send; - gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); - coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + assert(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg <= UINT16_MAX); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg); + none = PyStackRef_None; null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + lazy_frame = PyStackRef_Wrap(callee); } // _PUSH_FRAME { - new_frame = gen_frame; + new_frame = lazy_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer[0] = coro; + stack_pointer[0] = none; stack_pointer[1] = null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -6490,9 +6502,6 @@ null_in = stack_pointer[-3]; sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -9203,7 +9212,9 @@ index_or_null = stack_pointer[-2]; receiver = stack_pointer[-3]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o) || + Py_IsNone(receiver_o)) + { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); @@ -9867,8 +9878,30 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + stack_pointer[-1] = retval; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -9883,6 +9916,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -9898,6 +9934,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); } @@ -12990,7 +13031,7 @@ next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; - if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + if (frame->lazy_coro) { res = PyStackRef_None; } else { @@ -15130,8 +15171,30 @@ { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + PyGenObject *gen; + int lazy = frame->owner == FRAME_OWNED_BY_THREAD; + if (lazy) { + assert(frame->lazy_coro); + stack_pointer[-1] = retval; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + gen = (PyGenObject *)_PyCoro_MaterializeLazy(frame); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + _PyErr_StackItem *exc_info = tstate->exc_info; + gen->gi_exc_state.exc_value = exc_info->exc_value; + gen->gi_exc_state.previous_item = exc_info->previous_item; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, (_PyInterpreterFrame *)exc_info); + _PyFrame_StackPointerInvalidate(frame); + } + else { + gen = _PyGen_GetGeneratorFromFrame(frame); + } frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; @@ -15146,6 +15209,9 @@ _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; + if (lazy) { + gen_frame = _PyEval_MoveLazyCoroFrame(tstate, gen, gen_frame); + } gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); @@ -15161,6 +15227,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + if (lazy) { + assert(PyStackRef_IsNone(stack_pointer[-2])); + assert(PyStackRef_IsNull(stack_pointer[-1])); + stack_pointer[-2] = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + } value = temp; LLTRACE_RESUME_FRAME(); } diff --git a/Python/intrinsics.c b/Python/intrinsics.c index dfa81baa654e123..e3b0fd406a612a4 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -144,7 +144,11 @@ static PyObject * stopiteration_error(PyThreadState* tstate, PyObject *exc) { _PyInterpreterFrame *frame = tstate->current_frame; - assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + /* A coroutine that is awaited immediately runs in a thread-owned + * frame until it first suspends (see RETURN_GENERATOR). */ + assert(frame->owner == FRAME_OWNED_BY_GENERATOR || + (frame->owner == FRAME_OWNED_BY_THREAD && + (_PyFrame_GetCode(frame)->co_flags & CO_COROUTINE))); assert(PyExceptionInstance_Check(exc)); const char *msg = NULL; if (PyErr_GivenExceptionMatches(exc, PyExc_StopIteration)) { diff --git a/Python/optimizer.c b/Python/optimizer.c index 55675b7ef4acabe..a382835cac614aa 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -745,9 +745,8 @@ _PyJit_translate_single_bytecode_to_trace( } // RETURN_GENERATOR in a coroutine that ASYNC_CALL_* already started - // stays in its frame: it only pushes None (see _ASYNC_CALL_GEN_FRAME). - bool coro_running = opcode == RETURN_GENERATOR && - frame == tracer->prev_state.instr_frame; + // stays in its frame: it only pushes None (see _ASYNC_CALL_LAZY_FRAME). + bool coro_running = opcode == RETURN_GENERATOR && frame->lazy_coro; bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode) && !coro_running; if (has_dynamic_jump_taken && !needs_guard_ip) { DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 9bc41b022189d71..2a0695f682aa37c 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1528,16 +1528,12 @@ dummy_func(void) { gen_frame = PyJitRef_WrapInvalid(new_frame); } - op(_GUARD_ASYNC_CALL_TAIL, (--)) { - // Only needed in tier 1: executors are invalidated - // when the code is instrumented. - ADD_OP(_NOP, 0, 0); - } - - op(_ASYNC_CALL_GEN_FRAME, (new_frame -- coro, null, gen_frame)) { - coro = sym_new_type(ctx, &PyCoro_Type); + op(_ASYNC_CALL_LAZY_FRAME, (new_frame -- none, null, lazy_frame)) { + // The None is replaced by the coroutine when the body suspends, + // so it must not be treated as a constant. + none = sym_new_not_null(ctx); null = sym_new_null(ctx); - gen_frame = new_frame; + lazy_frame = new_frame; } op(_SEND_GEN_FRAME, (receiver, null, v -- receiver, null, gen_frame)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 0d36e5ac09cc8f1..1118653a8d78423 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -5063,23 +5063,22 @@ } case _GUARD_ASYNC_CALL_TAIL: { - ADD_OP(_NOP, 0, 0); break; } - case _ASYNC_CALL_GEN_FRAME: { + case _ASYNC_CALL_LAZY_FRAME: { JitOptRef new_frame; - JitOptRef coro; + JitOptRef none; JitOptRef null; - JitOptRef gen_frame; + JitOptRef lazy_frame; new_frame = stack_pointer[-1]; - coro = sym_new_type(ctx, &PyCoro_Type); + none = sym_new_not_null(ctx); null = sym_new_null(ctx); - gen_frame = new_frame; + lazy_frame = new_frame; CHECK_STACK_BOUNDS(2); - stack_pointer[-1] = coro; + stack_pointer[-1] = none; stack_pointer[0] = null; - stack_pointer[1] = gen_frame; + stack_pointer[1] = lazy_frame; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; diff --git a/Python/specialize.c b/Python/specialize.c index ffb1fb15780924f..29eaa9b9a4f1130 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1754,8 +1754,10 @@ specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, /* Can the ASYNC_CALL or ASYNC_CALL_KW at "instr", a call to a function * with the code "code", be specialized to one of the ASYNC_CALL_* - * instructions? These start the new coroutine in place of the SEND that - * awaits it. GET_AWAITABLE never fails for a new coroutine. */ + * instructions? These start the body of the coroutine in place of the + * SEND that awaits it, without creating the coroutine object first. + * GET_AWAITABLE never fails for a new coroutine. Coroutine origin + * tracking needs the object at once (see _GUARD_ASYNC_CALL_TAIL). */ static bool can_start_coroutine(PyCodeObject *code, _Py_CODEUNIT *instr) { @@ -1764,6 +1766,9 @@ can_start_coroutine(PyCodeObject *code, _Py_CODEUNIT *instr) if (coro_flags != CO_COROUTINE) { return false; } + if (_PyThreadState_GET()->coroutine_origin_tracking_depth != 0) { + return false; + } _Py_CODEUNIT *next = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[instr->op.code]]; return _PyCode_AwaitedCallSend(next) != NULL; }