From 592e160c131a8cd34b536df7e56ea750bc081f79 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 7 Nov 2026 10:49:58 +0100 Subject: [PATCH 01/15] Add ASYNC_CALL_* superinstructions for awaited calls to coroutine functions Specialize CALL to ASYNC_CALL_PY_EXACT_ARGS, ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, ASYNC_CALL_PY_GENERAL or ASYNC_CALL_BOUND_METHOD_GENERAL when the called function is a coroutine function and the call is awaited at once, as in `await f(...)`, `async with` and async comprehensions. These superinstructions merge the CALL with the first execution of the GET_AWAITABLE, PUSH_NULL, LOAD_COMMON_CONSTANT None and SEND instructions that follow it: the new coroutine is started directly instead of being returned to the caller first. They reuse the uops of the matching CALL_* specialization, with the new _ASYNC_CALL_GEN_FRAME uop in place of _SAVE_RETURN_OFFSET, and the specializer reuses specialize_py_call(). RETURN_GENERATOR now does nothing in a frame that already belongs to a generator, so that coroutine functions whose prologue has COPY_FREE_VARS or MAKE_CELL can use the fast path too. Co-Authored-By: Claude Fable 5.1 --- Include/internal/pycore_opcode_metadata.h | 42 +- Include/internal/pycore_uop_ids.h | 11 +- Include/internal/pycore_uop_metadata.h | 36 + Include/opcode_ids.h | 184 ++--- InternalDocs/generators.md | 15 + Lib/_opcode_metadata.py | 188 ++--- Lib/test/test_capi/test_opt.py | 31 + Lib/test/test_coroutines.py | 337 +++++++++ Lib/test/test_monitoring.py | 43 ++ Lib/test/test_opcache.py | 141 ++++ Modules/_testinternalcapi/test_cases.c.h | 829 +++++++++++++++++++++- Modules/_testinternalcapi/test_targets.h | 36 +- Python/bytecodes.c | 178 ++++- Python/executor_cases.c.h | 237 ++++++- Python/generated_cases.c.h | 829 +++++++++++++++++++++- Python/opcode_targets.h | 36 +- Python/optimizer.c | 7 + Python/optimizer_bytecodes.c | 12 + Python/optimizer_cases.c.h | 23 + Python/record_functions.c.h | 8 + Python/specialize.c | 32 +- 21 files changed, 2927 insertions(+), 328 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 04b826ed09144e..4c38aa7891df2a 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -38,6 +38,14 @@ int _PyOpcode_num_popped(int opcode, int oparg) { switch(opcode) { case ANNOTATIONS_PLACEHOLDER: return 0; + case ASYNC_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case ASYNC_CALL_BOUND_METHOD_GENERAL: + return 2 + oparg; + case ASYNC_CALL_PY_EXACT_ARGS: + return 2 + oparg; + case ASYNC_CALL_PY_GENERAL: + return 2 + oparg; case BINARY_OP: return 2; case BINARY_OP_ADD_FLOAT: @@ -533,6 +541,14 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { switch(opcode) { case ANNOTATIONS_PLACEHOLDER: return 0; + case ASYNC_CALL_BOUND_METHOD_EXACT_ARGS: + return 2; + case ASYNC_CALL_BOUND_METHOD_GENERAL: + return 2; + case ASYNC_CALL_PY_EXACT_ARGS: + return 2; + case ASYNC_CALL_PY_GENERAL: + return 2; case BINARY_OP: return 1; case BINARY_OP_ADD_FLOAT: @@ -1100,6 +1116,10 @@ struct opcode_metadata { PyAPI_DATA(const struct opcode_metadata) _PyOpcode_opcode_metadata[267]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, @@ -1345,7 +1365,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { }; #endif -#define MAX_UOP_PER_EXPANSION 11 +#define MAX_UOP_PER_EXPANSION 12 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1355,6 +1375,10 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 12, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = { .nuops = 9, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_PY_EXACT_ARGS] = { .nuops = 10, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_PY_GENERAL] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [BINARY_OP] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _BINARY_OP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [BINARY_OP_ADD_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_ADD_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, @@ -1553,6 +1577,10 @@ PyAPI_DATA(const char) *_PyOpcode_OpName[267]; #ifdef NEED_OPCODE_METADATA const char *_PyOpcode_OpName[267] = { [ANNOTATIONS_PLACEHOLDER] = "ANNOTATIONS_PLACEHOLDER", + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = "ASYNC_CALL_BOUND_METHOD_EXACT_ARGS", + [ASYNC_CALL_BOUND_METHOD_GENERAL] = "ASYNC_CALL_BOUND_METHOD_GENERAL", + [ASYNC_CALL_PY_EXACT_ARGS] = "ASYNC_CALL_PY_EXACT_ARGS", + [ASYNC_CALL_PY_GENERAL] = "ASYNC_CALL_PY_GENERAL", [BINARY_OP] = "BINARY_OP", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", @@ -1839,10 +1867,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [219] = 219, - [220] = 220, - [221] = 221, - [222] = 222, [223] = 223, [224] = 224, [225] = 225, @@ -1853,6 +1877,10 @@ const uint8_t _PyOpcode_Deopt[256] = { [230] = 230, [231] = 231, [232] = 232, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = CALL, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = CALL, + [ASYNC_CALL_PY_EXACT_ARGS] = CALL, + [ASYNC_CALL_PY_GENERAL] = CALL, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -2100,10 +2128,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 219: \ - case 220: \ - case 221: \ - case 222: \ case 223: \ case 224: \ case 225: \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ba6e0252024a6d..cd0132eb8cd66e 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -13,6 +13,7 @@ enum { _EXIT_TRACE = 300, _SET_IP, _ALLOCATE_OBJECT, + _ASYNC_CALL_GEN_FRAME, _BINARY_OP, _BINARY_OP_ADD_FLOAT, _BINARY_OP_ADD_FLOAT_INPLACE, @@ -128,6 +129,7 @@ enum { _GET_ITER, _GET_ITER_TRAD, _GUARD_3OS_ASYNC_GEN_ASEND, + _GUARD_ASYNC_CALL_TAIL, _GUARD_BINARY_OP_EXTEND, _GUARD_BINARY_OP_EXTEND_LHS, _GUARD_BINARY_OP_EXTEND_RHS, @@ -366,7 +368,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 654 +#define MAX_UOP_ID 656 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -443,6 +445,7 @@ enum { enum { _ALLOCATE_OBJECT_r00 = MAX_UOP_ID + 1, + _ASYNC_CALL_GEN_FRAME_r13, _BINARY_OP_r23, _BINARY_OP_ADD_FLOAT_r03, _BINARY_OP_ADD_FLOAT_r13, @@ -698,6 +701,10 @@ enum { _GUARD_3OS_ASYNC_GEN_ASEND_r13, _GUARD_3OS_ASYNC_GEN_ASEND_r23, _GUARD_3OS_ASYNC_GEN_ASEND_r33, + _GUARD_ASYNC_CALL_TAIL_r00, + _GUARD_ASYNC_CALL_TAIL_r11, + _GUARD_ASYNC_CALL_TAIL_r22, + _GUARD_ASYNC_CALL_TAIL_r33, _GUARD_BINARY_OP_EXTEND_r22, _GUARD_BINARY_OP_EXTEND_LHS_r02, _GUARD_BINARY_OP_EXTEND_LHS_r12, @@ -1444,7 +1451,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1655 +#define MAX_UOP_REGS_ID 1662 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0cd82c53aa932a..b4fa92a266df4b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -352,6 +352,8 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_ASYNC_CALL_TAIL] = HAS_DEOPT_FLAG, + [_ASYNC_CALL_GEN_FRAME] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -3300,6 +3302,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_ASYNC_CALL_TAIL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_ASYNC_CALL_TAIL_r00 }, + { 1, 1, _GUARD_ASYNC_CALL_TAIL_r11 }, + { 2, 2, _GUARD_ASYNC_CALL_TAIL_r22 }, + { 3, 3, _GUARD_ASYNC_CALL_TAIL_r33 }, + }, + }, + [_ASYNC_CALL_GEN_FRAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 3, 3, _ASYNC_CALL_GEN_FRAME_r13 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_MAYBE_EXPAND_METHOD_KW] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -4642,6 +4662,11 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, [_CALL_METHOD_DESCRIPTOR_FAST_r00] = _CALL_METHOD_DESCRIPTOR_FAST, [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_INLINE, + [_GUARD_ASYNC_CALL_TAIL_r00] = _GUARD_ASYNC_CALL_TAIL, + [_GUARD_ASYNC_CALL_TAIL_r11] = _GUARD_ASYNC_CALL_TAIL, + [_GUARD_ASYNC_CALL_TAIL_r22] = _GUARD_ASYNC_CALL_TAIL, + [_GUARD_ASYNC_CALL_TAIL_r33] = _GUARD_ASYNC_CALL_TAIL, + [_ASYNC_CALL_GEN_FRAME_r13] = _ASYNC_CALL_GEN_FRAME, [_MAYBE_EXPAND_METHOD_KW_r11] = _MAYBE_EXPAND_METHOD_KW, [_PY_FRAME_KW_r11] = _PY_FRAME_KW, [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, @@ -4867,6 +4892,8 @@ const uint16_t _PyUop_SpillsAndReloads[4][4] = { const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_ALLOCATE_OBJECT] = "_ALLOCATE_OBJECT", [_ALLOCATE_OBJECT_r00] = "_ALLOCATE_OBJECT_r00", + [_ASYNC_CALL_GEN_FRAME] = "_ASYNC_CALL_GEN_FRAME", + [_ASYNC_CALL_GEN_FRAME_r13] = "_ASYNC_CALL_GEN_FRAME_r13", [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_r23] = "_BINARY_OP_r23", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", @@ -5250,6 +5277,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = "_GUARD_3OS_ASYNC_GEN_ASEND_r13", [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = "_GUARD_3OS_ASYNC_GEN_ASEND_r23", [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = "_GUARD_3OS_ASYNC_GEN_ASEND_r33", + [_GUARD_ASYNC_CALL_TAIL] = "_GUARD_ASYNC_CALL_TAIL", + [_GUARD_ASYNC_CALL_TAIL_r00] = "_GUARD_ASYNC_CALL_TAIL_r00", + [_GUARD_ASYNC_CALL_TAIL_r11] = "_GUARD_ASYNC_CALL_TAIL_r11", + [_GUARD_ASYNC_CALL_TAIL_r22] = "_GUARD_ASYNC_CALL_TAIL_r22", + [_GUARD_ASYNC_CALL_TAIL_r33] = "_GUARD_ASYNC_CALL_TAIL_r33", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", [_GUARD_BINARY_OP_EXTEND_LHS] = "_GUARD_BINARY_OP_EXTEND_LHS", @@ -6832,6 +6864,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_METHOD_DESCRIPTOR_FAST_INLINE: return 0; + case _GUARD_ASYNC_CALL_TAIL: + return 0; + case _ASYNC_CALL_GEN_FRAME: + return 1; case _MAYBE_EXPAND_METHOD_KW: return 0; case _PY_FRAME_KW: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 11342ae451b9f6..7a9179dd97271c 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -128,96 +128,100 @@ extern "C" { #define UNPACK_SEQUENCE 115 #define YIELD_VALUE 116 #define RESUME 128 -#define BINARY_OP_ADD_FLOAT 129 -#define BINARY_OP_ADD_INT 130 -#define BINARY_OP_ADD_UNICODE 131 -#define BINARY_OP_EXTEND 132 -#define BINARY_OP_MULTIPLY_FLOAT 133 -#define BINARY_OP_MULTIPLY_INT 134 -#define BINARY_OP_SUBSCR_DICT 135 -#define BINARY_OP_SUBSCR_GETITEM 136 -#define BINARY_OP_SUBSCR_LIST_INT 137 -#define BINARY_OP_SUBSCR_LIST_SLICE 138 -#define BINARY_OP_SUBSCR_STR_INT 139 -#define BINARY_OP_SUBSCR_TUPLE_INT 140 -#define BINARY_OP_SUBSCR_USTR_INT 141 -#define BINARY_OP_SUBTRACT_FLOAT 142 -#define BINARY_OP_SUBTRACT_INT 143 -#define CALL_ALLOC_AND_ENTER_INIT 144 -#define CALL_BOUND_METHOD_EXACT_ARGS 145 -#define CALL_BOUND_METHOD_GENERAL 146 -#define CALL_BUILTIN_CLASS 147 -#define CALL_BUILTIN_FAST 148 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 149 -#define CALL_BUILTIN_O 150 -#define CALL_EX_NON_PY_GENERAL 151 -#define CALL_EX_PY 152 -#define CALL_ISINSTANCE 153 -#define CALL_KW_BOUND_METHOD 154 -#define CALL_KW_NON_PY 155 -#define CALL_KW_PY 156 -#define CALL_LEN 157 -#define CALL_LIST_APPEND 158 -#define CALL_METHOD_DESCRIPTOR_FAST 159 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 160 -#define CALL_METHOD_DESCRIPTOR_NOARGS 161 -#define CALL_METHOD_DESCRIPTOR_O 162 -#define CALL_NON_PY_GENERAL 163 -#define CALL_PY_EXACT_ARGS 164 -#define CALL_PY_GENERAL 165 -#define CALL_STR_1 166 -#define CALL_TUPLE_1 167 -#define CALL_TYPE_1 168 -#define COMPARE_OP_FLOAT 169 -#define COMPARE_OP_INT 170 -#define COMPARE_OP_STR 171 -#define CONTAINS_OP_DICT 172 -#define CONTAINS_OP_SET 173 -#define FOR_ITER_GEN 174 -#define FOR_ITER_LIST 175 -#define FOR_ITER_RANGE 176 -#define FOR_ITER_TUPLE 177 -#define FOR_ITER_VIRTUAL 178 -#define GET_ITER_SELF 179 -#define GET_ITER_VIRTUAL 180 -#define JUMP_BACKWARD_JIT 181 -#define JUMP_BACKWARD_NO_JIT 182 -#define LOAD_ATTR_CLASS 183 -#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 184 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 185 -#define LOAD_ATTR_INSTANCE_VALUE 186 -#define LOAD_ATTR_METHOD_LAZY_DICT 187 -#define LOAD_ATTR_METHOD_NO_DICT 188 -#define LOAD_ATTR_METHOD_WITH_VALUES 189 -#define LOAD_ATTR_MODULE 190 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 191 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 192 -#define LOAD_ATTR_PROPERTY 193 -#define LOAD_ATTR_SLOT 194 -#define LOAD_ATTR_WITH_HINT 195 -#define LOAD_GLOBAL_BUILTIN 196 -#define LOAD_GLOBAL_MODULE 197 -#define LOAD_SUPER_ATTR_ATTR 198 -#define LOAD_SUPER_ATTR_METHOD 199 -#define RESUME_CHECK 200 -#define RESUME_CHECK_JIT 201 -#define SEND_ASYNC_GEN 202 -#define SEND_GEN 203 -#define SEND_VIRTUAL 204 -#define STORE_ATTR_INSTANCE_VALUE 205 -#define STORE_ATTR_SLOT 206 -#define STORE_ATTR_WITH_HINT 207 -#define STORE_SUBSCR_DICT 208 -#define STORE_SUBSCR_LIST_INT 209 -#define TO_BOOL_ALWAYS_TRUE 210 -#define TO_BOOL_BOOL 211 -#define TO_BOOL_INT 212 -#define TO_BOOL_LIST 213 -#define TO_BOOL_NONE 214 -#define TO_BOOL_STR 215 -#define UNPACK_SEQUENCE_LIST 216 -#define UNPACK_SEQUENCE_TUPLE 217 -#define UNPACK_SEQUENCE_TWO_TUPLE 218 +#define ASYNC_CALL_BOUND_METHOD_EXACT_ARGS 129 +#define ASYNC_CALL_BOUND_METHOD_GENERAL 130 +#define ASYNC_CALL_PY_EXACT_ARGS 131 +#define ASYNC_CALL_PY_GENERAL 132 +#define BINARY_OP_ADD_FLOAT 133 +#define BINARY_OP_ADD_INT 134 +#define BINARY_OP_ADD_UNICODE 135 +#define BINARY_OP_EXTEND 136 +#define BINARY_OP_MULTIPLY_FLOAT 137 +#define BINARY_OP_MULTIPLY_INT 138 +#define BINARY_OP_SUBSCR_DICT 139 +#define BINARY_OP_SUBSCR_GETITEM 140 +#define BINARY_OP_SUBSCR_LIST_INT 141 +#define BINARY_OP_SUBSCR_LIST_SLICE 142 +#define BINARY_OP_SUBSCR_STR_INT 143 +#define BINARY_OP_SUBSCR_TUPLE_INT 144 +#define BINARY_OP_SUBSCR_USTR_INT 145 +#define BINARY_OP_SUBTRACT_FLOAT 146 +#define BINARY_OP_SUBTRACT_INT 147 +#define CALL_ALLOC_AND_ENTER_INIT 148 +#define CALL_BOUND_METHOD_EXACT_ARGS 149 +#define CALL_BOUND_METHOD_GENERAL 150 +#define CALL_BUILTIN_CLASS 151 +#define CALL_BUILTIN_FAST 152 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 153 +#define CALL_BUILTIN_O 154 +#define CALL_EX_NON_PY_GENERAL 155 +#define CALL_EX_PY 156 +#define CALL_ISINSTANCE 157 +#define CALL_KW_BOUND_METHOD 158 +#define CALL_KW_NON_PY 159 +#define CALL_KW_PY 160 +#define CALL_LEN 161 +#define CALL_LIST_APPEND 162 +#define CALL_METHOD_DESCRIPTOR_FAST 163 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 164 +#define CALL_METHOD_DESCRIPTOR_NOARGS 165 +#define CALL_METHOD_DESCRIPTOR_O 166 +#define CALL_NON_PY_GENERAL 167 +#define CALL_PY_EXACT_ARGS 168 +#define CALL_PY_GENERAL 169 +#define CALL_STR_1 170 +#define CALL_TUPLE_1 171 +#define CALL_TYPE_1 172 +#define COMPARE_OP_FLOAT 173 +#define COMPARE_OP_INT 174 +#define COMPARE_OP_STR 175 +#define CONTAINS_OP_DICT 176 +#define CONTAINS_OP_SET 177 +#define FOR_ITER_GEN 178 +#define FOR_ITER_LIST 179 +#define FOR_ITER_RANGE 180 +#define FOR_ITER_TUPLE 181 +#define FOR_ITER_VIRTUAL 182 +#define GET_ITER_SELF 183 +#define GET_ITER_VIRTUAL 184 +#define JUMP_BACKWARD_JIT 185 +#define JUMP_BACKWARD_NO_JIT 186 +#define LOAD_ATTR_CLASS 187 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 188 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 189 +#define LOAD_ATTR_INSTANCE_VALUE 190 +#define LOAD_ATTR_METHOD_LAZY_DICT 191 +#define LOAD_ATTR_METHOD_NO_DICT 192 +#define LOAD_ATTR_METHOD_WITH_VALUES 193 +#define LOAD_ATTR_MODULE 194 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 195 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 196 +#define LOAD_ATTR_PROPERTY 197 +#define LOAD_ATTR_SLOT 198 +#define LOAD_ATTR_WITH_HINT 199 +#define LOAD_GLOBAL_BUILTIN 200 +#define LOAD_GLOBAL_MODULE 201 +#define LOAD_SUPER_ATTR_ATTR 202 +#define LOAD_SUPER_ATTR_METHOD 203 +#define RESUME_CHECK 204 +#define RESUME_CHECK_JIT 205 +#define SEND_ASYNC_GEN 206 +#define SEND_GEN 207 +#define SEND_VIRTUAL 208 +#define STORE_ATTR_INSTANCE_VALUE 209 +#define STORE_ATTR_SLOT 210 +#define STORE_ATTR_WITH_HINT 211 +#define STORE_SUBSCR_DICT 212 +#define STORE_SUBSCR_LIST_INT 213 +#define TO_BOOL_ALWAYS_TRUE 214 +#define TO_BOOL_BOOL 215 +#define TO_BOOL_INT 216 +#define TO_BOOL_LIST 217 +#define TO_BOOL_NONE 218 +#define TO_BOOL_STR 219 +#define UNPACK_SEQUENCE_LIST 220 +#define UNPACK_SEQUENCE_TUPLE 221 +#define UNPACK_SEQUENCE_TWO_TUPLE 222 #define INSTRUMENTED_END_FOR 233 #define INSTRUMENTED_POP_ITER 234 #define INSTRUMENTED_END_SEND 235 diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index fa652cd231cfa9..18e7dd0a920c39 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -104,3 +104,18 @@ so this data flow works along the chain (see `gen_send_ex2()` in Recall that a generator's `__next__` function simply calls `self.send(None)`, so all this works the same in generators and coroutines, but only coroutines use the value of the argument to `send`. + +An `await` expression is compiled like `yield from`, with +[`GET_AWAITABLE`](https://docs.python.org/dev/library/dis.html#opcode-GET_AWAITABLE) +in place of `GET_ITER`. When the awaited object is the result of a call to +a coroutine function, as in `await f()`, the `CALL` is +[`specialized`](interpreter.md) to one of the `ASYNC_CALL_*` instructions, +such as `ASYNC_CALL_PY_EXACT_ARGS`. These are superinstructions: they merge +the `CALL` with the first execution of the `GET_AWAITABLE`, `PUSH_NULL`, +`LOAD_COMMON_CONSTANT` and `SEND` instructions that follow it. + +`ASYNC_CALL_*` create the frame with the micro-ops of the matching `CALL_*` +specialization. They then turn it into the frame of a new coroutine and push +it at once, as `RETURN_GENERATOR` and `SEND_GEN` would do. The coroutine is +not returned to the caller first and the frame of the caller is left as if +it were executing the `SEND`. diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index df92eae151d248..1e99a825a366f5 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -120,6 +120,10 @@ "CALL_PY_GENERAL", "CALL_BOUND_METHOD_GENERAL", "CALL_NON_PY_GENERAL", + "ASYNC_CALL_PY_EXACT_ARGS", + "ASYNC_CALL_BOUND_METHOD_EXACT_ARGS", + "ASYNC_CALL_PY_GENERAL", + "ASYNC_CALL_BOUND_METHOD_GENERAL", ), CALL_KW=( "CALL_KW_BOUND_METHOD", @@ -133,97 +137,101 @@ ) _specialized_opmap = frozendict( - BINARY_OP_ADD_FLOAT=129, - BINARY_OP_ADD_INT=130, - BINARY_OP_ADD_UNICODE=131, - BINARY_OP_EXTEND=132, + ASYNC_CALL_BOUND_METHOD_EXACT_ARGS=129, + ASYNC_CALL_BOUND_METHOD_GENERAL=130, + ASYNC_CALL_PY_EXACT_ARGS=131, + ASYNC_CALL_PY_GENERAL=132, + BINARY_OP_ADD_FLOAT=133, + BINARY_OP_ADD_INT=134, + BINARY_OP_ADD_UNICODE=135, + BINARY_OP_EXTEND=136, BINARY_OP_INPLACE_ADD_UNICODE=3, - BINARY_OP_MULTIPLY_FLOAT=133, - BINARY_OP_MULTIPLY_INT=134, - BINARY_OP_SUBSCR_DICT=135, - BINARY_OP_SUBSCR_GETITEM=136, - BINARY_OP_SUBSCR_LIST_INT=137, - BINARY_OP_SUBSCR_LIST_SLICE=138, - BINARY_OP_SUBSCR_STR_INT=139, - BINARY_OP_SUBSCR_TUPLE_INT=140, - BINARY_OP_SUBSCR_USTR_INT=141, - BINARY_OP_SUBTRACT_FLOAT=142, - BINARY_OP_SUBTRACT_INT=143, - CALL_ALLOC_AND_ENTER_INIT=144, - CALL_BOUND_METHOD_EXACT_ARGS=145, - CALL_BOUND_METHOD_GENERAL=146, - CALL_BUILTIN_CLASS=147, - CALL_BUILTIN_FAST=148, - CALL_BUILTIN_FAST_WITH_KEYWORDS=149, - CALL_BUILTIN_O=150, - CALL_EX_NON_PY_GENERAL=151, - CALL_EX_PY=152, - CALL_ISINSTANCE=153, - CALL_KW_BOUND_METHOD=154, - CALL_KW_NON_PY=155, - CALL_KW_PY=156, - CALL_LEN=157, - CALL_LIST_APPEND=158, - CALL_METHOD_DESCRIPTOR_FAST=159, - CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=160, - CALL_METHOD_DESCRIPTOR_NOARGS=161, - CALL_METHOD_DESCRIPTOR_O=162, - CALL_NON_PY_GENERAL=163, - CALL_PY_EXACT_ARGS=164, - CALL_PY_GENERAL=165, - CALL_STR_1=166, - CALL_TUPLE_1=167, - CALL_TYPE_1=168, - COMPARE_OP_FLOAT=169, - COMPARE_OP_INT=170, - COMPARE_OP_STR=171, - CONTAINS_OP_DICT=172, - CONTAINS_OP_SET=173, - FOR_ITER_GEN=174, - FOR_ITER_LIST=175, - FOR_ITER_RANGE=176, - FOR_ITER_TUPLE=177, - FOR_ITER_VIRTUAL=178, - GET_ITER_SELF=179, - GET_ITER_VIRTUAL=180, - JUMP_BACKWARD_JIT=181, - JUMP_BACKWARD_NO_JIT=182, - LOAD_ATTR_CLASS=183, - LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=184, - LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=185, - LOAD_ATTR_INSTANCE_VALUE=186, - LOAD_ATTR_METHOD_LAZY_DICT=187, - LOAD_ATTR_METHOD_NO_DICT=188, - LOAD_ATTR_METHOD_WITH_VALUES=189, - LOAD_ATTR_MODULE=190, - LOAD_ATTR_NONDESCRIPTOR_NO_DICT=191, - LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=192, - LOAD_ATTR_PROPERTY=193, - LOAD_ATTR_SLOT=194, - LOAD_ATTR_WITH_HINT=195, - LOAD_GLOBAL_BUILTIN=196, - LOAD_GLOBAL_MODULE=197, - LOAD_SUPER_ATTR_ATTR=198, - LOAD_SUPER_ATTR_METHOD=199, - RESUME_CHECK=200, - RESUME_CHECK_JIT=201, - SEND_ASYNC_GEN=202, - SEND_GEN=203, - SEND_VIRTUAL=204, - STORE_ATTR_INSTANCE_VALUE=205, - STORE_ATTR_SLOT=206, - STORE_ATTR_WITH_HINT=207, - STORE_SUBSCR_DICT=208, - STORE_SUBSCR_LIST_INT=209, - TO_BOOL_ALWAYS_TRUE=210, - TO_BOOL_BOOL=211, - TO_BOOL_INT=212, - TO_BOOL_LIST=213, - TO_BOOL_NONE=214, - TO_BOOL_STR=215, - UNPACK_SEQUENCE_LIST=216, - UNPACK_SEQUENCE_TUPLE=217, - UNPACK_SEQUENCE_TWO_TUPLE=218, + BINARY_OP_MULTIPLY_FLOAT=137, + BINARY_OP_MULTIPLY_INT=138, + BINARY_OP_SUBSCR_DICT=139, + BINARY_OP_SUBSCR_GETITEM=140, + BINARY_OP_SUBSCR_LIST_INT=141, + BINARY_OP_SUBSCR_LIST_SLICE=142, + BINARY_OP_SUBSCR_STR_INT=143, + BINARY_OP_SUBSCR_TUPLE_INT=144, + BINARY_OP_SUBSCR_USTR_INT=145, + BINARY_OP_SUBTRACT_FLOAT=146, + BINARY_OP_SUBTRACT_INT=147, + CALL_ALLOC_AND_ENTER_INIT=148, + CALL_BOUND_METHOD_EXACT_ARGS=149, + CALL_BOUND_METHOD_GENERAL=150, + CALL_BUILTIN_CLASS=151, + CALL_BUILTIN_FAST=152, + CALL_BUILTIN_FAST_WITH_KEYWORDS=153, + CALL_BUILTIN_O=154, + CALL_EX_NON_PY_GENERAL=155, + CALL_EX_PY=156, + CALL_ISINSTANCE=157, + CALL_KW_BOUND_METHOD=158, + CALL_KW_NON_PY=159, + CALL_KW_PY=160, + CALL_LEN=161, + CALL_LIST_APPEND=162, + CALL_METHOD_DESCRIPTOR_FAST=163, + CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=164, + CALL_METHOD_DESCRIPTOR_NOARGS=165, + CALL_METHOD_DESCRIPTOR_O=166, + CALL_NON_PY_GENERAL=167, + CALL_PY_EXACT_ARGS=168, + CALL_PY_GENERAL=169, + CALL_STR_1=170, + CALL_TUPLE_1=171, + CALL_TYPE_1=172, + COMPARE_OP_FLOAT=173, + COMPARE_OP_INT=174, + COMPARE_OP_STR=175, + CONTAINS_OP_DICT=176, + CONTAINS_OP_SET=177, + FOR_ITER_GEN=178, + FOR_ITER_LIST=179, + FOR_ITER_RANGE=180, + FOR_ITER_TUPLE=181, + FOR_ITER_VIRTUAL=182, + GET_ITER_SELF=183, + GET_ITER_VIRTUAL=184, + JUMP_BACKWARD_JIT=185, + JUMP_BACKWARD_NO_JIT=186, + LOAD_ATTR_CLASS=187, + LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=188, + LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=189, + LOAD_ATTR_INSTANCE_VALUE=190, + LOAD_ATTR_METHOD_LAZY_DICT=191, + LOAD_ATTR_METHOD_NO_DICT=192, + LOAD_ATTR_METHOD_WITH_VALUES=193, + LOAD_ATTR_MODULE=194, + LOAD_ATTR_NONDESCRIPTOR_NO_DICT=195, + LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=196, + LOAD_ATTR_PROPERTY=197, + LOAD_ATTR_SLOT=198, + LOAD_ATTR_WITH_HINT=199, + LOAD_GLOBAL_BUILTIN=200, + LOAD_GLOBAL_MODULE=201, + LOAD_SUPER_ATTR_ATTR=202, + LOAD_SUPER_ATTR_METHOD=203, + RESUME_CHECK=204, + RESUME_CHECK_JIT=205, + SEND_ASYNC_GEN=206, + SEND_GEN=207, + SEND_VIRTUAL=208, + STORE_ATTR_INSTANCE_VALUE=209, + STORE_ATTR_SLOT=210, + STORE_ATTR_WITH_HINT=211, + STORE_SUBSCR_DICT=212, + STORE_SUBSCR_LIST_INT=213, + TO_BOOL_ALWAYS_TRUE=214, + TO_BOOL_BOOL=215, + TO_BOOL_INT=216, + TO_BOOL_LIST=217, + TO_BOOL_NONE=218, + TO_BOOL_STR=219, + UNPACK_SEQUENCE_LIST=220, + UNPACK_SEQUENCE_TUPLE=221, + UNPACK_SEQUENCE_TWO_TUPLE=222, ) opmap = frozendict( diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 88077aacecbf5b..2c8b490b07efcf 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -5917,6 +5917,37 @@ def testfunc(n): # _POP_TOP_NOP is a sign the optimizer ran and didn't hit bottom. self.assertGreaterEqual(count_ops(ex, "_POP_TOP_NOP"), 1) + def test_async_call_gen_frame(self): + + async def inner(i): + return i + 1 + async def outer(n): + total = 0 + for i in range(n): + total += await inner(i) + return total + def testfunc(n): + try: + outer(n).send(None) + except StopIteration as ex: + return ex.value + + n = TIER2_THRESHOLD * 2 + self.assertEqual(testfunc(n), sum(range(1, n + 1))) + ex = get_first_executor(outer) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + # The coroutine is started without being returned to outer() first. + self.assertIn("_ASYNC_CALL_GEN_FRAME", uops) + self.assertNotIn("_RETURN_GENERATOR", uops) + self.assertNotIn("_SEND_GEN_FRAME", uops) + # The guard is only needed in tier 1. + self.assertNotIn("_GUARD_ASYNC_CALL_TAIL", uops) + # inner() returns to the END_SEND of outer(). + self.assertIn("_RETURN_VALUE", uops) + self.assertIn("_END_SEND", uops) + def test_send_virtual(self): def send_list(n): diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index c5900da09501b3..62b690d2d5c84f 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2401,6 +2401,343 @@ async def coro(): support.is_emscripten or support.is_wasi, "asyncio does not work under Emscripten/WASI yet." ) +class AwaitCallTest(unittest.TestCase): + # The CALL in `await f(...)` is specialized to ASYNC_CALL_* when f() is + # a coroutine function: the coroutine is then started without being + # returned to the caller first. + # Run each call often enough to exercise the specialized forms too. + N = 100 + + def test_function(self): + async def add(a, b=10): + return a + b + + async def caller(): + total = 0 + for i in range(self.N): + total += await add(i, 1) # Exact arguments + total += await add(i) # Default argument + return total + + expected = sum(i + 1 + i + 10 for i in range(self.N)) + self.assertEqual(run_async(caller()), ([], expected)) + + def test_method(self): + class A: + async def add(self, a, b=10): + return a + b + + async def caller(): + obj = A() + bound = obj.add + total = 0 + for i in range(self.N): + total += await obj.add(i, 1) + total += await obj.add(i) + total += await bound(i, 1) + total += await bound(i) + return total + + expected = sum(2 * (i + 1 + i + 10) for i in range(self.N)) + self.assertEqual(run_async(caller()), ([], expected)) + + def test_closure(self): + # COPY_FREE_VARS and MAKE_CELL precede RETURN_GENERATOR + def outer(free): + async def inner(arg): + cell = arg + return (lambda: (cell, free, __class__))() + return inner + + async def caller(): + inner = outer("free") + for i in range(self.N): + self.assertEqual(await inner(i), (i, "free", AwaitCallTest)) + + self.assertEqual(run_async(caller()), ([], None)) + + def test_suspend(self): + class Suspend: + def __init__(self, value): + self.value = value + + def __await__(self): + return (yield self.value) + + async def inner(i): + sent = await Suspend(i) + return sent * 2 + + async def middle(i): + return await inner(i) + 1 + + async def outer(): + return [await middle(i) for i in range(self.N)] + + coro = outer() + value = coro.send(None) + for i in range(self.N): + self.assertEqual(value, i) + self.assertEqual(inspect.getcoroutinestate(coro), "CORO_SUSPENDED") + # outer() -> middle() -> inner() -> Suspend().__await__() + self.assertEqual(coro.cr_await.cr_await.cr_code, inner.__code__) + try: + value = coro.send(i + 100) + except StopIteration as ex: + self.assertEqual(i, self.N - 1) + result = ex.value + self.assertEqual(result, [(i + 100) * 2 + 1 for i in range(self.N)]) + + def test_async_with_and_comprehension(self): + class CM: + async def __aenter__(self): + return "entered" + + async def __aexit__(self, *exc_info): + return exc_info[0] is ZeroDivisionError + + async def caller(): + cm = CM() + for i in range(self.N): + async with cm as value: + self.assertEqual(value, "entered") + 1 / 0 + # The call of the comprehension function is awaited. + self.assertEqual([x async for x in asynciter([i])], [i]) + + self.assertEqual(run_async(caller()), ([], None)) + + def test_generator_based_coroutine(self): + @types.coroutine + def gen_coro(value): + sent = yield value + return sent + 1 + + async def caller(): + return [await gen_coro(i) for i in range(self.N)] + + coro = caller() + value = coro.send(None) + for i in range(self.N): + self.assertEqual(value, i) + self.assertIs(coro.cr_await.gi_code, gen_coro.__code__) + try: + value = coro.send(i * 2) + except StopIteration as ex: + self.assertEqual(i, self.N - 1) + result = ex.value + self.assertEqual(result, [i * 2 + 1 for i in range(self.N)]) + + def test_throw_and_close(self): + async def inner(log): + try: + await AsyncYield("suspended") + except ZeroDivisionError: + log.append("caught") + return "handled" + finally: + log.append("finally") + + async def outer(log): + return await inner(log) + + for _ in range(self.N): + log = [] + coro = outer(log) + self.assertEqual(coro.send(None), "suspended") + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertEqual(cm.exception.value, "handled") + self.assertEqual(log, ["caught", "finally"]) + + log = [] + coro = outer(log) + self.assertEqual(coro.send(None), "suspended") + coro.close() + self.assertEqual(log, ["finally"]) + + def test_exception(self): + async def fail(i): + raise ZeroDivisionError(i) + + async def caller(): + for i in range(self.N): + try: + await fail(i) + except ZeroDivisionError as ex: + self.assertEqual(ex.args, (i,)) + tb = traceback.extract_tb(ex.__traceback__) + self.assertEqual([entry.name for entry in tb], + ["caller", "fail"]) + self.assertEqual(tb[0].line, "await fail(i)") + else: + self.fail("ZeroDivisionError not raised") + + self.assertEqual(run_async(caller()), ([], None)) + + def test_wrong_arguments(self): + async def func(a, b=1): + return a + + async def caller(*args): + for i in range(self.N): + try: + await func() + except TypeError as ex: + self.assertIn("missing 1 required positional", str(ex)) + else: + self.fail("TypeError not raised") + + self.assertEqual(run_async(caller()), ([], None)) + + def test_not_a_coroutine_function(self): + async def coro(i): + return i + + def function(i): + return coro(i) + + class Awaitable: + def __init__(self, i): + self.i = i + + def __await__(self): + return self.i + yield + + @types.coroutine + def gen_coro(i): + return i + yield + + def gen(i): + yield + + async def agen(i): + yield + + async def caller(): + for i in range(self.N): + self.assertEqual(await function(i), i) + self.assertEqual(await Awaitable(i), i) + self.assertEqual(await gen_coro(i), i) + for func in int, gen, agen: + with self.assertRaisesRegex(TypeError, "can't be awaited"): + await func(i) + + self.assertEqual(run_async(caller()), ([], None)) + + def test_change_callable(self): + class Awaitable: + def __await__(self): + return 5 + yield + + async def coro(a=1): + return a + + async def coro2(a): + return a + 1 + + def function(a): + return coro(a + 2) + + class A: + async def method(self, a): + return a + 3 + + async def caller(func, expected): + self.assertEqual(await func(1), expected) + + obj = A() + callables = [(coro, 1), (coro2, 2), (function, 3), (obj.method, 4), + (lambda a: Awaitable(), 5)] + for _ in range(self.N): + for func, expected in callables: + with self.subTest(func=func): + self.assertEqual(run_async(caller(func, expected)), + ([], None)) + + def test_already_awaited(self): + async def inner(): + await AsyncYield(None) + + async def outer(coro): + await coro + + async def caller(coro): + return await outer(coro) + + for _ in range(self.N): + awaited = inner() + first = caller(awaited) + first.send(None) + second = caller(awaited) + with self.assertRaisesRegex(RuntimeError, "already"): + second.send(None) + first.close() + + def test_recursion(self): + async def recurse(n): + if n: + return await recurse(n - 1) + 1 + return 0 + + self.assertEqual(run_async(recurse(self.N)), ([], self.N)) + + async def infinite(): + await infinite() + + with support.infinite_recursion(), self.assertRaises(RecursionError): + run_async(infinite()) + + def test_origin_tracking(self): + async def get_origin(): + coro = sys._getframe().f_generator + return coro.cr_origin + + async def caller(): + origins = set() + for _ in range(self.N): + origins.add(await get_origin()) + return origins + + orig_depth = sys.get_coroutine_origin_tracking_depth() + self.addCleanup(sys.set_coroutine_origin_tracking_depth, orig_depth) + sys.set_coroutine_origin_tracking_depth(1) + lineno = caller.__code__.co_firstlineno + 3 + self.assertEqual(run_async(caller()), + ([], {((__file__, lineno, "caller"),)})) + sys.set_coroutine_origin_tracking_depth(0) + self.assertEqual(run_async(caller()), ([], {None})) + + def test_line_events(self): + # The GET_AWAITABLE that follows the CALL starts a new line here, + # so it must not be skipped when tracing lines. + async def inner(): + pass + + async def caller(): + for _ in range(self.N): + await ( + inner() + ) + + def trace(frame, event, arg): + if frame.f_code is caller.__code__ and event == "line": + lines.append(frame.f_lineno - first) + return trace + + first = caller.__code__.co_firstlineno + run_async(caller()) + lines = [] + self.addCleanup(sys.settrace, sys.gettrace()) + sys.settrace(trace) + run_async(caller()) + sys.settrace(None) + self.assertEqual(lines, [1] + [3, 2, 1] * self.N) + + class CoroAsyncIOCompatTest(unittest.TestCase): def test_asyncio_1(self): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 24cc51f4b4053a..689550f78bba5c 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -2573,6 +2573,49 @@ def f(a=1, b=2): self.assertEqual(call_data[0], (f, 1)) self.assertEqual(call_data[1], (f, sys.monitoring.MISSING)) + def test_async_call(self): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + + async def coro(a): + return a + + async def caller(n): + for i in range(n): + await coro(i) + + def run(coro): + with self.assertRaises(StopIteration): + coro.send(None) + + def call(code, offset, callable, arg0): + if code is caller.__code__: + calls.append((callable, arg0)) + + def start(code, offset): + if code is coro.__code__: + starts.append(code) + + n = _testinternalcapi.SPECIALIZATION_THRESHOLD + 1 + expected = [(range, n)] + [(coro, i) for i in range(n)] + # Specialize the CALL to ASYNC_CALL_* before it is instrumented. + run(caller(n)) + + calls = [] + starts = [] + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.CALL, call) + sys.monitoring.register_callback(0, E.PY_START, start) + sys.monitoring.set_events(0, E.CALL | E.PY_START) + run(caller(n)) + sys.monitoring.set_events(0, 0) + self.assertEqual(calls, expected) + self.assertEqual(len(starts), n) + + run(caller(n)) + self.assertEqual(calls, expected) + def test_instruction_explicit_callback(self): # gh-122247 # Calling the instruction event callback explicitly should not diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 5f7c15f93a8332..b6654957f1c22b 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1630,8 +1630,15 @@ def run_async(coro): except StopIteration: break + @types.coroutine + def suspend(): + yield + class CM: async def __aenter__(self): + # ASYNC_CALL_* starts the coroutine in place of the SEND + # in send_with(), which is only executed to resume it. + await suspend() return self async def __aexit__(self, *exc): @@ -1693,6 +1700,140 @@ async def send_for(): self.assert_specialized(send_for, "SEND_ASYNC_GEN") self.assert_no_opcode(send_for, "SEND") + @cpython_only + @requires_specialization + def test_async_call(self): + def run_async(coro): + try: + coro.send(None) + except StopIteration as ex: + return ex.value + self.fail("coroutine did not finish") + + async def coro(a, b=1): + return a + b + + class C: + async def method(self, a, b=1): + return a + b + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + pass + + def function(a): + return coro(a) + + obj = C() + + async def py_exact_args(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await coro(i, 2), i + 2) + self.assertEqual(await obj.method(i, 2), i + 2) + + run_async(py_exact_args()) + self.assert_specialized(py_exact_args, "ASYNC_CALL_PY_EXACT_ARGS") + self.assert_no_opcode(py_exact_args, "CALL_PY_EXACT_ARGS") + + async def py_general(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await coro(i), i + 1) + + run_async(py_general()) + self.assert_specialized(py_general, "ASYNC_CALL_PY_GENERAL") + + async def bound_method_exact_args(): + bound = obj.method + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await bound(i, 2), i + 2) + + run_async(bound_method_exact_args()) + self.assert_specialized(bound_method_exact_args, + "ASYNC_CALL_BOUND_METHOD_EXACT_ARGS") + self.assert_no_opcode(bound_method_exact_args, + "CALL_BOUND_METHOD_EXACT_ARGS") + + async def bound_method_general(): + bound = obj.method + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await bound(i), i + 1) + + run_async(bound_method_general()) + self.assert_specialized(bound_method_general, + "ASYNC_CALL_BOUND_METHOD_GENERAL") + self.assert_no_opcode(bound_method_general, + "CALL_BOUND_METHOD_GENERAL") + + # __aenter__() and __aexit__() are awaited too. + async def async_with(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + async with obj as value: + self.assertIs(value, obj) + + run_async(async_with()) + self.assert_specialized(async_with, "ASYNC_CALL_PY_EXACT_ARGS") + self.assert_specialized(async_with, "ASYNC_CALL_PY_GENERAL") + + # Only calls of coroutine functions are merged with the await. + async def not_a_coroutine_function(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await function(i), i + 1) + + run_async(not_a_coroutine_function()) + self.assert_specialized(not_a_coroutine_function, "CALL_PY_EXACT_ARGS") + self.assert_no_opcode(not_a_coroutine_function, + "ASYNC_CALL_PY_EXACT_ARGS") + + # Only awaited calls are merged. + async def not_awaited(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + c = coro(i) + self.assertEqual(await c, i + 1) + + run_async(not_awaited()) + self.assert_specialized(not_awaited, "CALL_PY_GENERAL") + self.assert_no_opcode(not_awaited, "ASYNC_CALL_PY_GENERAL") + + @cpython_only + @requires_specialization + def test_async_call_deopt(self): + def run_async(coro): + try: + coro.send(None) + except StopIteration as ex: + return ex.value + self.fail("coroutine did not finish") + + async def coro(a): + return a + 1 + + async def other(a): + return a + 2 + + def function(a): + return coro(a + 2) + + async def caller(func, n): + return [await func(i) for i in range(n)] + + n = _testinternalcapi.SPECIALIZATION_THRESHOLD + self.assertEqual(run_async(caller(coro, n)), list(range(1, n + 1))) + self.assert_specialized(caller, "ASYNC_CALL_PY_EXACT_ARGS") + + n = _testinternalcapi.SPECIALIZATION_COOLDOWN + self.assertEqual(run_async(caller(function, n)), list(range(3, n + 3))) + self.assert_no_opcode(caller, "ASYNC_CALL_PY_EXACT_ARGS") + + self.assertEqual(run_async(caller(other, n)), list(range(2, n + 2))) + self.assert_specialized(caller, "ASYNC_CALL_PY_EXACT_ARGS") + + # Changing the code of the function invalidates the specialization. + other.__code__ = (lambda a: a).__code__ + with self.assertRaisesRegex(TypeError, "can't be awaited"): + run_async(caller(other, n)) + @cpython_only @requires_specialization def test_store_attr_slot(self): diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 8e7ba1f69553dd..25832238fb2a36 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -18,6 +18,776 @@ /* BEGIN INSTRUCTIONS */ + TARGET(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + #endif + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + tstate->current_frame = callee; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + tstate->current_frame = frame; + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyFrame_Copy(callee, pushed_frame); + _PyFrame_StackPointerInvalidate(frame); + assert(callee->frame_obj == NULL); + pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_BOUND_METHOD_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + #endif + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + tstate->current_frame = callee; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + tstate->current_frame = frame; + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyFrame_Copy(callee, pushed_frame); + _PyFrame_StackPointerInvalidate(frame); + assert(callee->frame_obj == NULL); + pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + tstate->current_frame = callee; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + tstate->current_frame = frame; + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyFrame_Copy(callee, pushed_frame); + _PyFrame_StackPointerInvalidate(frame); + assert(callee->frame_obj == NULL); + pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + tstate->current_frame = callee; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + tstate->current_frame = frame; + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyFrame_Copy(callee, pushed_frame); + _PyFrame_StackPointerInvalidate(frame); + assert(callee->frame_obj == NULL); + pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(BINARY_OP) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP; @@ -11440,34 +12210,39 @@ next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4..d3b5ac10b2f440 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -129,6 +129,10 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&TARGET_RESUME, + &&TARGET_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_ASYNC_CALL_BOUND_METHOD_GENERAL, + &&TARGET_ASYNC_CALL_PY_EXACT_ARGS, + &&TARGET_ASYNC_CALL_PY_GENERAL, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, @@ -229,10 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -478,10 +478,10 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -531,6 +531,10 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PA static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_GENERAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); @@ -774,6 +778,10 @@ JUMP_TO_LABEL(error); } static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_ASYNC_CALL_PY_GENERAL, [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -1016,10 +1024,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, [223] = _TAIL_CALL_UNKNOWN_OPCODE, [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1032,6 +1036,10 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, @@ -1274,10 +1282,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, [223] = _TAIL_CALL_UNKNOWN_OPCODE, [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 42ad83daa1ea46..ddd4c5c186e65a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4314,6 +4314,10 @@ dummy_func( CALL_PY_GENERAL, CALL_BOUND_METHOD_GENERAL, CALL_NON_PY_GENERAL, + ASYNC_CALL_PY_EXACT_ARGS, + ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + ASYNC_CALL_PY_GENERAL, + ASYNC_CALL_BOUND_METHOD_GENERAL, }; specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { @@ -5345,6 +5349,130 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; + // The ASYNC_CALL_* instructions are specializations of CALL for + // calls to coroutine functions that are awaited at once, as in + // `await f(...)`. They are superinstructions: they merge the CALL + // with the first execution of the instructions that follow it, + // GET_AWAITABLE; PUSH_NULL; LOAD_COMMON_CONSTANT None; SEND + // and start the new coroutine without returning it to the caller + // first. They are built from the uops of the matching CALL_* + // specialization, with _ASYNC_CALL_GEN_FRAME in place of + // _SAVE_RETURN_OFFSET. + + // ASYNC_CALL_* skip the instructions up to and including SEND, so + // none of them may be instrumented. Only the first and the last of + // them can start a line. + // Executors are invalidated when the code is instrumented. + op(_GUARD_ASYNC_CALL_TAIL, (--)) { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + DEOPT_IF(next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE); + DEOPT_IF(next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE); + #endif + } + + // Turns the frame of a call to a coroutine function into a running + // coroutine: the equivalent of executing RETURN_GENERATOR in that + // frame, followed by GET_AWAITABLE, PUSH_NULL, LOAD_COMMON_CONSTANT + // None and SEND_GEN in this one. + op(_ASYNC_CALL_GEN_FRAME, (new_frame -- coro, null, gen_frame)) { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + DEAD(new_frame); + SYNC_SP(); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + // _Py_MakeCoro() expects the current frame to be the frame + // of the coroutine function, as it is in RETURN_GENERATOR. + tstate->current_frame = callee; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + tstate->current_frame = frame; + if (gen == NULL) { + _PyEval_FrameClearAndPop(tstate, callee); + ERROR_NO_POP(); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + // Skip RETURN_GENERATOR and the POP_TOP that follows it. + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + // Otherwise RETURN_GENERATOR is preceded by COPY_FREE_VARS or + // MAKE_CELL. They run in the frame of the coroutine and + // RETURN_GENERATOR does nothing in such a frame. + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_Copy(callee, pushed_frame); + assert(callee->frame_obj == NULL); + pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; + _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); + _PyThreadState_PopFrame(tstate, callee); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + // The coroutine yields and returns to the SEND, which follows + // GET_AWAITABLE, PUSH_NULL and LOAD_COMMON_CONSTANT. + frame->instr_ptr += INSTRUCTION_SIZE + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + + macro(ASYNC_CALL_PY_EXACT_ARGS) = + _RECORD_CALLABLE + + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_FUNCTION_VERSION + + _CHECK_FUNCTION_EXACT_ARGS + + _CHECK_STACK_SPACE + + _CHECK_RECURSION_REMAINING + + _INIT_CALL_PY_EXACT_ARGS + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS) = + _RECORD_BOUND_METHOD + + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + + _INIT_CALL_BOUND_METHOD_EXACT_ARGS + + flush + // In case the following deopt + _CHECK_FUNCTION_VERSION + + _CHECK_FUNCTION_EXACT_ARGS + + _CHECK_STACK_SPACE + + _CHECK_RECURSION_REMAINING + + _INIT_CALL_PY_EXACT_ARGS + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_PY_GENERAL) = + _RECORD_CALLABLE + + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_FUNCTION_VERSION + + _CHECK_RECURSION_REMAINING + + _PY_FRAME_GENERAL + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_BOUND_METHOD_GENERAL) = + _RECORD_BOUND_METHOD + + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_METHOD_VERSION + + _EXPAND_METHOD + + flush + // so that self is in the argument array + _CHECK_RECURSION_REMAINING + + _PY_FRAME_GENERAL + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + // Cache layout: counter/1, func_version/2 family(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW) = { CALL_KW_BOUND_METHOD, @@ -5804,27 +5932,35 @@ dummy_func( } inst(RETURN_GENERATOR, (-- res)) { - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - ERROR_IF(gen == NULL); - assert(STACK_LEVEL() <= 2); - SAVE_STACK(); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - RELOAD_STACK(); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + // ASYNC_CALL_* has already created the coroutine and it is + // running in this frame. See _ASYNC_CALL_GEN_FRAME. + // Push the value that the following POP_TOP discards. + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + ERROR_IF(gen == NULL); + assert(STACK_LEVEL() <= 2); + SAVE_STACK(); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + RELOAD_STACK(); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + } } inst(BUILD_SLICE, (args[oparg] -- slice)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bc85ccd9ddb4fc..cfed39174dd924 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -19918,6 +19918,182 @@ break; } + case _GUARD_ASYNC_CALL_TAIL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #endif + SET_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_ASYNC_CALL_TAIL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_ASYNC_CALL_TAIL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_ASYNC_CALL_TAIL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + #endif + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _ASYNC_CALL_GEN_FRAME_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + new_frame = _stack_item_0; + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + tstate->current_frame = callee; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + tstate->current_frame = frame; + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyFrame_Copy(callee, pushed_frame); + _PyFrame_StackPointerInvalidate(frame); + assert(callee->frame_obj == NULL); + pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = null; + _tos_cache0 = coro; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _MAYBE_EXPAND_METHOD_KW_r11: { @@ -20718,35 +20894,40 @@ CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); _tos_cache0 = res; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ee9b64107bb9f2..6c43314f26c510 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -18,6 +18,776 @@ /* BEGIN INSTRUCTIONS */ + TARGET(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + #endif + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + tstate->current_frame = callee; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + tstate->current_frame = frame; + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyFrame_Copy(callee, pushed_frame); + _PyFrame_StackPointerInvalidate(frame); + assert(callee->frame_obj == NULL); + pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_BOUND_METHOD_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + #endif + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + tstate->current_frame = callee; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + tstate->current_frame = frame; + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyFrame_Copy(callee, pushed_frame); + _PyFrame_StackPointerInvalidate(frame); + assert(callee->frame_obj == NULL); + pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + tstate->current_frame = callee; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + tstate->current_frame = frame; + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyFrame_Copy(callee, pushed_frame); + _PyFrame_StackPointerInvalidate(frame); + assert(callee->frame_obj == NULL); + pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + tstate->current_frame = callee; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + tstate->current_frame = frame; + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyFrame_Copy(callee, pushed_frame); + _PyFrame_StackPointerInvalidate(frame); + assert(callee->frame_obj == NULL); + pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyThreadState_PopFrame(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(BINARY_OP) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP; @@ -11437,34 +12207,39 @@ next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4..d3b5ac10b2f440 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -129,6 +129,10 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&TARGET_RESUME, + &&TARGET_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_ASYNC_CALL_BOUND_METHOD_GENERAL, + &&TARGET_ASYNC_CALL_PY_EXACT_ARGS, + &&TARGET_ASYNC_CALL_PY_GENERAL, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, @@ -229,10 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -478,10 +478,10 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -531,6 +531,10 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PA static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_GENERAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); @@ -774,6 +778,10 @@ JUMP_TO_LABEL(error); } static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_ASYNC_CALL_PY_GENERAL, [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -1016,10 +1024,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, [223] = _TAIL_CALL_UNKNOWN_OPCODE, [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1032,6 +1036,10 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, @@ -1274,10 +1282,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, [223] = _TAIL_CALL_UNKNOWN_OPCODE, [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer.c b/Python/optimizer.c index e05adb344c8d06..e3a0bc2208060d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -783,6 +783,13 @@ _PyJit_translate_single_bytecode_to_trace( goto unsupported; } + // The coroutine was created by ASYNC_CALL_*, so RETURN_GENERATOR + // did not leave its frame. _RETURN_GENERATOR cannot express that. + if (opcode == RETURN_GENERATOR && frame == tracer->prev_state.instr_frame) { + DPRINTF(2, "Unsupported: RETURN_GENERATOR in a running coroutine\n"); + goto unsupported; + } + if (oparg > 0xFFFF) { DPRINTF(2, "Unsupported: oparg too large\n"); unsupported: diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6284885c428832..9bc41b022189d7 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1528,6 +1528,18 @@ dummy_func(void) { gen_frame = PyJitRef_WrapInvalid(new_frame); } + op(_GUARD_ASYNC_CALL_TAIL, (--)) { + // Only needed in tier 1: executors are invalidated + // when the code is instrumented. + ADD_OP(_NOP, 0, 0); + } + + op(_ASYNC_CALL_GEN_FRAME, (new_frame -- coro, null, gen_frame)) { + coro = sym_new_type(ctx, &PyCoro_Type); + null = sym_new_null(ctx); + gen_frame = new_frame; + } + op(_SEND_GEN_FRAME, (receiver, null, v -- receiver, null, gen_frame)) { _Py_UOpsAbstractFrame *new_frame = frame_new_from_symbol(ctx, receiver, NULL, 0); if (new_frame == NULL) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9cd20c495db441..0d36e5ac09cc8f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -5062,6 +5062,29 @@ break; } + case _GUARD_ASYNC_CALL_TAIL: { + ADD_OP(_NOP, 0, 0); + break; + } + + case _ASYNC_CALL_GEN_FRAME: { + JitOptRef new_frame; + JitOptRef coro; + JitOptRef null; + JitOptRef gen_frame; + new_frame = stack_pointer[-1]; + coro = sym_new_type(ctx, &PyCoro_Type); + null = sym_new_null(ctx); + gen_frame = new_frame; + CHECK_STACK_BOUNDS(2); + stack_pointer[-1] = coro; + stack_pointer[0] = null; + stack_pointer[1] = gen_frame; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 */ case _MAYBE_EXPAND_METHOD_KW: { diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 98abe3d0505e20..fbaa089ffeed04 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -190,6 +190,10 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_METHOD_DESCRIPTOR_NOARGS] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_METHOD_DESCRIPTOR_FAST] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_PY_EXACT_ARGS] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_PY_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_KW_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW_BOUND_METHOD] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW] = {1, {_RECORD_CALLABLE_KW_INDEX}}, @@ -241,6 +245,10 @@ const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = { [CALL_METHOD_DESCRIPTOR_O] = {1, 0, {0}}, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = {1, 0, {0}}, [CALL_METHOD_DESCRIPTOR_NOARGS] = {1, 0, {0}}, + [ASYNC_CALL_PY_EXACT_ARGS] = {1, 0, {0}}, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = {1, 1, {0}}, + [ASYNC_CALL_PY_GENERAL] = {1, 0, {0}}, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = {1, 1, {0}}, [CALL_KW_PY] = {1, 0, {0}}, [CALL_KW_BOUND_METHOD] = {1, 0, {0}}, [CALL_EX_PY] = {1, 0, {0}}, diff --git a/Python/specialize.c b/Python/specialize.c index 067bae59f6073b..2d253893e2e729 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1751,6 +1751,19 @@ specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, return 0; } +/* Is the result of the CALL at "instr" awaited at once? */ +static bool +call_is_awaited(_Py_CODEUNIT *instr) +{ + /* None of the first three instructions can end the code. */ + _Py_CODEUNIT *next = instr + 1 + INLINE_CACHE_ENTRIES_CALL; + return next[0].op.code == GET_AWAITABLE + && next[1].op.code == PUSH_NULL + && next[2].op.code == LOAD_COMMON_CONSTANT + && next[2].op.arg == CONSTANT_NONE + && _PyOpcode_Deopt[next[3].op.code] == SEND; +} + static int specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, bool bound_method) @@ -1781,10 +1794,25 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return -1; } write_u32(cache->func_version, version); + /* ASYNC_CALL_* start the new coroutine in place of the SEND that + * awaits it. GET_AWAITABLE never fails for a new coroutine. */ + int coro_flags = code->co_flags & + (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); + bool awaited = coro_flags == CO_COROUTINE && call_is_awaited(instr); uint8_t opcode; if (argcount == nargs + bound_method) { - opcode = - bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; + if (awaited) { + opcode = bound_method ? + ASYNC_CALL_BOUND_METHOD_EXACT_ARGS : ASYNC_CALL_PY_EXACT_ARGS; + } + else { + opcode = bound_method ? + CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; + } + } + else if (awaited) { + opcode = bound_method ? + ASYNC_CALL_BOUND_METHOD_GENERAL : ASYNC_CALL_PY_GENERAL; } else { opcode = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL; From 9ed5b31f978c75fcb37643cc1dd1079fe363d709 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 7 Nov 2026 23:48:45 +0100 Subject: [PATCH 02/15] Drop redundant comments in ASYNC_CALL_* macros Co-Authored-By: Claude Fable 5.1 --- Python/bytecodes.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ddd4c5c186e65a..61307c380ed318 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5422,7 +5422,7 @@ dummy_func( macro(ASYNC_CALL_PY_EXACT_ARGS) = _RECORD_CALLABLE + - unused/1 + // Skip over the counter + unused/1 + _CHECK_PEP_523 + _GUARD_ASYNC_CALL_TAIL + _CHECK_FUNCTION_VERSION + @@ -5435,7 +5435,7 @@ dummy_func( macro(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS) = _RECORD_BOUND_METHOD + - unused/1 + // Skip over the counter + unused/1 + _CHECK_PEP_523 + _GUARD_ASYNC_CALL_TAIL + _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + @@ -5451,7 +5451,7 @@ dummy_func( macro(ASYNC_CALL_PY_GENERAL) = _RECORD_CALLABLE + - unused/1 + // Skip over the counter + unused/1 + _CHECK_PEP_523 + _GUARD_ASYNC_CALL_TAIL + _CHECK_FUNCTION_VERSION + @@ -5462,7 +5462,7 @@ dummy_func( macro(ASYNC_CALL_BOUND_METHOD_GENERAL) = _RECORD_BOUND_METHOD + - unused/1 + // Skip over the counter + unused/1 + _CHECK_PEP_523 + _GUARD_ASYNC_CALL_TAIL + _CHECK_METHOD_VERSION + From 0e7d877190f7f579f4b6c0686dd04c0c4e6b7e8a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sat, 7 Nov 2026 23:54:03 +0100 Subject: [PATCH 03/15] Remove comments that restate the code Co-Authored-By: Claude Fable 5.1 --- Lib/test/test_coroutines.py | 7 ++----- Lib/test/test_opcache.py | 3 --- Python/bytecodes.c | 1 - Python/specialize.c | 1 - 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 62b690d2d5c84f..b3ab79f9a46722 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2402,9 +2402,6 @@ async def coro(): "asyncio does not work under Emscripten/WASI yet." ) class AwaitCallTest(unittest.TestCase): - # The CALL in `await f(...)` is specialized to ASYNC_CALL_* when f() is - # a coroutine function: the coroutine is then started without being - # returned to the caller first. # Run each call often enough to exercise the specialized forms too. N = 100 @@ -2415,8 +2412,8 @@ async def add(a, b=10): async def caller(): total = 0 for i in range(self.N): - total += await add(i, 1) # Exact arguments - total += await add(i) # Default argument + total += await add(i, 1) + total += await add(i) return total expected = sum(i + 1 + i + 10 for i in range(self.N)) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index b6654957f1c22b..934de968f115fd 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1766,7 +1766,6 @@ async def bound_method_general(): self.assert_no_opcode(bound_method_general, "CALL_BOUND_METHOD_GENERAL") - # __aenter__() and __aexit__() are awaited too. async def async_with(): for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): async with obj as value: @@ -1776,7 +1775,6 @@ async def async_with(): self.assert_specialized(async_with, "ASYNC_CALL_PY_EXACT_ARGS") self.assert_specialized(async_with, "ASYNC_CALL_PY_GENERAL") - # Only calls of coroutine functions are merged with the await. async def not_a_coroutine_function(): for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): self.assertEqual(await function(i), i + 1) @@ -1786,7 +1784,6 @@ async def not_a_coroutine_function(): self.assert_no_opcode(not_a_coroutine_function, "ASYNC_CALL_PY_EXACT_ARGS") - # Only awaited calls are merged. async def not_awaited(): for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): c = coro(i) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 61307c380ed318..21f64bcea25dd9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5393,7 +5393,6 @@ dummy_func( } assert(Py_TYPE(gen) == &PyCoro_Type); if (callee->instr_ptr->op.code == RETURN_GENERATOR) { - // Skip RETURN_GENERATOR and the POP_TOP that follows it. assert(callee->instr_ptr[1].op.code == POP_TOP); callee->instr_ptr += 2; } diff --git a/Python/specialize.c b/Python/specialize.c index 2d253893e2e729..9d3d5053156088 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1751,7 +1751,6 @@ specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, return 0; } -/* Is the result of the CALL at "instr" awaited at once? */ static bool call_is_awaited(_Py_CODEUNIT *instr) { From 885cc1a02749925dd4c7a1bcba339c4b9ab8abec Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 00:03:51 +0100 Subject: [PATCH 04/15] Trace RETURN_GENERATOR of a coroutine started by ASYNC_CALL_* as a None load Ending the trace there cut the trace of every awaited call to a closure or to a coroutine function with cells, since COPY_FREE_VARS and MAKE_CELL precede RETURN_GENERATOR in such frames. Co-Authored-By: Claude Fable 5.1 --- Lib/test/test_capi/test_opt.py | 52 +++++++++++++++++++++------------- Python/optimizer.c | 19 +++++++------ 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 2c8b490b07efcf..320af175affe0f 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -5921,32 +5921,44 @@ def test_async_call_gen_frame(self): async def inner(i): return i + 1 - async def outer(n): - total = 0 - for i in range(n): - total += await inner(i) - return total - def testfunc(n): + def make_closure(v): + async def inner(i): + return i + v + return inner + def make_outer(callee): + ns = {"callee": callee} + exec(textwrap.dedent(""" + async def outer(n): + total = 0 + for i in range(n): + total += await callee(i) + return total + """), ns) + return ns["outer"] + def run(coro): try: - outer(n).send(None) + coro.send(None) except StopIteration as ex: return ex.value n = TIER2_THRESHOLD * 2 - self.assertEqual(testfunc(n), sum(range(1, n + 1))) - ex = get_first_executor(outer) - self.assertIsNotNone(ex) - uops = get_opnames(ex) + for callee in inner, make_closure(1): + with self.subTest(callee=callee): + outer = make_outer(callee) + self.assertEqual(run(outer(n)), sum(range(1, n + 1))) + ex = get_first_executor(outer) + self.assertIsNotNone(ex) + uops = get_opnames(ex) - # The coroutine is started without being returned to outer() first. - self.assertIn("_ASYNC_CALL_GEN_FRAME", uops) - self.assertNotIn("_RETURN_GENERATOR", uops) - self.assertNotIn("_SEND_GEN_FRAME", uops) - # The guard is only needed in tier 1. - self.assertNotIn("_GUARD_ASYNC_CALL_TAIL", uops) - # inner() returns to the END_SEND of outer(). - self.assertIn("_RETURN_VALUE", uops) - self.assertIn("_END_SEND", uops) + # The coroutine is started without being returned to outer() + # first, and runs to its return within the trace. + self.assertIn("_ASYNC_CALL_GEN_FRAME", uops) + self.assertNotIn("_RETURN_GENERATOR", uops) + self.assertNotIn("_SEND_GEN_FRAME", uops) + self.assertIn("_RETURN_VALUE", uops) + self.assertIn("_END_SEND", uops) + # The guard is only needed in tier 1. + self.assertNotIn("_GUARD_ASYNC_CALL_TAIL", uops) def test_send_virtual(self): diff --git a/Python/optimizer.c b/Python/optimizer.c index e3a0bc2208060d..55675b7ef4acab 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -744,7 +744,11 @@ _PyJit_translate_single_bytecode_to_trace( assert(!OPCODE_HAS_DEOPT(opcode)); } - bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode); + // RETURN_GENERATOR in a coroutine that ASYNC_CALL_* already started + // stays in its frame: it only pushes None (see _ASYNC_CALL_GEN_FRAME). + bool coro_running = opcode == RETURN_GENERATOR && + frame == tracer->prev_state.instr_frame; + bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode) && !coro_running; if (has_dynamic_jump_taken && !needs_guard_ip) { DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]); goto unsupported; @@ -783,13 +787,6 @@ _PyJit_translate_single_bytecode_to_trace( goto unsupported; } - // The coroutine was created by ASYNC_CALL_*, so RETURN_GENERATOR - // did not leave its frame. _RETURN_GENERATOR cannot express that. - if (opcode == RETURN_GENERATOR && frame == tracer->prev_state.instr_frame) { - DPRINTF(2, "Unsupported: RETURN_GENERATOR in a running coroutine\n"); - goto unsupported; - } - if (oparg > 0xFFFF) { DPRINTF(2, "Unsupported: oparg too large\n"); unsupported: @@ -915,6 +912,12 @@ _PyJit_translate_single_bytecode_to_trace( * start with RESUME_CHECK */ ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); break; + case RETURN_GENERATOR: + if (coro_running) { + ADD_TO_TRACE(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_None, target); + break; + } + _Py_FALLTHROUGH; default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; From e62a1ab8b90f3c0d88a38507be2f6d66bae863a3 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 20:03:43 +0100 Subject: [PATCH 05/15] Keep the Emscripten/WASI skip on CoroAsyncIOCompatTest The new AwaitCallTest class was inserted between the decorator and the class it applied to. Co-Authored-By: Claude Fable 5.1 --- Lib/test/test_coroutines.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index b3ab79f9a46722..fcddd49bc00249 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2397,10 +2397,6 @@ async def coro(): self.assertEqual(yielded, []) self.assertEqual(result, 42) -@unittest.skipIf( - support.is_emscripten or support.is_wasi, - "asyncio does not work under Emscripten/WASI yet." -) class AwaitCallTest(unittest.TestCase): # Run each call often enough to exercise the specialized forms too. N = 100 @@ -2735,6 +2731,10 @@ def trace(frame, event, arg): self.assertEqual(lines, [1] + [3, 2, 1] * self.N) +@unittest.skipIf( + support.is_emscripten or support.is_wasi, + "asyncio does not work under Emscripten/WASI yet." +) class CoroAsyncIOCompatTest(unittest.TestCase): def test_asyncio_1(self): From 5e36f9b99cb2f4836ca25156b40d69973f18aa97 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 20:04:28 +0100 Subject: [PATCH 06/15] Drop an inlined comprehension from test_async_with In a function scope the async comprehension is inlined, so no call to a comprehension function is awaited there. Co-Authored-By: Claude Fable 5.1 --- Lib/test/test_coroutines.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index fcddd49bc00249..df23df45b73d3b 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2481,7 +2481,7 @@ async def outer(): result = ex.value self.assertEqual(result, [(i + 100) * 2 + 1 for i in range(self.N)]) - def test_async_with_and_comprehension(self): + def test_async_with(self): class CM: async def __aenter__(self): return "entered" @@ -2495,8 +2495,6 @@ async def caller(): async with cm as value: self.assertEqual(value, "entered") 1 / 0 - # The call of the comprehension function is awaited. - self.assertEqual([x async for x in asynciter([i])], [i]) self.assertEqual(run_async(caller()), ([], None)) From 88b811b27f5004553d2f87a3771bb5e0550ee6dc Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 20:07:16 +0100 Subject: [PATCH 07/15] Reject f_lineno jumps into the prologue of a generator frame A trace function could jump to the "def" line of a generator, which re-executed RETURN_GENERATOR in a frame that already belongs to one. Co-Authored-By: Claude Fable 5.1 --- Lib/test/test_coroutines.py | 2 +- Lib/test/test_sys_settrace.py | 21 +++++++++++++++++++++ Objects/frameobject.c | 4 +++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index df23df45b73d3b..669a39a5f00512 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2570,7 +2570,7 @@ def test_wrong_arguments(self): async def func(a, b=1): return a - async def caller(*args): + async def caller(): for i in range(self.N): try: await func() diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a5e7bbbfad5ffc..672f1a333e9a9d 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2994,6 +2994,27 @@ def test_jump_or_pop(output): output.append(1) _ = output.append(2) and "Spam" output.append(3) + def test_no_jump_to_first_line_of_generator(self): + # The "def" line only owns the prologue of a generator frame + # (RETURN_GENERATOR), which must not run again. + def gen(): + output.append(1) + yield 1 + + output = [] + def trace(frame, event, arg): + if event == 'line' and frame.f_code is gen.__code__: + frame.f_lineno = gen.__code__.co_firstlineno + return trace + + sys.settrace(trace) + try: + with self.assertRaisesRegex(ValueError, "cannot find bytecode"): + list(gen()) + finally: + sys.settrace(None) + self.assertEqual(output, []) + class TestExtendedArgs(unittest.TestCase): diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 475be76dc3da21..fdef52383cbd89 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1840,7 +1840,9 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value) int64_t start_stack = stacks[_PyInterpreterFrame_LASTI(self->f_frame)]; int err = -1; const char *msg = "cannot find bytecode for specified line"; - for (int i = 0; i < len; i++) { + /* The instructions before _co_firsttraceable set up a generator + * frame and must not run again once the frame belongs to one. */ + for (int i = code->_co_firsttraceable; i < len; i++) { if (lines[i] == new_lineno) { int64_t target_stack = stacks[i]; if (compatible_stack(start_stack, target_stack)) { From b007c077229626a3e1a782b8868b41a925d38222 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 20:08:06 +0100 Subject: [PATCH 08/15] Assert SEND_GEN on a stored coroutine; fix a comment Co-Authored-By: Claude Fable 5.1 --- Lib/test/test_opcache.py | 1 + Python/bytecodes.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 934de968f115fd..de1931e851dded 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1792,6 +1792,7 @@ async def not_awaited(): run_async(not_awaited()) self.assert_specialized(not_awaited, "CALL_PY_GENERAL") self.assert_no_opcode(not_awaited, "ASYNC_CALL_PY_GENERAL") + self.assert_specialized(not_awaited, "SEND_GEN") @cpython_only @requires_specialization diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 21f64bcea25dd9..6ed526360f190f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5397,8 +5397,8 @@ dummy_func( callee->instr_ptr += 2; } // Otherwise RETURN_GENERATOR is preceded by COPY_FREE_VARS or - // MAKE_CELL. They run in the frame of the coroutine and - // RETURN_GENERATOR does nothing in such a frame. + // MAKE_CELL. They run in the frame of the coroutine, where + // RETURN_GENERATOR only pushes the None that POP_TOP discards. _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_Copy(callee, pushed_frame); assert(callee->frame_obj == NULL); From 1249117d4b6eed430bec9781387689f795583e9e Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 20:34:06 +0100 Subject: [PATCH 09/15] Share the generator frame set-up between RETURN_GENERATOR and the gen frame uops _Py_MakeCoro() takes the frame of the generator function explicitly, gen_take_frame() moves that frame into the generator, and gen_link_frame() links a generator frame to the frame entering it. Co-Authored-By: Claude Fable 5.1 --- Include/internal/pycore_ceval.h | 4 +- Include/internal/pycore_uop_ids.h | 5 +- Include/internal/pycore_uop_metadata.h | 14 +- Modules/_testinternalcapi/test_cases.c.h | 133 ++++------------- Objects/genobject.c | 3 +- Python/bytecodes.c | 42 ++---- Python/ceval_macros.h | 26 ++++ Python/executor_cases.c.h | 176 +++++++++++++++++------ Python/generated_cases.c.h | 133 ++++------------- Tools/cases_generator/analyzer.py | 2 + 10 files changed, 235 insertions(+), 303 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a0fea389b6502c..36fec51ab081d1 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -266,7 +266,9 @@ extern int _PyEval_EnsureBuiltinsWithModule( PyObject *, PyObject **p_builtins); -PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); +// Create the generator object for the frame of a call to a generator +// function; the frame has not started executing yet. +PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func, _PyInterpreterFrame *frame); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index cd0132eb8cd66e..a04629e5dd0ed2 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -1316,6 +1316,9 @@ enum { _SAVE_RETURN_OFFSET_r33, _SEND_ASYNC_GEN_r33, _SEND_ASYNC_GEN_TIER_TWO_r33, + _SEND_GEN_FRAME_r03, + _SEND_GEN_FRAME_r13, + _SEND_GEN_FRAME_r23, _SEND_GEN_FRAME_r33, _SEND_VIRTUAL_r33, _SEND_VIRTUAL_TIER_TWO_r03, @@ -1451,7 +1454,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1662 +#define MAX_UOP_REGS_ID 1665 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index b4fa92a266df4b..53cf9ed65fa47c 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -1647,11 +1647,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_SEND_GEN_FRAME] = { - .best = { 3, 3, 3, 3 }, + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 3, 0, _SEND_GEN_FRAME_r03 }, + { 3, 1, _SEND_GEN_FRAME_r13 }, + { 3, 2, _SEND_GEN_FRAME_r23 }, { 3, 3, _SEND_GEN_FRAME_r33 }, }, }, @@ -4305,6 +4305,9 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GET_AITER_r11] = _GET_AITER, [_GET_ANEXT_r12] = _GET_ANEXT, [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_SEND_GEN_FRAME_r03] = _SEND_GEN_FRAME, + [_SEND_GEN_FRAME_r13] = _SEND_GEN_FRAME, + [_SEND_GEN_FRAME_r23] = _SEND_GEN_FRAME, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, @@ -6043,6 +6046,9 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_SEND_ASYNC_GEN_TIER_TWO] = "_SEND_ASYNC_GEN_TIER_TWO", [_SEND_ASYNC_GEN_TIER_TWO_r33] = "_SEND_ASYNC_GEN_TIER_TWO_r33", [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", + [_SEND_GEN_FRAME_r03] = "_SEND_GEN_FRAME_r03", + [_SEND_GEN_FRAME_r13] = "_SEND_GEN_FRAME_r13", + [_SEND_GEN_FRAME_r23] = "_SEND_GEN_FRAME_r23", [_SEND_GEN_FRAME_r33] = "_SEND_GEN_FRAME_r33", [_SEND_VIRTUAL_TIER_TWO] = "_SEND_VIRTUAL_TIER_TWO", [_SEND_VIRTUAL_TIER_TWO_r03] = "_SEND_VIRTUAL_TIER_TWO_r03", diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 25832238fb2a36..7ea3e13bf3025f 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -161,12 +161,10 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(PyStackRef_FunctionCheck(callee->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - tstate->current_frame = callee; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); _PyFrame_StackPointerInvalidate(frame); - tstate->current_frame = frame; if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -179,31 +177,15 @@ assert(callee->instr_ptr[1].op.code == POP_TOP); callee->instr_ptr += 2; } - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyFrame_Copy(callee, pushed_frame); - _PyFrame_StackPointerInvalidate(frame); - assert(callee->frame_obj == NULL); - pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_PopFrame(tstate, callee); - _PyFrame_StackPointerInvalidate(frame); + gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; frame->instr_ptr += 4u + 3; assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); - frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(pushed_frame); + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); } // _PUSH_FRAME { @@ -361,12 +343,10 @@ _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); assert(PyStackRef_FunctionCheck(callee->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - tstate->current_frame = callee; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); _PyFrame_StackPointerInvalidate(frame); - tstate->current_frame = frame; if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -379,31 +359,15 @@ assert(callee->instr_ptr[1].op.code == POP_TOP); callee->instr_ptr += 2; } - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyFrame_Copy(callee, pushed_frame); - _PyFrame_StackPointerInvalidate(frame); - assert(callee->frame_obj == NULL); - pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_PopFrame(tstate, callee); - _PyFrame_StackPointerInvalidate(frame); + gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; frame->instr_ptr += 4u + 3; assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); - frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(pushed_frame); + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); } // _PUSH_FRAME { @@ -545,12 +509,10 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(PyStackRef_FunctionCheck(callee->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - tstate->current_frame = callee; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); _PyFrame_StackPointerInvalidate(frame); - tstate->current_frame = frame; if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -563,31 +525,15 @@ assert(callee->instr_ptr[1].op.code == POP_TOP); callee->instr_ptr += 2; } - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyFrame_Copy(callee, pushed_frame); - _PyFrame_StackPointerInvalidate(frame); - assert(callee->frame_obj == NULL); - pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_PopFrame(tstate, callee); - _PyFrame_StackPointerInvalidate(frame); + gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; frame->instr_ptr += 4u + 3; assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); - frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(pushed_frame); + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); } // _PUSH_FRAME { @@ -717,12 +663,10 @@ _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); assert(PyStackRef_FunctionCheck(callee->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - tstate->current_frame = callee; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); _PyFrame_StackPointerInvalidate(frame); - tstate->current_frame = frame; if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -735,31 +679,15 @@ assert(callee->instr_ptr[1].op.code == POP_TOP); callee->instr_ptr += 2; } - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyFrame_Copy(callee, pushed_frame); - _PyFrame_StackPointerInvalidate(frame); - assert(callee->frame_obj == NULL); - pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_PopFrame(tstate, callee); - _PyFrame_StackPointerInvalidate(frame); + gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; frame->instr_ptr += 4u + 3; assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); - frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(pushed_frame); + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); } // _PUSH_FRAME { @@ -7201,10 +7129,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME @@ -12218,7 +12143,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, frame); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { JUMP_TO_LABEL(error); @@ -12226,16 +12151,11 @@ assert(STACK_LEVEL() <= 2); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -12496,11 +12416,8 @@ STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME diff --git a/Objects/genobject.c b/Objects/genobject.c index 82a1bb1ff42068..c07adaf6bf93d9 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1128,7 +1128,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) } PyObject * -_Py_MakeCoro(PyFunctionObject *func) +_Py_MakeCoro(PyFunctionObject *func, _PyInterpreterFrame *frame) { int coro_flags = ((PyCodeObject *)func->func_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); @@ -1160,7 +1160,6 @@ _Py_MakeCoro(PyFunctionObject *func) if (origin_depth == 0) { ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; } else { - _PyInterpreterFrame *frame = tstate->current_frame; assert(frame); assert(_PyFrame_IsIncomplete(frame)); frame = _PyFrame_GetFirstComplete(frame->previous); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6ed526360f190f..465723c4d65d8a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1730,11 +1730,8 @@ dummy_func( _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); DEAD(v); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)(INSTRUCTION_SIZE + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } @@ -4078,11 +4075,8 @@ dummy_func( STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; // oparg is the return offset from the next instruction. - frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)(INSTRUCTION_SIZE + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } @@ -5382,11 +5376,7 @@ dummy_func( SYNC_SP(); assert(PyStackRef_FunctionCheck(callee->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - // _Py_MakeCoro() expects the current frame to be the frame - // of the coroutine function, as it is in RETURN_GENERATOR. - tstate->current_frame = callee; - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - tstate->current_frame = frame; + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); if (gen == NULL) { _PyEval_FrameClearAndPop(tstate, callee); ERROR_NO_POP(); @@ -5399,24 +5389,17 @@ dummy_func( // Otherwise RETURN_GENERATOR is preceded by COPY_FREE_VARS or // MAKE_CELL. They run in the frame of the coroutine, where // RETURN_GENERATOR only pushes the None that POP_TOP discards. - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_Copy(callee, pushed_frame); - assert(callee->frame_obj == NULL); - pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; - _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); - _PyThreadState_PopFrame(tstate, callee); + gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; // The coroutine yields and returns to the SEND, which follows // GET_AWAITABLE, PUSH_NULL and LOAD_COMMON_CONSTANT. frame->instr_ptr += INSTRUCTION_SIZE + 3; assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); - frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(pushed_frame); + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); } macro(ASYNC_CALL_PY_EXACT_ARGS) = @@ -5940,20 +5923,15 @@ dummy_func( else { assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, frame); ERROR_IF(gen == NULL); assert(STACK_LEVEL() <= 2); SAVE_STACK(); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); RELOAD_STACK(); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 36921e35533433..f36a53ad614ea1 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -594,6 +594,32 @@ gen_try_set_executing(PyGenObject *gen) return false; } +// Move the frame of a call to a generator function into the generator +// and pop it from the thread's data stack. `prev` is the frame that +// takes its place. +static inline void +gen_take_frame(PyThreadState *tstate, PyGenObject *gen, + _PyInterpreterFrame *frame, _PyInterpreterFrame *prev) +{ + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); +} + +// Link the frame of a generator to the frame that is about to enter it. +static inline void +gen_link_frame(PyThreadState *tstate, PyGenObject *gen, + _PyInterpreterFrame *frame, uint16_t return_offset) +{ + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + gen->gi_iframe.previous = frame; + frame->return_offset = return_offset; +} + // Macro for inplace float binary ops (tier 2 only). // Mutates the uniquely-referenced TARGET operand in place. // TARGET must be either left or right. diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index cfed39174dd924..493a61b398501f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8940,6 +8940,123 @@ break; } + case _SEND_GEN_FRAME_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + _PyStackRef receiver; + _PyStackRef gen_frame; + oparg = CURRENT_OPARG(); + v = stack_pointer[-1]; + receiver = stack_pointer[-3]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + assert( 2u + oparg <= UINT16_MAX); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _SEND_GEN_FRAME_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + _PyStackRef receiver; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = v; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = v; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + assert( 2u + oparg <= UINT16_MAX); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _SEND_GEN_FRAME_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + _PyStackRef receiver; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + v = _stack_item_1; + receiver = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = v; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = v; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + assert( 2u + oparg <= UINT16_MAX); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = _stack_item_0; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _SEND_GEN_FRAME_r33: { CHECK_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -8972,11 +9089,8 @@ STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = _stack_item_1; @@ -16203,10 +16317,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = stack_pointer[-1]; @@ -16242,10 +16353,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = _stack_item_0; @@ -16284,10 +16392,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = _stack_item_1; @@ -20042,12 +20147,10 @@ _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); assert(PyStackRef_FunctionCheck(callee->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - tstate->current_frame = callee; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); _PyFrame_StackPointerInvalidate(frame); - tstate->current_frame = frame; if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -20061,31 +20164,15 @@ assert(callee->instr_ptr[1].op.code == POP_TOP); callee->instr_ptr += 2; } - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyFrame_Copy(callee, pushed_frame); - _PyFrame_StackPointerInvalidate(frame); - assert(callee->frame_obj == NULL); - pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_PopFrame(tstate, callee); - _PyFrame_StackPointerInvalidate(frame); + gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; frame->instr_ptr += 4u + 3; assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); - frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(pushed_frame); + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); _tos_cache2 = gen_frame; _tos_cache1 = null; _tos_cache0 = coro; @@ -20902,7 +20989,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, frame); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { SET_CURRENT_CACHED_VALUES(0); @@ -20911,16 +20998,11 @@ assert(STACK_LEVEL() <= 2); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6c43314f26c510..d598655227021d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -161,12 +161,10 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(PyStackRef_FunctionCheck(callee->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - tstate->current_frame = callee; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); _PyFrame_StackPointerInvalidate(frame); - tstate->current_frame = frame; if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -179,31 +177,15 @@ assert(callee->instr_ptr[1].op.code == POP_TOP); callee->instr_ptr += 2; } - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyFrame_Copy(callee, pushed_frame); - _PyFrame_StackPointerInvalidate(frame); - assert(callee->frame_obj == NULL); - pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_PopFrame(tstate, callee); - _PyFrame_StackPointerInvalidate(frame); + gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; frame->instr_ptr += 4u + 3; assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); - frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(pushed_frame); + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); } // _PUSH_FRAME { @@ -361,12 +343,10 @@ _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); assert(PyStackRef_FunctionCheck(callee->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - tstate->current_frame = callee; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); _PyFrame_StackPointerInvalidate(frame); - tstate->current_frame = frame; if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -379,31 +359,15 @@ assert(callee->instr_ptr[1].op.code == POP_TOP); callee->instr_ptr += 2; } - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyFrame_Copy(callee, pushed_frame); - _PyFrame_StackPointerInvalidate(frame); - assert(callee->frame_obj == NULL); - pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_PopFrame(tstate, callee); - _PyFrame_StackPointerInvalidate(frame); + gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; frame->instr_ptr += 4u + 3; assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); - frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(pushed_frame); + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); } // _PUSH_FRAME { @@ -545,12 +509,10 @@ ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(PyStackRef_FunctionCheck(callee->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - tstate->current_frame = callee; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); _PyFrame_StackPointerInvalidate(frame); - tstate->current_frame = frame; if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -563,31 +525,15 @@ assert(callee->instr_ptr[1].op.code == POP_TOP); callee->instr_ptr += 2; } - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyFrame_Copy(callee, pushed_frame); - _PyFrame_StackPointerInvalidate(frame); - assert(callee->frame_obj == NULL); - pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_PopFrame(tstate, callee); - _PyFrame_StackPointerInvalidate(frame); + gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; frame->instr_ptr += 4u + 3; assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); - frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(pushed_frame); + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); } // _PUSH_FRAME { @@ -717,12 +663,10 @@ _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); assert(PyStackRef_FunctionCheck(callee->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - tstate->current_frame = callee; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); _PyFrame_StackPointerInvalidate(frame); - tstate->current_frame = frame; if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -735,31 +679,15 @@ assert(callee->instr_ptr[1].op.code == POP_TOP); callee->instr_ptr += 2; } - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyFrame_Copy(callee, pushed_frame); - _PyFrame_StackPointerInvalidate(frame); - assert(callee->frame_obj == NULL); - pushed_frame->owner = FRAME_OWNED_BY_GENERATOR; - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_UpdateLastProfiledFrame(tstate, callee, frame); - _PyFrame_StackPointerInvalidate(frame); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyThreadState_PopFrame(tstate, callee); - _PyFrame_StackPointerInvalidate(frame); + gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; frame->instr_ptr += 4u + 3; assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); - frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; - gen_frame = PyStackRef_Wrap(pushed_frame); + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); } // _PUSH_FRAME { @@ -7201,10 +7129,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME @@ -12215,7 +12140,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, frame); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { JUMP_TO_LABEL(error); @@ -12223,16 +12148,11 @@ assert(STACK_LEVEL() <= 2); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -12493,11 +12413,8 @@ STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index c3e0b0dbff2199..3b4c3664ae7fc4 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -603,6 +603,8 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyFunction_SetVersion", "_PyGen_GetGeneratorFromFrame", "gen_try_set_executing", + "gen_link_frame", + "gen_take_frame", "_PyInterpreterState_GET", "_PyList_AppendTakeRef", "_PyList_ITEMS", From 8d25e78a0ada88c1a76f87a2b9e59fe355699846 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 20:38:41 +0100 Subject: [PATCH 10/15] Add ASYNC_CALL_KW_PY and ASYNC_CALL_KW_BOUND_METHOD Awaited calls with keyword arguments compile to CALL_KW, which gets the same two awaited specializations as CALL for Python functions and bound methods. Co-Authored-By: Claude Fable 5.1 --- Include/internal/pycore_opcode_metadata.h | 20 +- Include/opcode_ids.h | 186 ++++++------ InternalDocs/generators.md | 6 +- Lib/_opcode_metadata.py | 188 ++++++------ Lib/test/test_coroutines.py | 22 ++ Lib/test/test_opcache.py | 18 ++ Modules/_testinternalcapi/test_cases.c.h | 350 ++++++++++++++++++++++ Modules/_testinternalcapi/test_targets.h | 18 +- Python/bytecodes.c | 28 +- Python/generated_cases.c.h | 350 ++++++++++++++++++++++ Python/opcode_targets.h | 18 +- Python/record_functions.c.h | 4 + Python/specialize.c | 28 +- 13 files changed, 1020 insertions(+), 216 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 4c38aa7891df2a..dbd238ed8df491 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -42,6 +42,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case ASYNC_CALL_BOUND_METHOD_GENERAL: return 2 + oparg; + case ASYNC_CALL_KW_BOUND_METHOD: + return 3 + oparg; + case ASYNC_CALL_KW_PY: + return 3 + oparg; case ASYNC_CALL_PY_EXACT_ARGS: return 2 + oparg; case ASYNC_CALL_PY_GENERAL: @@ -545,6 +549,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2; case ASYNC_CALL_BOUND_METHOD_GENERAL: return 2; + case ASYNC_CALL_KW_BOUND_METHOD: + return 2; + case ASYNC_CALL_KW_PY: + return 2; case ASYNC_CALL_PY_EXACT_ARGS: return 2; case ASYNC_CALL_PY_GENERAL: @@ -1118,6 +1126,8 @@ PyAPI_DATA(const struct opcode_metadata) _PyOpcode_opcode_metadata[267]; const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [ASYNC_CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [ASYNC_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [ASYNC_CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, @@ -1377,6 +1387,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 12, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [ASYNC_CALL_BOUND_METHOD_GENERAL] = { .nuops = 9, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_KW_BOUND_METHOD] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_KW_PY] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [ASYNC_CALL_PY_EXACT_ARGS] = { .nuops = 10, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [ASYNC_CALL_PY_GENERAL] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [BINARY_OP] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _BINARY_OP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, @@ -1579,6 +1591,8 @@ const char *_PyOpcode_OpName[267] = { [ANNOTATIONS_PLACEHOLDER] = "ANNOTATIONS_PLACEHOLDER", [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = "ASYNC_CALL_BOUND_METHOD_EXACT_ARGS", [ASYNC_CALL_BOUND_METHOD_GENERAL] = "ASYNC_CALL_BOUND_METHOD_GENERAL", + [ASYNC_CALL_KW_BOUND_METHOD] = "ASYNC_CALL_KW_BOUND_METHOD", + [ASYNC_CALL_KW_PY] = "ASYNC_CALL_KW_PY", [ASYNC_CALL_PY_EXACT_ARGS] = "ASYNC_CALL_PY_EXACT_ARGS", [ASYNC_CALL_PY_GENERAL] = "ASYNC_CALL_PY_GENERAL", [BINARY_OP] = "BINARY_OP", @@ -1867,8 +1881,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [223] = 223, - [224] = 224, [225] = 225, [226] = 226, [227] = 227, @@ -1879,6 +1891,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [232] = 232, [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = CALL, [ASYNC_CALL_BOUND_METHOD_GENERAL] = CALL, + [ASYNC_CALL_KW_BOUND_METHOD] = CALL_KW, + [ASYNC_CALL_KW_PY] = CALL_KW, [ASYNC_CALL_PY_EXACT_ARGS] = CALL, [ASYNC_CALL_PY_GENERAL] = CALL, [BINARY_OP] = BINARY_OP, @@ -2128,8 +2142,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 223: \ - case 224: \ case 225: \ case 226: \ case 227: \ diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 7a9179dd97271c..0ce90edf933442 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -130,98 +130,100 @@ extern "C" { #define RESUME 128 #define ASYNC_CALL_BOUND_METHOD_EXACT_ARGS 129 #define ASYNC_CALL_BOUND_METHOD_GENERAL 130 -#define ASYNC_CALL_PY_EXACT_ARGS 131 -#define ASYNC_CALL_PY_GENERAL 132 -#define BINARY_OP_ADD_FLOAT 133 -#define BINARY_OP_ADD_INT 134 -#define BINARY_OP_ADD_UNICODE 135 -#define BINARY_OP_EXTEND 136 -#define BINARY_OP_MULTIPLY_FLOAT 137 -#define BINARY_OP_MULTIPLY_INT 138 -#define BINARY_OP_SUBSCR_DICT 139 -#define BINARY_OP_SUBSCR_GETITEM 140 -#define BINARY_OP_SUBSCR_LIST_INT 141 -#define BINARY_OP_SUBSCR_LIST_SLICE 142 -#define BINARY_OP_SUBSCR_STR_INT 143 -#define BINARY_OP_SUBSCR_TUPLE_INT 144 -#define BINARY_OP_SUBSCR_USTR_INT 145 -#define BINARY_OP_SUBTRACT_FLOAT 146 -#define BINARY_OP_SUBTRACT_INT 147 -#define CALL_ALLOC_AND_ENTER_INIT 148 -#define CALL_BOUND_METHOD_EXACT_ARGS 149 -#define CALL_BOUND_METHOD_GENERAL 150 -#define CALL_BUILTIN_CLASS 151 -#define CALL_BUILTIN_FAST 152 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 153 -#define CALL_BUILTIN_O 154 -#define CALL_EX_NON_PY_GENERAL 155 -#define CALL_EX_PY 156 -#define CALL_ISINSTANCE 157 -#define CALL_KW_BOUND_METHOD 158 -#define CALL_KW_NON_PY 159 -#define CALL_KW_PY 160 -#define CALL_LEN 161 -#define CALL_LIST_APPEND 162 -#define CALL_METHOD_DESCRIPTOR_FAST 163 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 164 -#define CALL_METHOD_DESCRIPTOR_NOARGS 165 -#define CALL_METHOD_DESCRIPTOR_O 166 -#define CALL_NON_PY_GENERAL 167 -#define CALL_PY_EXACT_ARGS 168 -#define CALL_PY_GENERAL 169 -#define CALL_STR_1 170 -#define CALL_TUPLE_1 171 -#define CALL_TYPE_1 172 -#define COMPARE_OP_FLOAT 173 -#define COMPARE_OP_INT 174 -#define COMPARE_OP_STR 175 -#define CONTAINS_OP_DICT 176 -#define CONTAINS_OP_SET 177 -#define FOR_ITER_GEN 178 -#define FOR_ITER_LIST 179 -#define FOR_ITER_RANGE 180 -#define FOR_ITER_TUPLE 181 -#define FOR_ITER_VIRTUAL 182 -#define GET_ITER_SELF 183 -#define GET_ITER_VIRTUAL 184 -#define JUMP_BACKWARD_JIT 185 -#define JUMP_BACKWARD_NO_JIT 186 -#define LOAD_ATTR_CLASS 187 -#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 188 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 189 -#define LOAD_ATTR_INSTANCE_VALUE 190 -#define LOAD_ATTR_METHOD_LAZY_DICT 191 -#define LOAD_ATTR_METHOD_NO_DICT 192 -#define LOAD_ATTR_METHOD_WITH_VALUES 193 -#define LOAD_ATTR_MODULE 194 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 195 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 196 -#define LOAD_ATTR_PROPERTY 197 -#define LOAD_ATTR_SLOT 198 -#define LOAD_ATTR_WITH_HINT 199 -#define LOAD_GLOBAL_BUILTIN 200 -#define LOAD_GLOBAL_MODULE 201 -#define LOAD_SUPER_ATTR_ATTR 202 -#define LOAD_SUPER_ATTR_METHOD 203 -#define RESUME_CHECK 204 -#define RESUME_CHECK_JIT 205 -#define SEND_ASYNC_GEN 206 -#define SEND_GEN 207 -#define SEND_VIRTUAL 208 -#define STORE_ATTR_INSTANCE_VALUE 209 -#define STORE_ATTR_SLOT 210 -#define STORE_ATTR_WITH_HINT 211 -#define STORE_SUBSCR_DICT 212 -#define STORE_SUBSCR_LIST_INT 213 -#define TO_BOOL_ALWAYS_TRUE 214 -#define TO_BOOL_BOOL 215 -#define TO_BOOL_INT 216 -#define TO_BOOL_LIST 217 -#define TO_BOOL_NONE 218 -#define TO_BOOL_STR 219 -#define UNPACK_SEQUENCE_LIST 220 -#define UNPACK_SEQUENCE_TUPLE 221 -#define UNPACK_SEQUENCE_TWO_TUPLE 222 +#define ASYNC_CALL_KW_BOUND_METHOD 131 +#define ASYNC_CALL_KW_PY 132 +#define ASYNC_CALL_PY_EXACT_ARGS 133 +#define ASYNC_CALL_PY_GENERAL 134 +#define BINARY_OP_ADD_FLOAT 135 +#define BINARY_OP_ADD_INT 136 +#define BINARY_OP_ADD_UNICODE 137 +#define BINARY_OP_EXTEND 138 +#define BINARY_OP_MULTIPLY_FLOAT 139 +#define BINARY_OP_MULTIPLY_INT 140 +#define BINARY_OP_SUBSCR_DICT 141 +#define BINARY_OP_SUBSCR_GETITEM 142 +#define BINARY_OP_SUBSCR_LIST_INT 143 +#define BINARY_OP_SUBSCR_LIST_SLICE 144 +#define BINARY_OP_SUBSCR_STR_INT 145 +#define BINARY_OP_SUBSCR_TUPLE_INT 146 +#define BINARY_OP_SUBSCR_USTR_INT 147 +#define BINARY_OP_SUBTRACT_FLOAT 148 +#define BINARY_OP_SUBTRACT_INT 149 +#define CALL_ALLOC_AND_ENTER_INIT 150 +#define CALL_BOUND_METHOD_EXACT_ARGS 151 +#define CALL_BOUND_METHOD_GENERAL 152 +#define CALL_BUILTIN_CLASS 153 +#define CALL_BUILTIN_FAST 154 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 155 +#define CALL_BUILTIN_O 156 +#define CALL_EX_NON_PY_GENERAL 157 +#define CALL_EX_PY 158 +#define CALL_ISINSTANCE 159 +#define CALL_KW_BOUND_METHOD 160 +#define CALL_KW_NON_PY 161 +#define CALL_KW_PY 162 +#define CALL_LEN 163 +#define CALL_LIST_APPEND 164 +#define CALL_METHOD_DESCRIPTOR_FAST 165 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 166 +#define CALL_METHOD_DESCRIPTOR_NOARGS 167 +#define CALL_METHOD_DESCRIPTOR_O 168 +#define CALL_NON_PY_GENERAL 169 +#define CALL_PY_EXACT_ARGS 170 +#define CALL_PY_GENERAL 171 +#define CALL_STR_1 172 +#define CALL_TUPLE_1 173 +#define CALL_TYPE_1 174 +#define COMPARE_OP_FLOAT 175 +#define COMPARE_OP_INT 176 +#define COMPARE_OP_STR 177 +#define CONTAINS_OP_DICT 178 +#define CONTAINS_OP_SET 179 +#define FOR_ITER_GEN 180 +#define FOR_ITER_LIST 181 +#define FOR_ITER_RANGE 182 +#define FOR_ITER_TUPLE 183 +#define FOR_ITER_VIRTUAL 184 +#define GET_ITER_SELF 185 +#define GET_ITER_VIRTUAL 186 +#define JUMP_BACKWARD_JIT 187 +#define JUMP_BACKWARD_NO_JIT 188 +#define LOAD_ATTR_CLASS 189 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 190 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 191 +#define LOAD_ATTR_INSTANCE_VALUE 192 +#define LOAD_ATTR_METHOD_LAZY_DICT 193 +#define LOAD_ATTR_METHOD_NO_DICT 194 +#define LOAD_ATTR_METHOD_WITH_VALUES 195 +#define LOAD_ATTR_MODULE 196 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 197 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 198 +#define LOAD_ATTR_PROPERTY 199 +#define LOAD_ATTR_SLOT 200 +#define LOAD_ATTR_WITH_HINT 201 +#define LOAD_GLOBAL_BUILTIN 202 +#define LOAD_GLOBAL_MODULE 203 +#define LOAD_SUPER_ATTR_ATTR 204 +#define LOAD_SUPER_ATTR_METHOD 205 +#define RESUME_CHECK 206 +#define RESUME_CHECK_JIT 207 +#define SEND_ASYNC_GEN 208 +#define SEND_GEN 209 +#define SEND_VIRTUAL 210 +#define STORE_ATTR_INSTANCE_VALUE 211 +#define STORE_ATTR_SLOT 212 +#define STORE_ATTR_WITH_HINT 213 +#define STORE_SUBSCR_DICT 214 +#define STORE_SUBSCR_LIST_INT 215 +#define TO_BOOL_ALWAYS_TRUE 216 +#define TO_BOOL_BOOL 217 +#define TO_BOOL_INT 218 +#define TO_BOOL_LIST 219 +#define TO_BOOL_NONE 220 +#define TO_BOOL_STR 221 +#define UNPACK_SEQUENCE_LIST 222 +#define UNPACK_SEQUENCE_TUPLE 223 +#define UNPACK_SEQUENCE_TWO_TUPLE 224 #define INSTRUMENTED_END_FOR 233 #define INSTRUMENTED_POP_ITER 234 #define INSTRUMENTED_END_SEND 235 diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index 18e7dd0a920c39..c1a7fa9d9e46c4 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -108,14 +108,14 @@ use the value of the argument to `send`. An `await` expression is compiled like `yield from`, with [`GET_AWAITABLE`](https://docs.python.org/dev/library/dis.html#opcode-GET_AWAITABLE) in place of `GET_ITER`. When the awaited object is the result of a call to -a coroutine function, as in `await f()`, the `CALL` is +a coroutine function, as in `await f()`, the `CALL` (or `CALL_KW`) is [`specialized`](interpreter.md) to one of the `ASYNC_CALL_*` instructions, such as `ASYNC_CALL_PY_EXACT_ARGS`. These are superinstructions: they merge -the `CALL` with the first execution of the `GET_AWAITABLE`, `PUSH_NULL`, +the call with the first execution of the `GET_AWAITABLE`, `PUSH_NULL`, `LOAD_COMMON_CONSTANT` and `SEND` instructions that follow it. `ASYNC_CALL_*` create the frame with the micro-ops of the matching `CALL_*` -specialization. They then turn it into the frame of a new coroutine and push +or `CALL_KW_*` specialization. They then turn it into the frame of a new coroutine and push it at once, as `RETURN_GENERATOR` and `SEND_GEN` would do. The coroutine is not returned to the caller first and the frame of the caller is left as if it were executing the `SEND`. diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 1e99a825a366f5..c05245bdfa530c 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -129,6 +129,8 @@ "CALL_KW_BOUND_METHOD", "CALL_KW_PY", "CALL_KW_NON_PY", + "ASYNC_CALL_KW_BOUND_METHOD", + "ASYNC_CALL_KW_PY", ), CALL_FUNCTION_EX=( "CALL_EX_PY", @@ -139,99 +141,101 @@ _specialized_opmap = frozendict( ASYNC_CALL_BOUND_METHOD_EXACT_ARGS=129, ASYNC_CALL_BOUND_METHOD_GENERAL=130, - ASYNC_CALL_PY_EXACT_ARGS=131, - ASYNC_CALL_PY_GENERAL=132, - BINARY_OP_ADD_FLOAT=133, - BINARY_OP_ADD_INT=134, - BINARY_OP_ADD_UNICODE=135, - BINARY_OP_EXTEND=136, + ASYNC_CALL_KW_BOUND_METHOD=131, + ASYNC_CALL_KW_PY=132, + ASYNC_CALL_PY_EXACT_ARGS=133, + ASYNC_CALL_PY_GENERAL=134, + BINARY_OP_ADD_FLOAT=135, + BINARY_OP_ADD_INT=136, + BINARY_OP_ADD_UNICODE=137, + BINARY_OP_EXTEND=138, BINARY_OP_INPLACE_ADD_UNICODE=3, - BINARY_OP_MULTIPLY_FLOAT=137, - BINARY_OP_MULTIPLY_INT=138, - BINARY_OP_SUBSCR_DICT=139, - BINARY_OP_SUBSCR_GETITEM=140, - BINARY_OP_SUBSCR_LIST_INT=141, - BINARY_OP_SUBSCR_LIST_SLICE=142, - BINARY_OP_SUBSCR_STR_INT=143, - BINARY_OP_SUBSCR_TUPLE_INT=144, - BINARY_OP_SUBSCR_USTR_INT=145, - BINARY_OP_SUBTRACT_FLOAT=146, - BINARY_OP_SUBTRACT_INT=147, - CALL_ALLOC_AND_ENTER_INIT=148, - CALL_BOUND_METHOD_EXACT_ARGS=149, - CALL_BOUND_METHOD_GENERAL=150, - CALL_BUILTIN_CLASS=151, - CALL_BUILTIN_FAST=152, - CALL_BUILTIN_FAST_WITH_KEYWORDS=153, - CALL_BUILTIN_O=154, - CALL_EX_NON_PY_GENERAL=155, - CALL_EX_PY=156, - CALL_ISINSTANCE=157, - CALL_KW_BOUND_METHOD=158, - CALL_KW_NON_PY=159, - CALL_KW_PY=160, - CALL_LEN=161, - CALL_LIST_APPEND=162, - CALL_METHOD_DESCRIPTOR_FAST=163, - CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=164, - CALL_METHOD_DESCRIPTOR_NOARGS=165, - CALL_METHOD_DESCRIPTOR_O=166, - CALL_NON_PY_GENERAL=167, - CALL_PY_EXACT_ARGS=168, - CALL_PY_GENERAL=169, - CALL_STR_1=170, - CALL_TUPLE_1=171, - CALL_TYPE_1=172, - COMPARE_OP_FLOAT=173, - COMPARE_OP_INT=174, - COMPARE_OP_STR=175, - CONTAINS_OP_DICT=176, - CONTAINS_OP_SET=177, - FOR_ITER_GEN=178, - FOR_ITER_LIST=179, - FOR_ITER_RANGE=180, - FOR_ITER_TUPLE=181, - FOR_ITER_VIRTUAL=182, - GET_ITER_SELF=183, - GET_ITER_VIRTUAL=184, - JUMP_BACKWARD_JIT=185, - JUMP_BACKWARD_NO_JIT=186, - LOAD_ATTR_CLASS=187, - LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=188, - LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=189, - LOAD_ATTR_INSTANCE_VALUE=190, - LOAD_ATTR_METHOD_LAZY_DICT=191, - LOAD_ATTR_METHOD_NO_DICT=192, - LOAD_ATTR_METHOD_WITH_VALUES=193, - LOAD_ATTR_MODULE=194, - LOAD_ATTR_NONDESCRIPTOR_NO_DICT=195, - LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=196, - LOAD_ATTR_PROPERTY=197, - LOAD_ATTR_SLOT=198, - LOAD_ATTR_WITH_HINT=199, - LOAD_GLOBAL_BUILTIN=200, - LOAD_GLOBAL_MODULE=201, - LOAD_SUPER_ATTR_ATTR=202, - LOAD_SUPER_ATTR_METHOD=203, - RESUME_CHECK=204, - RESUME_CHECK_JIT=205, - SEND_ASYNC_GEN=206, - SEND_GEN=207, - SEND_VIRTUAL=208, - STORE_ATTR_INSTANCE_VALUE=209, - STORE_ATTR_SLOT=210, - STORE_ATTR_WITH_HINT=211, - STORE_SUBSCR_DICT=212, - STORE_SUBSCR_LIST_INT=213, - TO_BOOL_ALWAYS_TRUE=214, - TO_BOOL_BOOL=215, - TO_BOOL_INT=216, - TO_BOOL_LIST=217, - TO_BOOL_NONE=218, - TO_BOOL_STR=219, - UNPACK_SEQUENCE_LIST=220, - UNPACK_SEQUENCE_TUPLE=221, - UNPACK_SEQUENCE_TWO_TUPLE=222, + BINARY_OP_MULTIPLY_FLOAT=139, + BINARY_OP_MULTIPLY_INT=140, + BINARY_OP_SUBSCR_DICT=141, + BINARY_OP_SUBSCR_GETITEM=142, + BINARY_OP_SUBSCR_LIST_INT=143, + BINARY_OP_SUBSCR_LIST_SLICE=144, + BINARY_OP_SUBSCR_STR_INT=145, + BINARY_OP_SUBSCR_TUPLE_INT=146, + BINARY_OP_SUBSCR_USTR_INT=147, + BINARY_OP_SUBTRACT_FLOAT=148, + BINARY_OP_SUBTRACT_INT=149, + CALL_ALLOC_AND_ENTER_INIT=150, + CALL_BOUND_METHOD_EXACT_ARGS=151, + CALL_BOUND_METHOD_GENERAL=152, + CALL_BUILTIN_CLASS=153, + CALL_BUILTIN_FAST=154, + CALL_BUILTIN_FAST_WITH_KEYWORDS=155, + CALL_BUILTIN_O=156, + CALL_EX_NON_PY_GENERAL=157, + CALL_EX_PY=158, + CALL_ISINSTANCE=159, + CALL_KW_BOUND_METHOD=160, + CALL_KW_NON_PY=161, + CALL_KW_PY=162, + CALL_LEN=163, + CALL_LIST_APPEND=164, + CALL_METHOD_DESCRIPTOR_FAST=165, + CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=166, + CALL_METHOD_DESCRIPTOR_NOARGS=167, + CALL_METHOD_DESCRIPTOR_O=168, + CALL_NON_PY_GENERAL=169, + CALL_PY_EXACT_ARGS=170, + CALL_PY_GENERAL=171, + CALL_STR_1=172, + CALL_TUPLE_1=173, + CALL_TYPE_1=174, + COMPARE_OP_FLOAT=175, + COMPARE_OP_INT=176, + COMPARE_OP_STR=177, + CONTAINS_OP_DICT=178, + CONTAINS_OP_SET=179, + FOR_ITER_GEN=180, + FOR_ITER_LIST=181, + FOR_ITER_RANGE=182, + FOR_ITER_TUPLE=183, + FOR_ITER_VIRTUAL=184, + GET_ITER_SELF=185, + GET_ITER_VIRTUAL=186, + JUMP_BACKWARD_JIT=187, + JUMP_BACKWARD_NO_JIT=188, + LOAD_ATTR_CLASS=189, + LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=190, + LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=191, + LOAD_ATTR_INSTANCE_VALUE=192, + LOAD_ATTR_METHOD_LAZY_DICT=193, + LOAD_ATTR_METHOD_NO_DICT=194, + LOAD_ATTR_METHOD_WITH_VALUES=195, + LOAD_ATTR_MODULE=196, + LOAD_ATTR_NONDESCRIPTOR_NO_DICT=197, + LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=198, + LOAD_ATTR_PROPERTY=199, + LOAD_ATTR_SLOT=200, + LOAD_ATTR_WITH_HINT=201, + LOAD_GLOBAL_BUILTIN=202, + LOAD_GLOBAL_MODULE=203, + LOAD_SUPER_ATTR_ATTR=204, + LOAD_SUPER_ATTR_METHOD=205, + RESUME_CHECK=206, + RESUME_CHECK_JIT=207, + SEND_ASYNC_GEN=208, + SEND_GEN=209, + SEND_VIRTUAL=210, + STORE_ATTR_INSTANCE_VALUE=211, + STORE_ATTR_SLOT=212, + STORE_ATTR_WITH_HINT=213, + STORE_SUBSCR_DICT=214, + STORE_SUBSCR_LIST_INT=215, + TO_BOOL_ALWAYS_TRUE=216, + TO_BOOL_BOOL=217, + TO_BOOL_INT=218, + TO_BOOL_LIST=219, + TO_BOOL_NONE=220, + TO_BOOL_STR=221, + UNPACK_SEQUENCE_LIST=222, + UNPACK_SEQUENCE_TUPLE=223, + UNPACK_SEQUENCE_TWO_TUPLE=224, ) opmap = frozendict( diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 669a39a5f00512..2e740066060bf9 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2434,6 +2434,28 @@ async def caller(): expected = sum(2 * (i + 1 + i + 10) for i in range(self.N)) self.assertEqual(run_async(caller()), ([], expected)) + def test_keyword_arguments(self): + async def add(a, b=10): + return a + b + + class A: + async def add(self, a, b=10): + return a + b + + async def caller(): + obj = A() + bound = obj.add + total = 0 + for i in range(self.N): + total += await add(i, b=1) + total += await add(a=i) + total += await obj.add(i, b=1) + total += await bound(b=1, a=i) + return total + + expected = sum(i + 1 + i + 10 + 2 * (i + 1) for i in range(self.N)) + self.assertEqual(run_async(caller()), ([], expected)) + def test_closure(self): # COPY_FREE_VARS and MAKE_CELL precede RETURN_GENERATOR def outer(free): diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index de1931e851dded..99558b72c5bb73 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1763,6 +1763,24 @@ async def bound_method_general(): run_async(bound_method_general()) self.assert_specialized(bound_method_general, "ASYNC_CALL_BOUND_METHOD_GENERAL") + + async def kw_py(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await coro(i, b=2), i + 2) + self.assertEqual(await obj.method(i, b=2), i + 2) + + run_async(kw_py()) + self.assert_specialized(kw_py, "ASYNC_CALL_KW_PY") + self.assert_no_opcode(kw_py, "CALL_KW_PY") + + async def kw_bound_method(): + bound = obj.method + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await bound(i, b=2), i + 2) + + run_async(kw_bound_method()) + self.assert_specialized(kw_bound_method, "ASYNC_CALL_KW_BOUND_METHOD") + self.assert_no_opcode(kw_bound_method, "CALL_KW_BOUND_METHOD") self.assert_no_opcode(bound_method_general, "CALL_BOUND_METHOD_GENERAL") diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 7ea3e13bf3025f..494750d86ca50f 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -396,6 +396,356 @@ DISPATCH(); } + TARGET(ASYNC_CALL_KW_BOUND_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + #endif + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(callable_s); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + #endif + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(ASYNC_CALL_PY_EXACT_ARGS) { #if _Py_TAIL_CALL_INTERP int opcode = ASYNC_CALL_PY_EXACT_ARGS; diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index d3b5ac10b2f440..b179dfdd6a4897 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -131,6 +131,8 @@ static void *opcode_targets_table[256] = { &&TARGET_RESUME, &&TARGET_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_ASYNC_CALL_BOUND_METHOD_GENERAL, + &&TARGET_ASYNC_CALL_KW_BOUND_METHOD, + &&TARGET_ASYNC_CALL_KW_PY, &&TARGET_ASYNC_CALL_PY_EXACT_ARGS, &&TARGET_ASYNC_CALL_PY_GENERAL, &&TARGET_BINARY_OP_ADD_FLOAT, @@ -231,8 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -482,8 +482,8 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -533,6 +533,8 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_PY(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_GENERAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); @@ -780,6 +782,8 @@ JUMP_TO_LABEL(error); static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_ASYNC_CALL_KW_PY, [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS, [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_ASYNC_CALL_PY_GENERAL, [BINARY_OP] = _TAIL_CALL_BINARY_OP, @@ -1024,8 +1028,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1038,6 +1040,8 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, @@ -1282,8 +1286,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 465723c4d65d8a..458845eb2bbc70 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5343,7 +5343,8 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; - // The ASYNC_CALL_* instructions are specializations of CALL for + // The ASYNC_CALL_* instructions are specializations of CALL and + // CALL_KW for // calls to coroutine functions that are awaited at once, as in // `await f(...)`. They are superinstructions: they merge the CALL // with the first execution of the instructions that follow it, @@ -5460,6 +5461,8 @@ dummy_func( CALL_KW_BOUND_METHOD, CALL_KW_PY, CALL_KW_NON_PY, + ASYNC_CALL_KW_BOUND_METHOD, + ASYNC_CALL_KW_PY, }; op(_MONITOR_CALL_KW, (callable, self_or_null, args[oparg], unused -- callable, self_or_null, args[oparg], unused)) { @@ -5627,6 +5630,29 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; + macro(ASYNC_CALL_KW_PY) = + _RECORD_CALLABLE_KW + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_FUNCTION_VERSION_KW + + _CHECK_RECURSION_REMAINING + + _PY_FRAME_KW + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_KW_BOUND_METHOD) = + _RECORD_CALLABLE_KW + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_METHOD_VERSION_KW + + _EXPAND_METHOD_KW + + flush + // so that self is in the argument array + _PY_FRAME_KW + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d598655227021d..8400c1b4bf6bc9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -396,6 +396,356 @@ DISPATCH(); } + TARGET(ASYNC_CALL_KW_BOUND_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + #endif + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(callable_s); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + assert(next_instr[1].op.code == PUSH_NULL); + assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + #endif + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + frame->instr_ptr += 4u + 3; + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(ASYNC_CALL_PY_EXACT_ARGS) { #if _Py_TAIL_CALL_INTERP int opcode = ASYNC_CALL_PY_EXACT_ARGS; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index d3b5ac10b2f440..b179dfdd6a4897 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -131,6 +131,8 @@ static void *opcode_targets_table[256] = { &&TARGET_RESUME, &&TARGET_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_ASYNC_CALL_BOUND_METHOD_GENERAL, + &&TARGET_ASYNC_CALL_KW_BOUND_METHOD, + &&TARGET_ASYNC_CALL_KW_PY, &&TARGET_ASYNC_CALL_PY_EXACT_ARGS, &&TARGET_ASYNC_CALL_PY_GENERAL, &&TARGET_BINARY_OP_ADD_FLOAT, @@ -231,8 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -482,8 +482,8 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -533,6 +533,8 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_PY(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_GENERAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); @@ -780,6 +782,8 @@ JUMP_TO_LABEL(error); static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_ASYNC_CALL_KW_PY, [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS, [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_ASYNC_CALL_PY_GENERAL, [BINARY_OP] = _TAIL_CALL_BINARY_OP, @@ -1024,8 +1028,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1038,6 +1040,8 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, @@ -1282,8 +1286,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index fbaa089ffeed04..b3fafc68c54638 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -196,6 +196,8 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [ASYNC_CALL_BOUND_METHOD_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_KW_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW_BOUND_METHOD] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [ASYNC_CALL_KW_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [ASYNC_CALL_KW_BOUND_METHOD] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW_NON_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_FUNCTION_EX] = {1, {_RECORD_4OS_INDEX}}, @@ -251,6 +253,8 @@ const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = { [ASYNC_CALL_BOUND_METHOD_GENERAL] = {1, 1, {0}}, [CALL_KW_PY] = {1, 0, {0}}, [CALL_KW_BOUND_METHOD] = {1, 0, {0}}, + [ASYNC_CALL_KW_PY] = {1, 0, {0}}, + [ASYNC_CALL_KW_BOUND_METHOD] = {1, 0, {0}}, [CALL_EX_PY] = {1, 0, {0}}, [BINARY_OP] = {2, 2, {1, 0}}, }; diff --git a/Python/specialize.c b/Python/specialize.c index 9d3d5053156088..e0746e86c7126d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1751,11 +1751,20 @@ specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, return 0; } +/* Is the result of the CALL or CALL_KW at "instr" awaited at once, so + * that the call to a coroutine function can be specialized to one of the + * ASYNC_CALL_* instructions? These start the new coroutine in place of + * the SEND that awaits it. GET_AWAITABLE never fails for a new coroutine. */ static bool -call_is_awaited(_Py_CODEUNIT *instr) +call_is_awaited(PyCodeObject *code, _Py_CODEUNIT *instr) { + int coro_flags = code->co_flags & + (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); + if (coro_flags != CO_COROUTINE) { + return false; + } /* None of the first three instructions can end the code. */ - _Py_CODEUNIT *next = instr + 1 + INLINE_CACHE_ENTRIES_CALL; + _Py_CODEUNIT *next = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[instr->op.code]]; return next[0].op.code == GET_AWAITABLE && next[1].op.code == PUSH_NULL && next[2].op.code == LOAD_COMMON_CONSTANT @@ -1793,11 +1802,7 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return -1; } write_u32(cache->func_version, version); - /* ASYNC_CALL_* start the new coroutine in place of the SEND that - * awaits it. GET_AWAITABLE never fails for a new coroutine. */ - int coro_flags = code->co_flags & - (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); - bool awaited = coro_flags == CO_COROUTINE && call_is_awaited(instr); + bool awaited = call_is_awaited(code, instr); uint8_t opcode; if (argcount == nargs + bound_method) { if (awaited) { @@ -1847,7 +1852,14 @@ specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return -1; } write_u32(cache->func_version, version); - specialize(instr, bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY); + uint8_t opcode; + if (call_is_awaited(code, instr)) { + opcode = bound_method ? ASYNC_CALL_KW_BOUND_METHOD : ASYNC_CALL_KW_PY; + } + else { + opcode = bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY; + } + specialize(instr, opcode); return 0; } From 462db8d9c32e051271cd5ee36bbea583c56528e1 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 20:55:05 +0100 Subject: [PATCH 11/15] Keep _Py_MakeCoro() as is and add _Py_MakeCoroWithFrame() Co-Authored-By: Claude Fable 5.1 --- Include/internal/pycore_ceval.h | 7 ++++--- Modules/_testinternalcapi/test_cases.c.h | 14 +++++++------- Objects/genobject.c | 8 +++++++- Python/bytecodes.c | 4 ++-- Python/executor_cases.c.h | 4 ++-- Python/generated_cases.c.h | 14 +++++++------- 6 files changed, 29 insertions(+), 22 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 36fec51ab081d1..0e66b7393c3082 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -266,9 +266,10 @@ extern int _PyEval_EnsureBuiltinsWithModule( PyObject *, PyObject **p_builtins); -// Create the generator object for the frame of a call to a generator -// function; the frame has not started executing yet. -PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func, _PyInterpreterFrame *frame); +PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); +// Same, for the frame of a call to a generator function that is not the +// current frame; the frame has not started executing yet. +extern PyObject *_Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 494750d86ca50f..ee5a78262c37d8 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -163,7 +163,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -345,7 +345,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -530,7 +530,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -695,7 +695,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -861,7 +861,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -1015,7 +1015,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -12493,7 +12493,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { JUMP_TO_LABEL(error); diff --git a/Objects/genobject.c b/Objects/genobject.c index c07adaf6bf93d9..f5dd4501b70180 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1128,7 +1128,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) } PyObject * -_Py_MakeCoro(PyFunctionObject *func, _PyInterpreterFrame *frame) +_Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame) { int coro_flags = ((PyCodeObject *)func->func_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); @@ -1173,6 +1173,12 @@ _Py_MakeCoro(PyFunctionObject *func, _PyInterpreterFrame *frame) return coro; } +PyObject * +_Py_MakeCoro(PyFunctionObject *func) +{ + return _Py_MakeCoroWithFrame(func, _PyThreadState_GET()->current_frame); +} + static PyObject * gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, PyObject *name, PyObject *qualname) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 458845eb2bbc70..638b7734d2581b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5377,7 +5377,7 @@ dummy_func( SYNC_SP(); assert(PyStackRef_FunctionCheck(callee->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); if (gen == NULL) { _PyEval_FrameClearAndPop(tstate, callee); ERROR_NO_POP(); @@ -5949,7 +5949,7 @@ dummy_func( else { assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); ERROR_IF(gen == NULL); assert(STACK_LEVEL() <= 2); SAVE_STACK(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 493a61b398501f..184d95e43139ae 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -20149,7 +20149,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -20989,7 +20989,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { SET_CURRENT_CACHED_VALUES(0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8400c1b4bf6bc9..11887176300538 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -163,7 +163,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -345,7 +345,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -530,7 +530,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -695,7 +695,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -861,7 +861,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -1015,7 +1015,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, callee); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); @@ -12490,7 +12490,7 @@ PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func, frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { JUMP_TO_LABEL(error); From d4107d8ca855dd5ec0d29d27bfe3286f2f5a6509 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 20:56:42 +0100 Subject: [PATCH 12/15] Declare _Py_MakeCoroWithFrame() with PyAPI_FUNC like _Py_MakeCoro() Co-Authored-By: Claude Fable 5.1 --- Include/internal/pycore_ceval.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 0e66b7393c3082..cd73712bd325af 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -269,7 +269,7 @@ extern int _PyEval_EnsureBuiltinsWithModule( PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); // Same, for the frame of a call to a generator function that is not the // current frame; the frame has not started executing yet. -extern PyObject *_Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame); +PyAPI_FUNC(PyObject *)_Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ From 539e937316a810c832b2d006d8235e6520592f72 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 20:57:15 +0100 Subject: [PATCH 13/15] Drop the comment on _Py_MakeCoroWithFrame() Co-Authored-By: Claude Fable 5.1 --- Include/internal/pycore_ceval.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index cd73712bd325af..f60605e609c3be 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -267,8 +267,6 @@ extern int _PyEval_EnsureBuiltinsWithModule( PyObject **p_builtins); PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); -// Same, for the frame of a call to a generator function that is not the -// current frame; the frame has not started executing yet. PyAPI_FUNC(PyObject *)_Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame); /* Handle signals, pending calls, GIL drop request From 6b3055680f827b8fb3314c78c9e236a1aac132f2 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 21:15:15 +0100 Subject: [PATCH 14/15] Shorten the comment in RETURN_GENERATOR Co-Authored-By: Claude Fable 5.1 --- Python/bytecodes.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 638b7734d2581b..bd97a94d42ccb0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5941,9 +5941,7 @@ dummy_func( inst(RETURN_GENERATOR, (-- res)) { if (frame->owner == FRAME_OWNED_BY_GENERATOR) { - // ASYNC_CALL_* has already created the coroutine and it is - // running in this frame. See _ASYNC_CALL_GEN_FRAME. - // Push the value that the following POP_TOP discards. + // Handled by ASYNC_CALL_* already. res = PyStackRef_None; } else { From 5b3c7b4848ec97e70c7aebfe8df5441f5d9ab4d3 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Sun, 8 Nov 2026 21:21:38 +0100 Subject: [PATCH 15/15] Match the awaited call tail in one place _PyCode_AwaitedCallSend() returns the SEND of `await f(...)` and is used by the specializer, the tier-1 guard and _ASYNC_CALL_GEN_FRAME. Co-Authored-By: Claude Fable 5.1 --- Include/internal/pycore_code.h | 21 +++++++++ Modules/_testinternalcapi/test_cases.c.h | 60 ++++++++++++------------ Python/bytecodes.c | 13 +++-- Python/executor_cases.c.h | 19 ++++---- Python/generated_cases.c.h | 60 ++++++++++++------------ Python/specialize.c | 7 +-- Tools/cases_generator/analyzer.py | 1 + 7 files changed, 97 insertions(+), 84 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index aeb2ad06ad07cb..ebb1f7c97593e8 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -9,6 +9,8 @@ extern "C" { #endif #include "pycore_backoff.h" // _Py_BackoffCounter +#include "pycore_opcode_utils.h" // CONSTANT_NONE +#include "opcode_ids.h" // GET_AWAITABLE #include "pycore_structs.h" // _Py_CODEUNIT #include "pycore_tstate.h" // _PyThreadStateImpl @@ -542,6 +544,25 @@ PyAPI_FUNC(int) _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(_Py_CODEUNIT) _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); +PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; + +/* The SEND that awaits the result of the CALL or CALL_KW whose cache ends + * at `tail` at once, as in `await f(...)`, or NULL. */ +static inline _Py_CODEUNIT * +_PyCode_AwaitedCallSend(_Py_CODEUNIT *tail) +{ + /* None of the first three instructions can end the code. */ + if (tail[0].op.code == GET_AWAITABLE + && tail[1].op.code == PUSH_NULL + && tail[2].op.code == LOAD_COMMON_CONSTANT + && tail[2].op.arg == CONSTANT_NONE + && _PyOpcode_Deopt[tail[3].op.code] == SEND) + { + return &tail[3]; + } + return NULL; +} + extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); extern PyObject *_PyInstrumentation_BranchesIterator(PyCodeObject *code); diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index ee5a78262c37d8..18752bf322fdc1 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -48,8 +48,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -60,6 +58,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS @@ -179,10 +178,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); @@ -244,8 +244,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -256,6 +254,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_METHOD_VERSION @@ -361,10 +360,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); @@ -427,8 +427,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); @@ -439,6 +437,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_METHOD_VERSION_KW @@ -546,10 +545,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); @@ -612,8 +612,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); @@ -624,6 +622,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_FUNCTION_VERSION_KW @@ -711,10 +710,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); @@ -776,8 +776,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -788,6 +786,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_FUNCTION_VERSION @@ -877,10 +876,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); @@ -942,8 +942,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -954,6 +952,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_FUNCTION_VERSION @@ -1031,10 +1030,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bd97a94d42ccb0..4fa3f6b6b7394e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5360,10 +5360,9 @@ dummy_func( // Executors are invalidated when the code is instrumented. op(_GUARD_ASYNC_CALL_TAIL, (--)) { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); DEOPT_IF(next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE); DEOPT_IF(next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE); + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } @@ -5392,12 +5391,12 @@ dummy_func( // RETURN_GENERATOR only pushes the None that POP_TOP discards. gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - // The coroutine yields and returns to the SEND, which follows - // GET_AWAITABLE, PUSH_NULL and LOAD_COMMON_CONSTANT. - frame->instr_ptr += INSTRUCTION_SIZE + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + // The coroutine yields and returns to the SEND of the await. + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + INSTRUCTION_SIZE); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 184d95e43139ae..1ee676c07903c4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -20027,8 +20027,6 @@ CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -20039,6 +20037,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif SET_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -20050,8 +20049,6 @@ ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UOP_STAT_INC(uopcode, miss); _tos_cache0 = _stack_item_0; @@ -20064,6 +20061,7 @@ SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); @@ -20077,8 +20075,6 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = _stack_item_1; @@ -20093,6 +20089,7 @@ SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; @@ -20108,8 +20105,6 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UOP_STAT_INC(uopcode, miss); _tos_cache2 = _stack_item_2; @@ -20126,6 +20121,7 @@ SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif _tos_cache2 = _stack_item_2; _tos_cache1 = _stack_item_1; @@ -20166,10 +20162,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 11887176300538..916f2a6fc46705 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -48,8 +48,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -60,6 +58,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS @@ -179,10 +178,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); @@ -244,8 +244,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -256,6 +254,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_METHOD_VERSION @@ -361,10 +360,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); @@ -427,8 +427,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); @@ -439,6 +437,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_METHOD_VERSION_KW @@ -546,10 +545,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); @@ -612,8 +612,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); @@ -624,6 +622,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_FUNCTION_VERSION_KW @@ -711,10 +710,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); @@ -776,8 +776,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -788,6 +786,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_FUNCTION_VERSION @@ -877,10 +876,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); @@ -942,8 +942,6 @@ // _GUARD_ASYNC_CALL_TAIL { #if TIER_ONE - assert(next_instr[1].op.code == PUSH_NULL); - assert(next_instr[2].op.code == LOAD_COMMON_CONSTANT); if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -954,6 +952,7 @@ assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); #endif } // _CHECK_FUNCTION_VERSION @@ -1031,10 +1030,11 @@ } gen_take_frame(tstate, gen, callee, frame); gen->gi_frame_state = FRAME_EXECUTING; - frame->instr_ptr += 4u + 3; - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND); + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; gen_link_frame(tstate, gen, frame, - (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + frame->instr_ptr->op.arg)); + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); null = PyStackRef_NULL; gen_frame = PyStackRef_Wrap(&gen->gi_iframe); diff --git a/Python/specialize.c b/Python/specialize.c index e0746e86c7126d..7a87f989d9b5f0 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1763,13 +1763,8 @@ call_is_awaited(PyCodeObject *code, _Py_CODEUNIT *instr) if (coro_flags != CO_COROUTINE) { return false; } - /* None of the first three instructions can end the code. */ _Py_CODEUNIT *next = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[instr->op.code]]; - return next[0].op.code == GET_AWAITABLE - && next[1].op.code == PUSH_NULL - && next[2].op.code == LOAD_COMMON_CONSTANT - && next[2].op.arg == CONSTANT_NONE - && _PyOpcode_Deopt[next[3].op.code] == SEND; + return _PyCode_AwaitedCallSend(next) != NULL; } static int diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 3b4c3664ae7fc4..01d0ecfb5a47ab 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -604,6 +604,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyGen_GetGeneratorFromFrame", "gen_try_set_executing", "gen_link_frame", + "_PyCode_AwaitedCallSend", "gen_take_frame", "_PyInterpreterState_GET", "_PyList_AppendTakeRef",