diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a0fea389b6502c..f60605e609c3be 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -267,6 +267,7 @@ extern int _PyEval_EnsureBuiltinsWithModule( PyObject **p_builtins); PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); +PyAPI_FUNC(PyObject *)_Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame); /* Handle signals, pending calls, GIL drop request and asynchronous exception */ diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index aeb2ad06ad07cb..ebb1f7c97593e8 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -9,6 +9,8 @@ extern "C" { #endif #include "pycore_backoff.h" // _Py_BackoffCounter +#include "pycore_opcode_utils.h" // CONSTANT_NONE +#include "opcode_ids.h" // GET_AWAITABLE #include "pycore_structs.h" // _Py_CODEUNIT #include "pycore_tstate.h" // _PyThreadStateImpl @@ -542,6 +544,25 @@ PyAPI_FUNC(int) _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(_Py_CODEUNIT) _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); +PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; + +/* The SEND that awaits the result of the CALL or CALL_KW whose cache ends + * at `tail` at once, as in `await f(...)`, or NULL. */ +static inline _Py_CODEUNIT * +_PyCode_AwaitedCallSend(_Py_CODEUNIT *tail) +{ + /* None of the first three instructions can end the code. */ + if (tail[0].op.code == GET_AWAITABLE + && tail[1].op.code == PUSH_NULL + && tail[2].op.code == LOAD_COMMON_CONSTANT + && tail[2].op.arg == CONSTANT_NONE + && _PyOpcode_Deopt[tail[3].op.code] == SEND) + { + return &tail[3]; + } + return NULL; +} + extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); extern PyObject *_PyInstrumentation_BranchesIterator(PyCodeObject *code); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 04b826ed09144e..dbd238ed8df491 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -38,6 +38,18 @@ int _PyOpcode_num_popped(int opcode, int oparg) { switch(opcode) { case ANNOTATIONS_PLACEHOLDER: return 0; + case ASYNC_CALL_BOUND_METHOD_EXACT_ARGS: + return 2 + oparg; + case ASYNC_CALL_BOUND_METHOD_GENERAL: + return 2 + oparg; + case ASYNC_CALL_KW_BOUND_METHOD: + return 3 + oparg; + case ASYNC_CALL_KW_PY: + return 3 + oparg; + case ASYNC_CALL_PY_EXACT_ARGS: + return 2 + oparg; + case ASYNC_CALL_PY_GENERAL: + return 2 + oparg; case BINARY_OP: return 2; case BINARY_OP_ADD_FLOAT: @@ -533,6 +545,18 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { switch(opcode) { case ANNOTATIONS_PLACEHOLDER: return 0; + case ASYNC_CALL_BOUND_METHOD_EXACT_ARGS: + return 2; + case ASYNC_CALL_BOUND_METHOD_GENERAL: + return 2; + case ASYNC_CALL_KW_BOUND_METHOD: + return 2; + case ASYNC_CALL_KW_PY: + return 2; + case ASYNC_CALL_PY_EXACT_ARGS: + return 2; + case ASYNC_CALL_PY_GENERAL: + return 2; case BINARY_OP: return 1; case BINARY_OP_ADD_FLOAT: @@ -1100,6 +1124,12 @@ struct opcode_metadata { PyAPI_DATA(const struct opcode_metadata) _PyOpcode_opcode_metadata[267]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [ASYNC_CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, @@ -1345,7 +1375,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { }; #endif -#define MAX_UOP_PER_EXPANSION 11 +#define MAX_UOP_PER_EXPANSION 12 struct opcode_macro_expansion { int nuops; struct { int16_t uop; int8_t size; int8_t offset; } uops[MAX_UOP_PER_EXPANSION]; @@ -1355,6 +1385,12 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 12, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = { .nuops = 9, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_KW_BOUND_METHOD] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_KW_PY] = { .nuops = 8, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_PY_EXACT_ARGS] = { .nuops = 10, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [ASYNC_CALL_PY_GENERAL] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_ASYNC_CALL_TAIL, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _ASYNC_CALL_GEN_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [BINARY_OP] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _BINARY_OP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [BINARY_OP_ADD_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_ADD_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, @@ -1553,6 +1589,12 @@ PyAPI_DATA(const char) *_PyOpcode_OpName[267]; #ifdef NEED_OPCODE_METADATA const char *_PyOpcode_OpName[267] = { [ANNOTATIONS_PLACEHOLDER] = "ANNOTATIONS_PLACEHOLDER", + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = "ASYNC_CALL_BOUND_METHOD_EXACT_ARGS", + [ASYNC_CALL_BOUND_METHOD_GENERAL] = "ASYNC_CALL_BOUND_METHOD_GENERAL", + [ASYNC_CALL_KW_BOUND_METHOD] = "ASYNC_CALL_KW_BOUND_METHOD", + [ASYNC_CALL_KW_PY] = "ASYNC_CALL_KW_PY", + [ASYNC_CALL_PY_EXACT_ARGS] = "ASYNC_CALL_PY_EXACT_ARGS", + [ASYNC_CALL_PY_GENERAL] = "ASYNC_CALL_PY_GENERAL", [BINARY_OP] = "BINARY_OP", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", @@ -1839,12 +1881,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [219] = 219, - [220] = 220, - [221] = 221, - [222] = 222, - [223] = 223, - [224] = 224, [225] = 225, [226] = 226, [227] = 227, @@ -1853,6 +1889,12 @@ const uint8_t _PyOpcode_Deopt[256] = { [230] = 230, [231] = 231, [232] = 232, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = CALL, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = CALL, + [ASYNC_CALL_KW_BOUND_METHOD] = CALL_KW, + [ASYNC_CALL_KW_PY] = CALL_KW, + [ASYNC_CALL_PY_EXACT_ARGS] = CALL, + [ASYNC_CALL_PY_GENERAL] = CALL, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -2100,12 +2142,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 219: \ - case 220: \ - case 221: \ - case 222: \ - case 223: \ - case 224: \ case 225: \ case 226: \ case 227: \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ba6e0252024a6d..a04629e5dd0ed2 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -13,6 +13,7 @@ enum { _EXIT_TRACE = 300, _SET_IP, _ALLOCATE_OBJECT, + _ASYNC_CALL_GEN_FRAME, _BINARY_OP, _BINARY_OP_ADD_FLOAT, _BINARY_OP_ADD_FLOAT_INPLACE, @@ -128,6 +129,7 @@ enum { _GET_ITER, _GET_ITER_TRAD, _GUARD_3OS_ASYNC_GEN_ASEND, + _GUARD_ASYNC_CALL_TAIL, _GUARD_BINARY_OP_EXTEND, _GUARD_BINARY_OP_EXTEND_LHS, _GUARD_BINARY_OP_EXTEND_RHS, @@ -366,7 +368,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 654 +#define MAX_UOP_ID 656 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -443,6 +445,7 @@ enum { enum { _ALLOCATE_OBJECT_r00 = MAX_UOP_ID + 1, + _ASYNC_CALL_GEN_FRAME_r13, _BINARY_OP_r23, _BINARY_OP_ADD_FLOAT_r03, _BINARY_OP_ADD_FLOAT_r13, @@ -698,6 +701,10 @@ enum { _GUARD_3OS_ASYNC_GEN_ASEND_r13, _GUARD_3OS_ASYNC_GEN_ASEND_r23, _GUARD_3OS_ASYNC_GEN_ASEND_r33, + _GUARD_ASYNC_CALL_TAIL_r00, + _GUARD_ASYNC_CALL_TAIL_r11, + _GUARD_ASYNC_CALL_TAIL_r22, + _GUARD_ASYNC_CALL_TAIL_r33, _GUARD_BINARY_OP_EXTEND_r22, _GUARD_BINARY_OP_EXTEND_LHS_r02, _GUARD_BINARY_OP_EXTEND_LHS_r12, @@ -1309,6 +1316,9 @@ enum { _SAVE_RETURN_OFFSET_r33, _SEND_ASYNC_GEN_r33, _SEND_ASYNC_GEN_TIER_TWO_r33, + _SEND_GEN_FRAME_r03, + _SEND_GEN_FRAME_r13, + _SEND_GEN_FRAME_r23, _SEND_GEN_FRAME_r33, _SEND_VIRTUAL_r33, _SEND_VIRTUAL_TIER_TWO_r03, @@ -1444,7 +1454,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1655 +#define MAX_UOP_REGS_ID 1665 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0cd82c53aa932a..53cf9ed65fa47c 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -352,6 +352,8 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_ASYNC_CALL_TAIL] = HAS_DEOPT_FLAG, + [_ASYNC_CALL_GEN_FRAME] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -1645,11 +1647,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_SEND_GEN_FRAME] = { - .best = { 3, 3, 3, 3 }, + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 3, 0, _SEND_GEN_FRAME_r03 }, + { 3, 1, _SEND_GEN_FRAME_r13 }, + { 3, 2, _SEND_GEN_FRAME_r23 }, { 3, 3, _SEND_GEN_FRAME_r33 }, }, }, @@ -3300,6 +3302,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_ASYNC_CALL_TAIL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 0, 0, _GUARD_ASYNC_CALL_TAIL_r00 }, + { 1, 1, _GUARD_ASYNC_CALL_TAIL_r11 }, + { 2, 2, _GUARD_ASYNC_CALL_TAIL_r22 }, + { 3, 3, _GUARD_ASYNC_CALL_TAIL_r33 }, + }, + }, + [_ASYNC_CALL_GEN_FRAME] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 3, 3, _ASYNC_CALL_GEN_FRAME_r13 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_MAYBE_EXPAND_METHOD_KW] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -4285,6 +4305,9 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GET_AITER_r11] = _GET_AITER, [_GET_ANEXT_r12] = _GET_ANEXT, [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_SEND_GEN_FRAME_r03] = _SEND_GEN_FRAME, + [_SEND_GEN_FRAME_r13] = _SEND_GEN_FRAME, + [_SEND_GEN_FRAME_r23] = _SEND_GEN_FRAME, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, @@ -4642,6 +4665,11 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, [_CALL_METHOD_DESCRIPTOR_FAST_r00] = _CALL_METHOD_DESCRIPTOR_FAST, [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_INLINE, + [_GUARD_ASYNC_CALL_TAIL_r00] = _GUARD_ASYNC_CALL_TAIL, + [_GUARD_ASYNC_CALL_TAIL_r11] = _GUARD_ASYNC_CALL_TAIL, + [_GUARD_ASYNC_CALL_TAIL_r22] = _GUARD_ASYNC_CALL_TAIL, + [_GUARD_ASYNC_CALL_TAIL_r33] = _GUARD_ASYNC_CALL_TAIL, + [_ASYNC_CALL_GEN_FRAME_r13] = _ASYNC_CALL_GEN_FRAME, [_MAYBE_EXPAND_METHOD_KW_r11] = _MAYBE_EXPAND_METHOD_KW, [_PY_FRAME_KW_r11] = _PY_FRAME_KW, [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, @@ -4867,6 +4895,8 @@ const uint16_t _PyUop_SpillsAndReloads[4][4] = { const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_ALLOCATE_OBJECT] = "_ALLOCATE_OBJECT", [_ALLOCATE_OBJECT_r00] = "_ALLOCATE_OBJECT_r00", + [_ASYNC_CALL_GEN_FRAME] = "_ASYNC_CALL_GEN_FRAME", + [_ASYNC_CALL_GEN_FRAME_r13] = "_ASYNC_CALL_GEN_FRAME_r13", [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_r23] = "_BINARY_OP_r23", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", @@ -5250,6 +5280,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = "_GUARD_3OS_ASYNC_GEN_ASEND_r13", [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = "_GUARD_3OS_ASYNC_GEN_ASEND_r23", [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = "_GUARD_3OS_ASYNC_GEN_ASEND_r33", + [_GUARD_ASYNC_CALL_TAIL] = "_GUARD_ASYNC_CALL_TAIL", + [_GUARD_ASYNC_CALL_TAIL_r00] = "_GUARD_ASYNC_CALL_TAIL_r00", + [_GUARD_ASYNC_CALL_TAIL_r11] = "_GUARD_ASYNC_CALL_TAIL_r11", + [_GUARD_ASYNC_CALL_TAIL_r22] = "_GUARD_ASYNC_CALL_TAIL_r22", + [_GUARD_ASYNC_CALL_TAIL_r33] = "_GUARD_ASYNC_CALL_TAIL_r33", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", [_GUARD_BINARY_OP_EXTEND_LHS] = "_GUARD_BINARY_OP_EXTEND_LHS", @@ -6011,6 +6046,9 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_SEND_ASYNC_GEN_TIER_TWO] = "_SEND_ASYNC_GEN_TIER_TWO", [_SEND_ASYNC_GEN_TIER_TWO_r33] = "_SEND_ASYNC_GEN_TIER_TWO_r33", [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", + [_SEND_GEN_FRAME_r03] = "_SEND_GEN_FRAME_r03", + [_SEND_GEN_FRAME_r13] = "_SEND_GEN_FRAME_r13", + [_SEND_GEN_FRAME_r23] = "_SEND_GEN_FRAME_r23", [_SEND_GEN_FRAME_r33] = "_SEND_GEN_FRAME_r33", [_SEND_VIRTUAL_TIER_TWO] = "_SEND_VIRTUAL_TIER_TWO", [_SEND_VIRTUAL_TIER_TWO_r03] = "_SEND_VIRTUAL_TIER_TWO_r03", @@ -6832,6 +6870,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_METHOD_DESCRIPTOR_FAST_INLINE: return 0; + case _GUARD_ASYNC_CALL_TAIL: + return 0; + case _ASYNC_CALL_GEN_FRAME: + return 1; case _MAYBE_EXPAND_METHOD_KW: return 0; case _PY_FRAME_KW: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 11342ae451b9f6..0ce90edf933442 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -128,96 +128,102 @@ extern "C" { #define UNPACK_SEQUENCE 115 #define YIELD_VALUE 116 #define RESUME 128 -#define BINARY_OP_ADD_FLOAT 129 -#define BINARY_OP_ADD_INT 130 -#define BINARY_OP_ADD_UNICODE 131 -#define BINARY_OP_EXTEND 132 -#define BINARY_OP_MULTIPLY_FLOAT 133 -#define BINARY_OP_MULTIPLY_INT 134 -#define BINARY_OP_SUBSCR_DICT 135 -#define BINARY_OP_SUBSCR_GETITEM 136 -#define BINARY_OP_SUBSCR_LIST_INT 137 -#define BINARY_OP_SUBSCR_LIST_SLICE 138 -#define BINARY_OP_SUBSCR_STR_INT 139 -#define BINARY_OP_SUBSCR_TUPLE_INT 140 -#define BINARY_OP_SUBSCR_USTR_INT 141 -#define BINARY_OP_SUBTRACT_FLOAT 142 -#define BINARY_OP_SUBTRACT_INT 143 -#define CALL_ALLOC_AND_ENTER_INIT 144 -#define CALL_BOUND_METHOD_EXACT_ARGS 145 -#define CALL_BOUND_METHOD_GENERAL 146 -#define CALL_BUILTIN_CLASS 147 -#define CALL_BUILTIN_FAST 148 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 149 -#define CALL_BUILTIN_O 150 -#define CALL_EX_NON_PY_GENERAL 151 -#define CALL_EX_PY 152 -#define CALL_ISINSTANCE 153 -#define CALL_KW_BOUND_METHOD 154 -#define CALL_KW_NON_PY 155 -#define CALL_KW_PY 156 -#define CALL_LEN 157 -#define CALL_LIST_APPEND 158 -#define CALL_METHOD_DESCRIPTOR_FAST 159 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 160 -#define CALL_METHOD_DESCRIPTOR_NOARGS 161 -#define CALL_METHOD_DESCRIPTOR_O 162 -#define CALL_NON_PY_GENERAL 163 -#define CALL_PY_EXACT_ARGS 164 -#define CALL_PY_GENERAL 165 -#define CALL_STR_1 166 -#define CALL_TUPLE_1 167 -#define CALL_TYPE_1 168 -#define COMPARE_OP_FLOAT 169 -#define COMPARE_OP_INT 170 -#define COMPARE_OP_STR 171 -#define CONTAINS_OP_DICT 172 -#define CONTAINS_OP_SET 173 -#define FOR_ITER_GEN 174 -#define FOR_ITER_LIST 175 -#define FOR_ITER_RANGE 176 -#define FOR_ITER_TUPLE 177 -#define FOR_ITER_VIRTUAL 178 -#define GET_ITER_SELF 179 -#define GET_ITER_VIRTUAL 180 -#define JUMP_BACKWARD_JIT 181 -#define JUMP_BACKWARD_NO_JIT 182 -#define LOAD_ATTR_CLASS 183 -#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 184 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 185 -#define LOAD_ATTR_INSTANCE_VALUE 186 -#define LOAD_ATTR_METHOD_LAZY_DICT 187 -#define LOAD_ATTR_METHOD_NO_DICT 188 -#define LOAD_ATTR_METHOD_WITH_VALUES 189 -#define LOAD_ATTR_MODULE 190 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 191 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 192 -#define LOAD_ATTR_PROPERTY 193 -#define LOAD_ATTR_SLOT 194 -#define LOAD_ATTR_WITH_HINT 195 -#define LOAD_GLOBAL_BUILTIN 196 -#define LOAD_GLOBAL_MODULE 197 -#define LOAD_SUPER_ATTR_ATTR 198 -#define LOAD_SUPER_ATTR_METHOD 199 -#define RESUME_CHECK 200 -#define RESUME_CHECK_JIT 201 -#define SEND_ASYNC_GEN 202 -#define SEND_GEN 203 -#define SEND_VIRTUAL 204 -#define STORE_ATTR_INSTANCE_VALUE 205 -#define STORE_ATTR_SLOT 206 -#define STORE_ATTR_WITH_HINT 207 -#define STORE_SUBSCR_DICT 208 -#define STORE_SUBSCR_LIST_INT 209 -#define TO_BOOL_ALWAYS_TRUE 210 -#define TO_BOOL_BOOL 211 -#define TO_BOOL_INT 212 -#define TO_BOOL_LIST 213 -#define TO_BOOL_NONE 214 -#define TO_BOOL_STR 215 -#define UNPACK_SEQUENCE_LIST 216 -#define UNPACK_SEQUENCE_TUPLE 217 -#define UNPACK_SEQUENCE_TWO_TUPLE 218 +#define ASYNC_CALL_BOUND_METHOD_EXACT_ARGS 129 +#define ASYNC_CALL_BOUND_METHOD_GENERAL 130 +#define ASYNC_CALL_KW_BOUND_METHOD 131 +#define ASYNC_CALL_KW_PY 132 +#define ASYNC_CALL_PY_EXACT_ARGS 133 +#define ASYNC_CALL_PY_GENERAL 134 +#define BINARY_OP_ADD_FLOAT 135 +#define BINARY_OP_ADD_INT 136 +#define BINARY_OP_ADD_UNICODE 137 +#define BINARY_OP_EXTEND 138 +#define BINARY_OP_MULTIPLY_FLOAT 139 +#define BINARY_OP_MULTIPLY_INT 140 +#define BINARY_OP_SUBSCR_DICT 141 +#define BINARY_OP_SUBSCR_GETITEM 142 +#define BINARY_OP_SUBSCR_LIST_INT 143 +#define BINARY_OP_SUBSCR_LIST_SLICE 144 +#define BINARY_OP_SUBSCR_STR_INT 145 +#define BINARY_OP_SUBSCR_TUPLE_INT 146 +#define BINARY_OP_SUBSCR_USTR_INT 147 +#define BINARY_OP_SUBTRACT_FLOAT 148 +#define BINARY_OP_SUBTRACT_INT 149 +#define CALL_ALLOC_AND_ENTER_INIT 150 +#define CALL_BOUND_METHOD_EXACT_ARGS 151 +#define CALL_BOUND_METHOD_GENERAL 152 +#define CALL_BUILTIN_CLASS 153 +#define CALL_BUILTIN_FAST 154 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 155 +#define CALL_BUILTIN_O 156 +#define CALL_EX_NON_PY_GENERAL 157 +#define CALL_EX_PY 158 +#define CALL_ISINSTANCE 159 +#define CALL_KW_BOUND_METHOD 160 +#define CALL_KW_NON_PY 161 +#define CALL_KW_PY 162 +#define CALL_LEN 163 +#define CALL_LIST_APPEND 164 +#define CALL_METHOD_DESCRIPTOR_FAST 165 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 166 +#define CALL_METHOD_DESCRIPTOR_NOARGS 167 +#define CALL_METHOD_DESCRIPTOR_O 168 +#define CALL_NON_PY_GENERAL 169 +#define CALL_PY_EXACT_ARGS 170 +#define CALL_PY_GENERAL 171 +#define CALL_STR_1 172 +#define CALL_TUPLE_1 173 +#define CALL_TYPE_1 174 +#define COMPARE_OP_FLOAT 175 +#define COMPARE_OP_INT 176 +#define COMPARE_OP_STR 177 +#define CONTAINS_OP_DICT 178 +#define CONTAINS_OP_SET 179 +#define FOR_ITER_GEN 180 +#define FOR_ITER_LIST 181 +#define FOR_ITER_RANGE 182 +#define FOR_ITER_TUPLE 183 +#define FOR_ITER_VIRTUAL 184 +#define GET_ITER_SELF 185 +#define GET_ITER_VIRTUAL 186 +#define JUMP_BACKWARD_JIT 187 +#define JUMP_BACKWARD_NO_JIT 188 +#define LOAD_ATTR_CLASS 189 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 190 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 191 +#define LOAD_ATTR_INSTANCE_VALUE 192 +#define LOAD_ATTR_METHOD_LAZY_DICT 193 +#define LOAD_ATTR_METHOD_NO_DICT 194 +#define LOAD_ATTR_METHOD_WITH_VALUES 195 +#define LOAD_ATTR_MODULE 196 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 197 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 198 +#define LOAD_ATTR_PROPERTY 199 +#define LOAD_ATTR_SLOT 200 +#define LOAD_ATTR_WITH_HINT 201 +#define LOAD_GLOBAL_BUILTIN 202 +#define LOAD_GLOBAL_MODULE 203 +#define LOAD_SUPER_ATTR_ATTR 204 +#define LOAD_SUPER_ATTR_METHOD 205 +#define RESUME_CHECK 206 +#define RESUME_CHECK_JIT 207 +#define SEND_ASYNC_GEN 208 +#define SEND_GEN 209 +#define SEND_VIRTUAL 210 +#define STORE_ATTR_INSTANCE_VALUE 211 +#define STORE_ATTR_SLOT 212 +#define STORE_ATTR_WITH_HINT 213 +#define STORE_SUBSCR_DICT 214 +#define STORE_SUBSCR_LIST_INT 215 +#define TO_BOOL_ALWAYS_TRUE 216 +#define TO_BOOL_BOOL 217 +#define TO_BOOL_INT 218 +#define TO_BOOL_LIST 219 +#define TO_BOOL_NONE 220 +#define TO_BOOL_STR 221 +#define UNPACK_SEQUENCE_LIST 222 +#define UNPACK_SEQUENCE_TUPLE 223 +#define UNPACK_SEQUENCE_TWO_TUPLE 224 #define INSTRUMENTED_END_FOR 233 #define INSTRUMENTED_POP_ITER 234 #define INSTRUMENTED_END_SEND 235 diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index fa652cd231cfa9..c1a7fa9d9e46c4 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -104,3 +104,18 @@ so this data flow works along the chain (see `gen_send_ex2()` in Recall that a generator's `__next__` function simply calls `self.send(None)`, so all this works the same in generators and coroutines, but only coroutines use the value of the argument to `send`. + +An `await` expression is compiled like `yield from`, with +[`GET_AWAITABLE`](https://docs.python.org/dev/library/dis.html#opcode-GET_AWAITABLE) +in place of `GET_ITER`. When the awaited object is the result of a call to +a coroutine function, as in `await f()`, the `CALL` (or `CALL_KW`) is +[`specialized`](interpreter.md) to one of the `ASYNC_CALL_*` instructions, +such as `ASYNC_CALL_PY_EXACT_ARGS`. These are superinstructions: they merge +the call with the first execution of the `GET_AWAITABLE`, `PUSH_NULL`, +`LOAD_COMMON_CONSTANT` and `SEND` instructions that follow it. + +`ASYNC_CALL_*` create the frame with the micro-ops of the matching `CALL_*` +or `CALL_KW_*` specialization. They then turn it into the frame of a new coroutine and push +it at once, as `RETURN_GENERATOR` and `SEND_GEN` would do. The coroutine is +not returned to the caller first and the frame of the caller is left as if +it were executing the `SEND`. diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index df92eae151d248..c05245bdfa530c 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -120,11 +120,17 @@ "CALL_PY_GENERAL", "CALL_BOUND_METHOD_GENERAL", "CALL_NON_PY_GENERAL", + "ASYNC_CALL_PY_EXACT_ARGS", + "ASYNC_CALL_BOUND_METHOD_EXACT_ARGS", + "ASYNC_CALL_PY_GENERAL", + "ASYNC_CALL_BOUND_METHOD_GENERAL", ), CALL_KW=( "CALL_KW_BOUND_METHOD", "CALL_KW_PY", "CALL_KW_NON_PY", + "ASYNC_CALL_KW_BOUND_METHOD", + "ASYNC_CALL_KW_PY", ), CALL_FUNCTION_EX=( "CALL_EX_PY", @@ -133,97 +139,103 @@ ) _specialized_opmap = frozendict( - BINARY_OP_ADD_FLOAT=129, - BINARY_OP_ADD_INT=130, - BINARY_OP_ADD_UNICODE=131, - BINARY_OP_EXTEND=132, + ASYNC_CALL_BOUND_METHOD_EXACT_ARGS=129, + ASYNC_CALL_BOUND_METHOD_GENERAL=130, + ASYNC_CALL_KW_BOUND_METHOD=131, + ASYNC_CALL_KW_PY=132, + ASYNC_CALL_PY_EXACT_ARGS=133, + ASYNC_CALL_PY_GENERAL=134, + BINARY_OP_ADD_FLOAT=135, + BINARY_OP_ADD_INT=136, + BINARY_OP_ADD_UNICODE=137, + BINARY_OP_EXTEND=138, BINARY_OP_INPLACE_ADD_UNICODE=3, - BINARY_OP_MULTIPLY_FLOAT=133, - BINARY_OP_MULTIPLY_INT=134, - BINARY_OP_SUBSCR_DICT=135, - BINARY_OP_SUBSCR_GETITEM=136, - BINARY_OP_SUBSCR_LIST_INT=137, - BINARY_OP_SUBSCR_LIST_SLICE=138, - BINARY_OP_SUBSCR_STR_INT=139, - BINARY_OP_SUBSCR_TUPLE_INT=140, - BINARY_OP_SUBSCR_USTR_INT=141, - BINARY_OP_SUBTRACT_FLOAT=142, - BINARY_OP_SUBTRACT_INT=143, - CALL_ALLOC_AND_ENTER_INIT=144, - CALL_BOUND_METHOD_EXACT_ARGS=145, - CALL_BOUND_METHOD_GENERAL=146, - CALL_BUILTIN_CLASS=147, - CALL_BUILTIN_FAST=148, - CALL_BUILTIN_FAST_WITH_KEYWORDS=149, - CALL_BUILTIN_O=150, - CALL_EX_NON_PY_GENERAL=151, - CALL_EX_PY=152, - CALL_ISINSTANCE=153, - CALL_KW_BOUND_METHOD=154, - CALL_KW_NON_PY=155, - CALL_KW_PY=156, - CALL_LEN=157, - CALL_LIST_APPEND=158, - CALL_METHOD_DESCRIPTOR_FAST=159, - CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=160, - CALL_METHOD_DESCRIPTOR_NOARGS=161, - CALL_METHOD_DESCRIPTOR_O=162, - CALL_NON_PY_GENERAL=163, - CALL_PY_EXACT_ARGS=164, - CALL_PY_GENERAL=165, - CALL_STR_1=166, - CALL_TUPLE_1=167, - CALL_TYPE_1=168, - COMPARE_OP_FLOAT=169, - COMPARE_OP_INT=170, - COMPARE_OP_STR=171, - CONTAINS_OP_DICT=172, - CONTAINS_OP_SET=173, - FOR_ITER_GEN=174, - FOR_ITER_LIST=175, - FOR_ITER_RANGE=176, - FOR_ITER_TUPLE=177, - FOR_ITER_VIRTUAL=178, - GET_ITER_SELF=179, - GET_ITER_VIRTUAL=180, - JUMP_BACKWARD_JIT=181, - JUMP_BACKWARD_NO_JIT=182, - LOAD_ATTR_CLASS=183, - LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=184, - LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=185, - LOAD_ATTR_INSTANCE_VALUE=186, - LOAD_ATTR_METHOD_LAZY_DICT=187, - LOAD_ATTR_METHOD_NO_DICT=188, - LOAD_ATTR_METHOD_WITH_VALUES=189, - LOAD_ATTR_MODULE=190, - LOAD_ATTR_NONDESCRIPTOR_NO_DICT=191, - LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=192, - LOAD_ATTR_PROPERTY=193, - LOAD_ATTR_SLOT=194, - LOAD_ATTR_WITH_HINT=195, - LOAD_GLOBAL_BUILTIN=196, - LOAD_GLOBAL_MODULE=197, - LOAD_SUPER_ATTR_ATTR=198, - LOAD_SUPER_ATTR_METHOD=199, - RESUME_CHECK=200, - RESUME_CHECK_JIT=201, - SEND_ASYNC_GEN=202, - SEND_GEN=203, - SEND_VIRTUAL=204, - STORE_ATTR_INSTANCE_VALUE=205, - STORE_ATTR_SLOT=206, - STORE_ATTR_WITH_HINT=207, - STORE_SUBSCR_DICT=208, - STORE_SUBSCR_LIST_INT=209, - TO_BOOL_ALWAYS_TRUE=210, - TO_BOOL_BOOL=211, - TO_BOOL_INT=212, - TO_BOOL_LIST=213, - TO_BOOL_NONE=214, - TO_BOOL_STR=215, - UNPACK_SEQUENCE_LIST=216, - UNPACK_SEQUENCE_TUPLE=217, - UNPACK_SEQUENCE_TWO_TUPLE=218, + BINARY_OP_MULTIPLY_FLOAT=139, + BINARY_OP_MULTIPLY_INT=140, + BINARY_OP_SUBSCR_DICT=141, + BINARY_OP_SUBSCR_GETITEM=142, + BINARY_OP_SUBSCR_LIST_INT=143, + BINARY_OP_SUBSCR_LIST_SLICE=144, + BINARY_OP_SUBSCR_STR_INT=145, + BINARY_OP_SUBSCR_TUPLE_INT=146, + BINARY_OP_SUBSCR_USTR_INT=147, + BINARY_OP_SUBTRACT_FLOAT=148, + BINARY_OP_SUBTRACT_INT=149, + CALL_ALLOC_AND_ENTER_INIT=150, + CALL_BOUND_METHOD_EXACT_ARGS=151, + CALL_BOUND_METHOD_GENERAL=152, + CALL_BUILTIN_CLASS=153, + CALL_BUILTIN_FAST=154, + CALL_BUILTIN_FAST_WITH_KEYWORDS=155, + CALL_BUILTIN_O=156, + CALL_EX_NON_PY_GENERAL=157, + CALL_EX_PY=158, + CALL_ISINSTANCE=159, + CALL_KW_BOUND_METHOD=160, + CALL_KW_NON_PY=161, + CALL_KW_PY=162, + CALL_LEN=163, + CALL_LIST_APPEND=164, + CALL_METHOD_DESCRIPTOR_FAST=165, + CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS=166, + CALL_METHOD_DESCRIPTOR_NOARGS=167, + CALL_METHOD_DESCRIPTOR_O=168, + CALL_NON_PY_GENERAL=169, + CALL_PY_EXACT_ARGS=170, + CALL_PY_GENERAL=171, + CALL_STR_1=172, + CALL_TUPLE_1=173, + CALL_TYPE_1=174, + COMPARE_OP_FLOAT=175, + COMPARE_OP_INT=176, + COMPARE_OP_STR=177, + CONTAINS_OP_DICT=178, + CONTAINS_OP_SET=179, + FOR_ITER_GEN=180, + FOR_ITER_LIST=181, + FOR_ITER_RANGE=182, + FOR_ITER_TUPLE=183, + FOR_ITER_VIRTUAL=184, + GET_ITER_SELF=185, + GET_ITER_VIRTUAL=186, + JUMP_BACKWARD_JIT=187, + JUMP_BACKWARD_NO_JIT=188, + LOAD_ATTR_CLASS=189, + LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=190, + LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=191, + LOAD_ATTR_INSTANCE_VALUE=192, + LOAD_ATTR_METHOD_LAZY_DICT=193, + LOAD_ATTR_METHOD_NO_DICT=194, + LOAD_ATTR_METHOD_WITH_VALUES=195, + LOAD_ATTR_MODULE=196, + LOAD_ATTR_NONDESCRIPTOR_NO_DICT=197, + LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=198, + LOAD_ATTR_PROPERTY=199, + LOAD_ATTR_SLOT=200, + LOAD_ATTR_WITH_HINT=201, + LOAD_GLOBAL_BUILTIN=202, + LOAD_GLOBAL_MODULE=203, + LOAD_SUPER_ATTR_ATTR=204, + LOAD_SUPER_ATTR_METHOD=205, + RESUME_CHECK=206, + RESUME_CHECK_JIT=207, + SEND_ASYNC_GEN=208, + SEND_GEN=209, + SEND_VIRTUAL=210, + STORE_ATTR_INSTANCE_VALUE=211, + STORE_ATTR_SLOT=212, + STORE_ATTR_WITH_HINT=213, + STORE_SUBSCR_DICT=214, + STORE_SUBSCR_LIST_INT=215, + TO_BOOL_ALWAYS_TRUE=216, + TO_BOOL_BOOL=217, + TO_BOOL_INT=218, + TO_BOOL_LIST=219, + TO_BOOL_NONE=220, + TO_BOOL_STR=221, + UNPACK_SEQUENCE_LIST=222, + UNPACK_SEQUENCE_TUPLE=223, + UNPACK_SEQUENCE_TWO_TUPLE=224, ) opmap = frozendict( diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 88077aacecbf5b..320af175affe0f 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -5917,6 +5917,49 @@ def testfunc(n): # _POP_TOP_NOP is a sign the optimizer ran and didn't hit bottom. self.assertGreaterEqual(count_ops(ex, "_POP_TOP_NOP"), 1) + def test_async_call_gen_frame(self): + + async def inner(i): + return i + 1 + def make_closure(v): + async def inner(i): + return i + v + return inner + def make_outer(callee): + ns = {"callee": callee} + exec(textwrap.dedent(""" + async def outer(n): + total = 0 + for i in range(n): + total += await callee(i) + return total + """), ns) + return ns["outer"] + def run(coro): + try: + coro.send(None) + except StopIteration as ex: + return ex.value + + n = TIER2_THRESHOLD * 2 + for callee in inner, make_closure(1): + with self.subTest(callee=callee): + outer = make_outer(callee) + self.assertEqual(run(outer(n)), sum(range(1, n + 1))) + ex = get_first_executor(outer) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + # The coroutine is started without being returned to outer() + # first, and runs to its return within the trace. + self.assertIn("_ASYNC_CALL_GEN_FRAME", uops) + self.assertNotIn("_RETURN_GENERATOR", uops) + self.assertNotIn("_SEND_GEN_FRAME", uops) + self.assertIn("_RETURN_VALUE", uops) + self.assertIn("_END_SEND", uops) + # The guard is only needed in tier 1. + self.assertNotIn("_GUARD_ASYNC_CALL_TAIL", uops) + def test_send_virtual(self): def send_list(n): diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index c5900da09501b3..2e740066060bf9 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2397,6 +2397,360 @@ async def coro(): self.assertEqual(yielded, []) self.assertEqual(result, 42) +class AwaitCallTest(unittest.TestCase): + # Run each call often enough to exercise the specialized forms too. + N = 100 + + def test_function(self): + async def add(a, b=10): + return a + b + + async def caller(): + total = 0 + for i in range(self.N): + total += await add(i, 1) + total += await add(i) + return total + + expected = sum(i + 1 + i + 10 for i in range(self.N)) + self.assertEqual(run_async(caller()), ([], expected)) + + def test_method(self): + class A: + async def add(self, a, b=10): + return a + b + + async def caller(): + obj = A() + bound = obj.add + total = 0 + for i in range(self.N): + total += await obj.add(i, 1) + total += await obj.add(i) + total += await bound(i, 1) + total += await bound(i) + return total + + expected = sum(2 * (i + 1 + i + 10) for i in range(self.N)) + self.assertEqual(run_async(caller()), ([], expected)) + + def test_keyword_arguments(self): + async def add(a, b=10): + return a + b + + class A: + async def add(self, a, b=10): + return a + b + + async def caller(): + obj = A() + bound = obj.add + total = 0 + for i in range(self.N): + total += await add(i, b=1) + total += await add(a=i) + total += await obj.add(i, b=1) + total += await bound(b=1, a=i) + return total + + expected = sum(i + 1 + i + 10 + 2 * (i + 1) for i in range(self.N)) + self.assertEqual(run_async(caller()), ([], expected)) + + def test_closure(self): + # COPY_FREE_VARS and MAKE_CELL precede RETURN_GENERATOR + def outer(free): + async def inner(arg): + cell = arg + return (lambda: (cell, free, __class__))() + return inner + + async def caller(): + inner = outer("free") + for i in range(self.N): + self.assertEqual(await inner(i), (i, "free", AwaitCallTest)) + + self.assertEqual(run_async(caller()), ([], None)) + + def test_suspend(self): + class Suspend: + def __init__(self, value): + self.value = value + + def __await__(self): + return (yield self.value) + + async def inner(i): + sent = await Suspend(i) + return sent * 2 + + async def middle(i): + return await inner(i) + 1 + + async def outer(): + return [await middle(i) for i in range(self.N)] + + coro = outer() + value = coro.send(None) + for i in range(self.N): + self.assertEqual(value, i) + self.assertEqual(inspect.getcoroutinestate(coro), "CORO_SUSPENDED") + # outer() -> middle() -> inner() -> Suspend().__await__() + self.assertEqual(coro.cr_await.cr_await.cr_code, inner.__code__) + try: + value = coro.send(i + 100) + except StopIteration as ex: + self.assertEqual(i, self.N - 1) + result = ex.value + self.assertEqual(result, [(i + 100) * 2 + 1 for i in range(self.N)]) + + def test_async_with(self): + class CM: + async def __aenter__(self): + return "entered" + + async def __aexit__(self, *exc_info): + return exc_info[0] is ZeroDivisionError + + async def caller(): + cm = CM() + for i in range(self.N): + async with cm as value: + self.assertEqual(value, "entered") + 1 / 0 + + self.assertEqual(run_async(caller()), ([], None)) + + def test_generator_based_coroutine(self): + @types.coroutine + def gen_coro(value): + sent = yield value + return sent + 1 + + async def caller(): + return [await gen_coro(i) for i in range(self.N)] + + coro = caller() + value = coro.send(None) + for i in range(self.N): + self.assertEqual(value, i) + self.assertIs(coro.cr_await.gi_code, gen_coro.__code__) + try: + value = coro.send(i * 2) + except StopIteration as ex: + self.assertEqual(i, self.N - 1) + result = ex.value + self.assertEqual(result, [i * 2 + 1 for i in range(self.N)]) + + def test_throw_and_close(self): + async def inner(log): + try: + await AsyncYield("suspended") + except ZeroDivisionError: + log.append("caught") + return "handled" + finally: + log.append("finally") + + async def outer(log): + return await inner(log) + + for _ in range(self.N): + log = [] + coro = outer(log) + self.assertEqual(coro.send(None), "suspended") + with self.assertRaises(StopIteration) as cm: + coro.throw(ZeroDivisionError) + self.assertEqual(cm.exception.value, "handled") + self.assertEqual(log, ["caught", "finally"]) + + log = [] + coro = outer(log) + self.assertEqual(coro.send(None), "suspended") + coro.close() + self.assertEqual(log, ["finally"]) + + def test_exception(self): + async def fail(i): + raise ZeroDivisionError(i) + + async def caller(): + for i in range(self.N): + try: + await fail(i) + except ZeroDivisionError as ex: + self.assertEqual(ex.args, (i,)) + tb = traceback.extract_tb(ex.__traceback__) + self.assertEqual([entry.name for entry in tb], + ["caller", "fail"]) + self.assertEqual(tb[0].line, "await fail(i)") + else: + self.fail("ZeroDivisionError not raised") + + self.assertEqual(run_async(caller()), ([], None)) + + def test_wrong_arguments(self): + async def func(a, b=1): + return a + + async def caller(): + for i in range(self.N): + try: + await func() + except TypeError as ex: + self.assertIn("missing 1 required positional", str(ex)) + else: + self.fail("TypeError not raised") + + self.assertEqual(run_async(caller()), ([], None)) + + def test_not_a_coroutine_function(self): + async def coro(i): + return i + + def function(i): + return coro(i) + + class Awaitable: + def __init__(self, i): + self.i = i + + def __await__(self): + return self.i + yield + + @types.coroutine + def gen_coro(i): + return i + yield + + def gen(i): + yield + + async def agen(i): + yield + + async def caller(): + for i in range(self.N): + self.assertEqual(await function(i), i) + self.assertEqual(await Awaitable(i), i) + self.assertEqual(await gen_coro(i), i) + for func in int, gen, agen: + with self.assertRaisesRegex(TypeError, "can't be awaited"): + await func(i) + + self.assertEqual(run_async(caller()), ([], None)) + + def test_change_callable(self): + class Awaitable: + def __await__(self): + return 5 + yield + + async def coro(a=1): + return a + + async def coro2(a): + return a + 1 + + def function(a): + return coro(a + 2) + + class A: + async def method(self, a): + return a + 3 + + async def caller(func, expected): + self.assertEqual(await func(1), expected) + + obj = A() + callables = [(coro, 1), (coro2, 2), (function, 3), (obj.method, 4), + (lambda a: Awaitable(), 5)] + for _ in range(self.N): + for func, expected in callables: + with self.subTest(func=func): + self.assertEqual(run_async(caller(func, expected)), + ([], None)) + + def test_already_awaited(self): + async def inner(): + await AsyncYield(None) + + async def outer(coro): + await coro + + async def caller(coro): + return await outer(coro) + + for _ in range(self.N): + awaited = inner() + first = caller(awaited) + first.send(None) + second = caller(awaited) + with self.assertRaisesRegex(RuntimeError, "already"): + second.send(None) + first.close() + + def test_recursion(self): + async def recurse(n): + if n: + return await recurse(n - 1) + 1 + return 0 + + self.assertEqual(run_async(recurse(self.N)), ([], self.N)) + + async def infinite(): + await infinite() + + with support.infinite_recursion(), self.assertRaises(RecursionError): + run_async(infinite()) + + def test_origin_tracking(self): + async def get_origin(): + coro = sys._getframe().f_generator + return coro.cr_origin + + async def caller(): + origins = set() + for _ in range(self.N): + origins.add(await get_origin()) + return origins + + orig_depth = sys.get_coroutine_origin_tracking_depth() + self.addCleanup(sys.set_coroutine_origin_tracking_depth, orig_depth) + sys.set_coroutine_origin_tracking_depth(1) + lineno = caller.__code__.co_firstlineno + 3 + self.assertEqual(run_async(caller()), + ([], {((__file__, lineno, "caller"),)})) + sys.set_coroutine_origin_tracking_depth(0) + self.assertEqual(run_async(caller()), ([], {None})) + + def test_line_events(self): + # The GET_AWAITABLE that follows the CALL starts a new line here, + # so it must not be skipped when tracing lines. + async def inner(): + pass + + async def caller(): + for _ in range(self.N): + await ( + inner() + ) + + def trace(frame, event, arg): + if frame.f_code is caller.__code__ and event == "line": + lines.append(frame.f_lineno - first) + return trace + + first = caller.__code__.co_firstlineno + run_async(caller()) + lines = [] + self.addCleanup(sys.settrace, sys.gettrace()) + sys.settrace(trace) + run_async(caller()) + sys.settrace(None) + self.assertEqual(lines, [1] + [3, 2, 1] * self.N) + + @unittest.skipIf( support.is_emscripten or support.is_wasi, "asyncio does not work under Emscripten/WASI yet." diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 24cc51f4b4053a..689550f78bba5c 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -2573,6 +2573,49 @@ def f(a=1, b=2): self.assertEqual(call_data[0], (f, 1)) self.assertEqual(call_data[1], (f, sys.monitoring.MISSING)) + def test_async_call(self): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + + async def coro(a): + return a + + async def caller(n): + for i in range(n): + await coro(i) + + def run(coro): + with self.assertRaises(StopIteration): + coro.send(None) + + def call(code, offset, callable, arg0): + if code is caller.__code__: + calls.append((callable, arg0)) + + def start(code, offset): + if code is coro.__code__: + starts.append(code) + + n = _testinternalcapi.SPECIALIZATION_THRESHOLD + 1 + expected = [(range, n)] + [(coro, i) for i in range(n)] + # Specialize the CALL to ASYNC_CALL_* before it is instrumented. + run(caller(n)) + + calls = [] + starts = [] + sys.monitoring.use_tool_id(0, "test") + self.addCleanup(sys.monitoring.free_tool_id, 0) + sys.monitoring.set_events(0, 0) + sys.monitoring.register_callback(0, E.CALL, call) + sys.monitoring.register_callback(0, E.PY_START, start) + sys.monitoring.set_events(0, E.CALL | E.PY_START) + run(caller(n)) + sys.monitoring.set_events(0, 0) + self.assertEqual(calls, expected) + self.assertEqual(len(starts), n) + + run(caller(n)) + self.assertEqual(calls, expected) + def test_instruction_explicit_callback(self): # gh-122247 # Calling the instruction event callback explicitly should not diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 5f7c15f93a8332..99558b72c5bb73 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1630,8 +1630,15 @@ def run_async(coro): except StopIteration: break + @types.coroutine + def suspend(): + yield + class CM: async def __aenter__(self): + # ASYNC_CALL_* starts the coroutine in place of the SEND + # in send_with(), which is only executed to resume it. + await suspend() return self async def __aexit__(self, *exc): @@ -1693,6 +1700,156 @@ async def send_for(): self.assert_specialized(send_for, "SEND_ASYNC_GEN") self.assert_no_opcode(send_for, "SEND") + @cpython_only + @requires_specialization + def test_async_call(self): + def run_async(coro): + try: + coro.send(None) + except StopIteration as ex: + return ex.value + self.fail("coroutine did not finish") + + async def coro(a, b=1): + return a + b + + class C: + async def method(self, a, b=1): + return a + b + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + pass + + def function(a): + return coro(a) + + obj = C() + + async def py_exact_args(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await coro(i, 2), i + 2) + self.assertEqual(await obj.method(i, 2), i + 2) + + run_async(py_exact_args()) + self.assert_specialized(py_exact_args, "ASYNC_CALL_PY_EXACT_ARGS") + self.assert_no_opcode(py_exact_args, "CALL_PY_EXACT_ARGS") + + async def py_general(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await coro(i), i + 1) + + run_async(py_general()) + self.assert_specialized(py_general, "ASYNC_CALL_PY_GENERAL") + + async def bound_method_exact_args(): + bound = obj.method + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await bound(i, 2), i + 2) + + run_async(bound_method_exact_args()) + self.assert_specialized(bound_method_exact_args, + "ASYNC_CALL_BOUND_METHOD_EXACT_ARGS") + self.assert_no_opcode(bound_method_exact_args, + "CALL_BOUND_METHOD_EXACT_ARGS") + + async def bound_method_general(): + bound = obj.method + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await bound(i), i + 1) + + run_async(bound_method_general()) + self.assert_specialized(bound_method_general, + "ASYNC_CALL_BOUND_METHOD_GENERAL") + + async def kw_py(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await coro(i, b=2), i + 2) + self.assertEqual(await obj.method(i, b=2), i + 2) + + run_async(kw_py()) + self.assert_specialized(kw_py, "ASYNC_CALL_KW_PY") + self.assert_no_opcode(kw_py, "CALL_KW_PY") + + async def kw_bound_method(): + bound = obj.method + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await bound(i, b=2), i + 2) + + run_async(kw_bound_method()) + self.assert_specialized(kw_bound_method, "ASYNC_CALL_KW_BOUND_METHOD") + self.assert_no_opcode(kw_bound_method, "CALL_KW_BOUND_METHOD") + self.assert_no_opcode(bound_method_general, + "CALL_BOUND_METHOD_GENERAL") + + async def async_with(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + async with obj as value: + self.assertIs(value, obj) + + run_async(async_with()) + self.assert_specialized(async_with, "ASYNC_CALL_PY_EXACT_ARGS") + self.assert_specialized(async_with, "ASYNC_CALL_PY_GENERAL") + + async def not_a_coroutine_function(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + self.assertEqual(await function(i), i + 1) + + run_async(not_a_coroutine_function()) + self.assert_specialized(not_a_coroutine_function, "CALL_PY_EXACT_ARGS") + self.assert_no_opcode(not_a_coroutine_function, + "ASYNC_CALL_PY_EXACT_ARGS") + + async def not_awaited(): + for i in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + c = coro(i) + self.assertEqual(await c, i + 1) + + run_async(not_awaited()) + self.assert_specialized(not_awaited, "CALL_PY_GENERAL") + self.assert_no_opcode(not_awaited, "ASYNC_CALL_PY_GENERAL") + self.assert_specialized(not_awaited, "SEND_GEN") + + @cpython_only + @requires_specialization + def test_async_call_deopt(self): + def run_async(coro): + try: + coro.send(None) + except StopIteration as ex: + return ex.value + self.fail("coroutine did not finish") + + async def coro(a): + return a + 1 + + async def other(a): + return a + 2 + + def function(a): + return coro(a + 2) + + async def caller(func, n): + return [await func(i) for i in range(n)] + + n = _testinternalcapi.SPECIALIZATION_THRESHOLD + self.assertEqual(run_async(caller(coro, n)), list(range(1, n + 1))) + self.assert_specialized(caller, "ASYNC_CALL_PY_EXACT_ARGS") + + n = _testinternalcapi.SPECIALIZATION_COOLDOWN + self.assertEqual(run_async(caller(function, n)), list(range(3, n + 3))) + self.assert_no_opcode(caller, "ASYNC_CALL_PY_EXACT_ARGS") + + self.assertEqual(run_async(caller(other, n)), list(range(2, n + 2))) + self.assert_specialized(caller, "ASYNC_CALL_PY_EXACT_ARGS") + + # Changing the code of the function invalidates the specialization. + other.__code__ = (lambda a: a).__code__ + with self.assertRaisesRegex(TypeError, "can't be awaited"): + run_async(caller(other, n)) + @cpython_only @requires_specialization def test_store_attr_slot(self): diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a5e7bbbfad5ffc..672f1a333e9a9d 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2994,6 +2994,27 @@ def test_jump_or_pop(output): output.append(1) _ = output.append(2) and "Spam" output.append(3) + def test_no_jump_to_first_line_of_generator(self): + # The "def" line only owns the prologue of a generator frame + # (RETURN_GENERATOR), which must not run again. + def gen(): + output.append(1) + yield 1 + + output = [] + def trace(frame, event, arg): + if event == 'line' and frame.f_code is gen.__code__: + frame.f_lineno = gen.__code__.co_firstlineno + return trace + + sys.settrace(trace) + try: + with self.assertRaisesRegex(ValueError, "cannot find bytecode"): + list(gen()) + finally: + sys.settrace(None) + self.assertEqual(output, []) + class TestExtendedArgs(unittest.TestCase): diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 8e7ba1f69553dd..18752bf322fdc1 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -18,6 +18,1054 @@ /* BEGIN INSTRUCTIONS */ + TARGET(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_BOUND_METHOD_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW_BOUND_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(callable_s); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(BINARY_OP) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP; @@ -6431,10 +7479,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME @@ -11440,34 +12485,34 @@ next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + frame->instr_ptr++; + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); + gen->gi_frame_state = FRAME_CREATED; + _Py_LeaveRecursiveCallPy(tstate); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -11721,11 +12766,8 @@ STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index 91b424773224e4..b179dfdd6a4897 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -129,6 +129,12 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&TARGET_RESUME, + &&TARGET_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_ASYNC_CALL_BOUND_METHOD_GENERAL, + &&TARGET_ASYNC_CALL_KW_BOUND_METHOD, + &&TARGET_ASYNC_CALL_KW_PY, + &&TARGET_ASYNC_CALL_PY_EXACT_ARGS, + &&TARGET_ASYNC_CALL_PY_GENERAL, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, @@ -227,12 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -478,12 +478,12 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -531,6 +531,12 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PA static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_PY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_GENERAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); @@ -774,6 +780,12 @@ JUMP_TO_LABEL(error); } static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_ASYNC_CALL_KW_PY, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_ASYNC_CALL_PY_GENERAL, [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -1016,12 +1028,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1032,6 +1038,12 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, @@ -1274,12 +1286,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 475be76dc3da21..fdef52383cbd89 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1840,7 +1840,9 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value) int64_t start_stack = stacks[_PyInterpreterFrame_LASTI(self->f_frame)]; int err = -1; const char *msg = "cannot find bytecode for specified line"; - for (int i = 0; i < len; i++) { + /* The instructions before _co_firsttraceable set up a generator + * frame and must not run again once the frame belongs to one. */ + for (int i = code->_co_firsttraceable; i < len; i++) { if (lines[i] == new_lineno) { int64_t target_stack = stacks[i]; if (compatible_stack(start_stack, target_stack)) { diff --git a/Objects/genobject.c b/Objects/genobject.c index 82a1bb1ff42068..f5dd4501b70180 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1128,7 +1128,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) } PyObject * -_Py_MakeCoro(PyFunctionObject *func) +_Py_MakeCoroWithFrame(PyFunctionObject *func, _PyInterpreterFrame *frame) { int coro_flags = ((PyCodeObject *)func->func_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); @@ -1160,7 +1160,6 @@ _Py_MakeCoro(PyFunctionObject *func) if (origin_depth == 0) { ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL; } else { - _PyInterpreterFrame *frame = tstate->current_frame; assert(frame); assert(_PyFrame_IsIncomplete(frame)); frame = _PyFrame_GetFirstComplete(frame->previous); @@ -1174,6 +1173,12 @@ _Py_MakeCoro(PyFunctionObject *func) return coro; } +PyObject * +_Py_MakeCoro(PyFunctionObject *func) +{ + return _Py_MakeCoroWithFrame(func, _PyThreadState_GET()->current_frame); +} + static PyObject * gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, PyObject *name, PyObject *qualname) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 42ad83daa1ea46..4fa3f6b6b7394e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1730,11 +1730,8 @@ dummy_func( _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); DEAD(v); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)(INSTRUCTION_SIZE + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } @@ -4078,11 +4075,8 @@ dummy_func( STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; // oparg is the return offset from the next instruction. - frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)(INSTRUCTION_SIZE + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } @@ -4314,6 +4308,10 @@ dummy_func( CALL_PY_GENERAL, CALL_BOUND_METHOD_GENERAL, CALL_NON_PY_GENERAL, + ASYNC_CALL_PY_EXACT_ARGS, + ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + ASYNC_CALL_PY_GENERAL, + ASYNC_CALL_BOUND_METHOD_GENERAL, }; specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { @@ -5345,11 +5343,125 @@ dummy_func( POP_TOP + _CHECK_PERIODIC_AT_END; + // The ASYNC_CALL_* instructions are specializations of CALL and + // CALL_KW for + // calls to coroutine functions that are awaited at once, as in + // `await f(...)`. They are superinstructions: they merge the CALL + // with the first execution of the instructions that follow it, + // GET_AWAITABLE; PUSH_NULL; LOAD_COMMON_CONSTANT None; SEND + // and start the new coroutine without returning it to the caller + // first. They are built from the uops of the matching CALL_* + // specialization, with _ASYNC_CALL_GEN_FRAME in place of + // _SAVE_RETURN_OFFSET. + + // ASYNC_CALL_* skip the instructions up to and including SEND, so + // none of them may be instrumented. Only the first and the last of + // them can start a line. + // Executors are invalidated when the code is instrumented. + op(_GUARD_ASYNC_CALL_TAIL, (--)) { + #if TIER_ONE + DEOPT_IF(next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE); + DEOPT_IF(next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE); + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + + // Turns the frame of a call to a coroutine function into a running + // coroutine: the equivalent of executing RETURN_GENERATOR in that + // frame, followed by GET_AWAITABLE, PUSH_NULL, LOAD_COMMON_CONSTANT + // None and SEND_GEN in this one. + op(_ASYNC_CALL_GEN_FRAME, (new_frame -- coro, null, gen_frame)) { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + DEAD(new_frame); + SYNC_SP(); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + if (gen == NULL) { + _PyEval_FrameClearAndPop(tstate, callee); + ERROR_NO_POP(); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + // Otherwise RETURN_GENERATOR is preceded by COPY_FREE_VARS or + // MAKE_CELL. They run in the frame of the coroutine, where + // RETURN_GENERATOR only pushes the None that POP_TOP discards. + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + // The coroutine yields and returns to the SEND of the await. + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + INSTRUCTION_SIZE); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + + macro(ASYNC_CALL_PY_EXACT_ARGS) = + _RECORD_CALLABLE + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_FUNCTION_VERSION + + _CHECK_FUNCTION_EXACT_ARGS + + _CHECK_STACK_SPACE + + _CHECK_RECURSION_REMAINING + + _INIT_CALL_PY_EXACT_ARGS + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS) = + _RECORD_BOUND_METHOD + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + + _INIT_CALL_BOUND_METHOD_EXACT_ARGS + + flush + // In case the following deopt + _CHECK_FUNCTION_VERSION + + _CHECK_FUNCTION_EXACT_ARGS + + _CHECK_STACK_SPACE + + _CHECK_RECURSION_REMAINING + + _INIT_CALL_PY_EXACT_ARGS + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_PY_GENERAL) = + _RECORD_CALLABLE + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_FUNCTION_VERSION + + _CHECK_RECURSION_REMAINING + + _PY_FRAME_GENERAL + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_BOUND_METHOD_GENERAL) = + _RECORD_BOUND_METHOD + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_METHOD_VERSION + + _EXPAND_METHOD + + flush + // so that self is in the argument array + _CHECK_RECURSION_REMAINING + + _PY_FRAME_GENERAL + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + // Cache layout: counter/1, func_version/2 family(CALL_KW, INLINE_CACHE_ENTRIES_CALL_KW) = { CALL_KW_BOUND_METHOD, CALL_KW_PY, CALL_KW_NON_PY, + ASYNC_CALL_KW_BOUND_METHOD, + ASYNC_CALL_KW_PY, }; op(_MONITOR_CALL_KW, (callable, self_or_null, args[oparg], unused -- callable, self_or_null, args[oparg], unused)) { @@ -5517,6 +5629,29 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; + macro(ASYNC_CALL_KW_PY) = + _RECORD_CALLABLE_KW + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_FUNCTION_VERSION_KW + + _CHECK_RECURSION_REMAINING + + _PY_FRAME_KW + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + + macro(ASYNC_CALL_KW_BOUND_METHOD) = + _RECORD_CALLABLE_KW + + unused/1 + + _CHECK_PEP_523 + + _GUARD_ASYNC_CALL_TAIL + + _CHECK_METHOD_VERSION_KW + + _EXPAND_METHOD_KW + + flush + // so that self is in the argument array + _PY_FRAME_KW + + _ASYNC_CALL_GEN_FRAME + + _PUSH_FRAME; + specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable, self_or_null, unused[oparg], unused -- callable, self_or_null, unused[oparg], unused)) { #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { @@ -5804,27 +5939,28 @@ dummy_func( } inst(RETURN_GENERATOR, (-- res)) { - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - ERROR_IF(gen == NULL); - assert(STACK_LEVEL() <= 2); - SAVE_STACK(); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - RELOAD_STACK(); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + // Handled by ASYNC_CALL_* already. + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + ERROR_IF(gen == NULL); + assert(STACK_LEVEL() <= 2); + SAVE_STACK(); + frame->instr_ptr++; + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); + gen->gi_frame_state = FRAME_CREATED; + _Py_LeaveRecursiveCallPy(tstate); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + RELOAD_STACK(); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + } } inst(BUILD_SLICE, (args[oparg] -- slice)) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 36921e35533433..f36a53ad614ea1 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -594,6 +594,32 @@ gen_try_set_executing(PyGenObject *gen) return false; } +// Move the frame of a call to a generator function into the generator +// and pop it from the thread's data stack. `prev` is the frame that +// takes its place. +static inline void +gen_take_frame(PyThreadState *tstate, PyGenObject *gen, + _PyInterpreterFrame *frame, _PyInterpreterFrame *prev) +{ + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); + _PyThreadState_PopFrame(tstate, frame); +} + +// Link the frame of a generator to the frame that is about to enter it. +static inline void +gen_link_frame(PyThreadState *tstate, PyGenObject *gen, + _PyInterpreterFrame *frame, uint16_t return_offset) +{ + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + gen->gi_iframe.previous = frame; + frame->return_offset = return_offset; +} + // Macro for inplace float binary ops (tier 2 only). // Mutates the uniquely-referenced TARGET operand in place. // TARGET must be either left or right. diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bc85ccd9ddb4fc..1ee676c07903c4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8940,6 +8940,123 @@ break; } + case _SEND_GEN_FRAME_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + _PyStackRef receiver; + _PyStackRef gen_frame; + oparg = CURRENT_OPARG(); + v = stack_pointer[-1]; + receiver = stack_pointer[-3]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + assert( 2u + oparg <= UINT16_MAX); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _SEND_GEN_FRAME_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + _PyStackRef receiver; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = v; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = v; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + assert( 2u + oparg <= UINT16_MAX); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _SEND_GEN_FRAME_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + _PyStackRef receiver; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + v = _stack_item_1; + receiver = stack_pointer[-1]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = v; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!gen_try_set_executing((PyGenObject *)gen)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = v; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + assert( 2u + oparg <= UINT16_MAX); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); + gen_frame = PyStackRef_Wrap(pushed_frame); + _tos_cache2 = gen_frame; + _tos_cache1 = _stack_item_0; + _tos_cache0 = receiver; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _SEND_GEN_FRAME_r33: { CHECK_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -8972,11 +9089,8 @@ STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = _stack_item_1; @@ -16203,10 +16317,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = stack_pointer[-1]; @@ -16242,10 +16353,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = _stack_item_0; @@ -16284,10 +16392,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); _tos_cache2 = gen_frame; _tos_cache1 = _stack_item_1; @@ -19918,6 +20023,161 @@ break; } + case _GUARD_ASYNC_CALL_TAIL_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + SET_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_ASYNC_CALL_TAIL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_ASYNC_CALL_TAIL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_ASYNC_CALL_TAIL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _ASYNC_CALL_GEN_FRAME_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + _PyStackRef _stack_item_0 = _tos_cache0; + new_frame = _stack_item_0; + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + _tos_cache2 = gen_frame; + _tos_cache1 = null; + _tos_cache0 = coro; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ case _MAYBE_EXPAND_METHOD_KW_r11: { @@ -20718,35 +20978,35 @@ CHECK_CURRENT_CACHED_VALUES(0); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + frame->instr_ptr++; + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); + gen->gi_frame_state = FRAME_CREATED; + _Py_LeaveRecursiveCallPy(tstate); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); _tos_cache0 = res; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ee9b64107bb9f2..916f2a6fc46705 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -18,6 +18,1054 @@ /* BEGIN INSTRUCTIONS */ + TARGET(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_BOUND_METHOD_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(temp); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW_BOUND_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(callable_s); + _PyFrame_StackPointerInvalidate(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_KW_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(kwnames); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + + TARGET(ASYNC_CALL_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = ASYNC_CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(ASYNC_CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + _PyStackRef coro; + _PyStackRef null; + _PyStackRef gen_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (IS_PEP523_HOOKED(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_ASYNC_CALL_TAIL + { + #if TIER_ONE + if (next_instr[0].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (next_instr[3].op.code >= MIN_INSTRUMENTED_OPCODE) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(_PyCode_AwaitedCallSend(next_instr) != NULL); + #endif + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _ASYNC_CALL_GEN_FRAME + { + _PyInterpreterFrame *callee = PyStackRef_Unwrap(new_frame); + assert(PyStackRef_FunctionCheck(callee->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(callee->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoroWithFrame(func, callee); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_FrameClearAndPop(tstate, callee); + _PyFrame_StackPointerInvalidate(frame); + JUMP_TO_LABEL(error); + } + assert(Py_TYPE(gen) == &PyCoro_Type); + if (callee->instr_ptr->op.code == RETURN_GENERATOR) { + assert(callee->instr_ptr[1].op.code == POP_TOP); + callee->instr_ptr += 2; + } + gen_take_frame(tstate, gen, callee, frame); + gen->gi_frame_state = FRAME_EXECUTING; + _Py_CODEUNIT *send = _PyCode_AwaitedCallSend(frame->instr_ptr + 4u ); + assert(send != NULL); + frame->instr_ptr = send; + gen_link_frame(tstate, gen, frame, + (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + send->op.arg)); + coro = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + null = PyStackRef_NULL; + gen_frame = PyStackRef_Wrap(&gen->gi_iframe); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(!IS_PEP523_HOOKED(tstate)); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer[0] = coro; + stack_pointer[1] = null; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(BINARY_OP) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP; @@ -6431,10 +7479,7 @@ STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME @@ -11437,34 +12482,34 @@ next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - _PyFrame_StackPointerInvalidate(frame); - if (gen == NULL) { - JUMP_TO_LABEL(error); + if (frame->owner == FRAME_OWNED_BY_GENERATOR) { + res = PyStackRef_None; + } + else { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + _PyFrame_StackPointerInvalidate(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() <= 2); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + frame->instr_ptr++; + _PyInterpreterFrame *prev = frame->previous; + gen_take_frame(tstate, gen, frame, prev); + gen->gi_frame_state = FRAME_CREATED; + _Py_LeaveRecursiveCallPy(tstate); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); } - assert(STACK_LEVEL() <= 2); - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - stack_pointer = _PyFrame_GetStackPointer(frame); - _PyFrame_StackPointerInvalidate(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -11718,11 +12763,8 @@ STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; + gen_link_frame(tstate, gen, frame, (uint16_t)( 2u + oparg)); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 91b424773224e4..b179dfdd6a4897 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -129,6 +129,12 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&TARGET_RESUME, + &&TARGET_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_ASYNC_CALL_BOUND_METHOD_GENERAL, + &&TARGET_ASYNC_CALL_KW_BOUND_METHOD, + &&TARGET_ASYNC_CALL_KW_PY, + &&TARGET_ASYNC_CALL_PY_EXACT_ARGS, + &&TARGET_ASYNC_CALL_PY_GENERAL, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, @@ -227,12 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -478,12 +478,12 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -531,6 +531,12 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PA static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_KW_PY(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_ASYNC_CALL_PY_GENERAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); @@ -774,6 +780,12 @@ JUMP_TO_LABEL(error); } static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_EXACT_ARGS, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_ASYNC_CALL_BOUND_METHOD_GENERAL, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_ASYNC_CALL_KW_BOUND_METHOD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_ASYNC_CALL_KW_PY, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_ASYNC_CALL_PY_EXACT_ARGS, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_ASYNC_CALL_PY_GENERAL, [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -1016,12 +1028,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1032,6 +1038,12 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [232] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [ASYNC_CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, @@ -1274,12 +1286,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [219] = _TAIL_CALL_UNKNOWN_OPCODE, - [220] = _TAIL_CALL_UNKNOWN_OPCODE, - [221] = _TAIL_CALL_UNKNOWN_OPCODE, - [222] = _TAIL_CALL_UNKNOWN_OPCODE, - [223] = _TAIL_CALL_UNKNOWN_OPCODE, - [224] = _TAIL_CALL_UNKNOWN_OPCODE, [225] = _TAIL_CALL_UNKNOWN_OPCODE, [226] = _TAIL_CALL_UNKNOWN_OPCODE, [227] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer.c b/Python/optimizer.c index e05adb344c8d06..55675b7ef4acab 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -744,7 +744,11 @@ _PyJit_translate_single_bytecode_to_trace( assert(!OPCODE_HAS_DEOPT(opcode)); } - bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode); + // RETURN_GENERATOR in a coroutine that ASYNC_CALL_* already started + // stays in its frame: it only pushes None (see _ASYNC_CALL_GEN_FRAME). + bool coro_running = opcode == RETURN_GENERATOR && + frame == tracer->prev_state.instr_frame; + bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode) && !coro_running; if (has_dynamic_jump_taken && !needs_guard_ip) { DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]); goto unsupported; @@ -908,6 +912,12 @@ _PyJit_translate_single_bytecode_to_trace( * start with RESUME_CHECK */ ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); break; + case RETURN_GENERATOR: + if (coro_running) { + ADD_TO_TRACE(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_None, target); + break; + } + _Py_FALLTHROUGH; default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6284885c428832..9bc41b022189d7 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1528,6 +1528,18 @@ dummy_func(void) { gen_frame = PyJitRef_WrapInvalid(new_frame); } + op(_GUARD_ASYNC_CALL_TAIL, (--)) { + // Only needed in tier 1: executors are invalidated + // when the code is instrumented. + ADD_OP(_NOP, 0, 0); + } + + op(_ASYNC_CALL_GEN_FRAME, (new_frame -- coro, null, gen_frame)) { + coro = sym_new_type(ctx, &PyCoro_Type); + null = sym_new_null(ctx); + gen_frame = new_frame; + } + op(_SEND_GEN_FRAME, (receiver, null, v -- receiver, null, gen_frame)) { _Py_UOpsAbstractFrame *new_frame = frame_new_from_symbol(ctx, receiver, NULL, 0); if (new_frame == NULL) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9cd20c495db441..0d36e5ac09cc8f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -5062,6 +5062,29 @@ break; } + case _GUARD_ASYNC_CALL_TAIL: { + ADD_OP(_NOP, 0, 0); + break; + } + + case _ASYNC_CALL_GEN_FRAME: { + JitOptRef new_frame; + JitOptRef coro; + JitOptRef null; + JitOptRef gen_frame; + new_frame = stack_pointer[-1]; + coro = sym_new_type(ctx, &PyCoro_Type); + null = sym_new_null(ctx); + gen_frame = new_frame; + CHECK_STACK_BOUNDS(2); + stack_pointer[-1] = coro; + stack_pointer[0] = null; + stack_pointer[1] = gen_frame; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 */ case _MAYBE_EXPAND_METHOD_KW: { diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 98abe3d0505e20..b3fafc68c54638 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -190,8 +190,14 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_METHOD_DESCRIPTOR_NOARGS] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_METHOD_DESCRIPTOR_FAST] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_PY_EXACT_ARGS] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_PY_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = {1, {_RECORD_CALLABLE_INDEX}}, [CALL_KW_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW_BOUND_METHOD] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [ASYNC_CALL_KW_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, + [ASYNC_CALL_KW_BOUND_METHOD] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_KW_NON_PY] = {1, {_RECORD_CALLABLE_KW_INDEX}}, [CALL_FUNCTION_EX] = {1, {_RECORD_4OS_INDEX}}, @@ -241,8 +247,14 @@ const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = { [CALL_METHOD_DESCRIPTOR_O] = {1, 0, {0}}, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = {1, 0, {0}}, [CALL_METHOD_DESCRIPTOR_NOARGS] = {1, 0, {0}}, + [ASYNC_CALL_PY_EXACT_ARGS] = {1, 0, {0}}, + [ASYNC_CALL_BOUND_METHOD_EXACT_ARGS] = {1, 1, {0}}, + [ASYNC_CALL_PY_GENERAL] = {1, 0, {0}}, + [ASYNC_CALL_BOUND_METHOD_GENERAL] = {1, 1, {0}}, [CALL_KW_PY] = {1, 0, {0}}, [CALL_KW_BOUND_METHOD] = {1, 0, {0}}, + [ASYNC_CALL_KW_PY] = {1, 0, {0}}, + [ASYNC_CALL_KW_BOUND_METHOD] = {1, 0, {0}}, [CALL_EX_PY] = {1, 0, {0}}, [BINARY_OP] = {2, 2, {1, 0}}, }; diff --git a/Python/specialize.c b/Python/specialize.c index 067bae59f6073b..7a87f989d9b5f0 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1751,6 +1751,22 @@ specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, return 0; } +/* Is the result of the CALL or CALL_KW at "instr" awaited at once, so + * that the call to a coroutine function can be specialized to one of the + * ASYNC_CALL_* instructions? These start the new coroutine in place of + * the SEND that awaits it. GET_AWAITABLE never fails for a new coroutine. */ +static bool +call_is_awaited(PyCodeObject *code, _Py_CODEUNIT *instr) +{ + int coro_flags = code->co_flags & + (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR); + if (coro_flags != CO_COROUTINE) { + return false; + } + _Py_CODEUNIT *next = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[instr->op.code]]; + return _PyCode_AwaitedCallSend(next) != NULL; +} + static int specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, bool bound_method) @@ -1781,10 +1797,21 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return -1; } write_u32(cache->func_version, version); + bool awaited = call_is_awaited(code, instr); uint8_t opcode; if (argcount == nargs + bound_method) { - opcode = - bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; + if (awaited) { + opcode = bound_method ? + ASYNC_CALL_BOUND_METHOD_EXACT_ARGS : ASYNC_CALL_PY_EXACT_ARGS; + } + else { + opcode = bound_method ? + CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; + } + } + else if (awaited) { + opcode = bound_method ? + ASYNC_CALL_BOUND_METHOD_GENERAL : ASYNC_CALL_PY_GENERAL; } else { opcode = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL; @@ -1820,7 +1847,14 @@ specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return -1; } write_u32(cache->func_version, version); - specialize(instr, bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY); + uint8_t opcode; + if (call_is_awaited(code, instr)) { + opcode = bound_method ? ASYNC_CALL_KW_BOUND_METHOD : ASYNC_CALL_KW_PY; + } + else { + opcode = bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY; + } + specialize(instr, opcode); return 0; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index c3e0b0dbff2199..01d0ecfb5a47ab 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -603,6 +603,9 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyFunction_SetVersion", "_PyGen_GetGeneratorFromFrame", "gen_try_set_executing", + "gen_link_frame", + "_PyCode_AwaitedCallSend", + "gen_take_frame", "_PyInterpreterState_GET", "_PyList_AppendTakeRef", "_PyList_ITEMS",