diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 703d7ec61ebab3..6429cf8b99a9bf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -157,6 +157,7 @@ dummy_func( if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { int err = _Py_HandlePending(tstate); ERROR_IF(err != 0, error); + LOAD_NEXT_OP_F(); } } @@ -167,6 +168,7 @@ dummy_func( if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { int err = _Py_HandlePending(tstate); ERROR_IF(err != 0, error); + LOAD_NEXT_OP_F(); } } } @@ -189,6 +191,7 @@ dummy_func( ERROR_NO_POP(); } next_instr = this_instr; + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -207,6 +210,7 @@ dummy_func( // Make sure this_instr gets reset correctley for any uops that // follow next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); DISPATCH(); } #endif @@ -241,6 +245,7 @@ dummy_func( /* Instrumentation has jumped */ next_instr = frame->instr_ptr; } + LOAD_NEXT_OP_F(); // maybe needs to be unconditional? } macro(INSTRUMENTED_RESUME) = @@ -362,6 +367,7 @@ dummy_func( pure inst(POP_TOP, (value --)) { DECREF_INPUTS(); + LOAD_NEXT_OP_F(); } pure inst(PUSH_NULL, (-- res)) { @@ -835,6 +841,7 @@ dummy_func( } else { err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + LOAD_NEXT_OP_F(); Py_DECREF(slice); } DECREF_INPUTS(); @@ -991,6 +998,7 @@ dummy_func( op(_STORE_SUBSCR, (v, container, sub -- )) { /* container[sub] = v */ int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + LOAD_NEXT_OP_F(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1234,6 +1242,7 @@ dummy_func( } PyStackRef_CLOSE(v); retval = PyStackRef_FromPyObjectSteal(retval_o); + LOAD_NEXT_OP_F(); } macro(SEND) = _SPECIALIZE_SEND + _SEND; @@ -1304,6 +1313,7 @@ dummy_func( } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -1418,6 +1428,7 @@ dummy_func( } else { err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + LOAD_NEXT_OP_F(); } DECREF_INPUTS(); ERROR_IF(err, error); @@ -1541,6 +1552,8 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + // SetAttr can call into specialize + LOAD_NEXT_OP_F(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1759,6 +1772,7 @@ dummy_func( _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; PyStackRef_XCLOSE(tmp); + LOAD_NEXT_OP_F(); } inst(MAKE_CELL, (--)) { @@ -1950,6 +1964,7 @@ dummy_func( ERROR_IF(ann_dict == NULL, error); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + LOAD_NEXT_OP_F(); Py_DECREF(ann_dict); ERROR_IF(err, error); } @@ -2180,6 +2195,7 @@ dummy_func( else { /* Classic, pushes one value. */ attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + LOAD_NEXT_OP_F(); DECREF_INPUTS(); ERROR_IF(attr_o == NULL, error); } @@ -2859,6 +2875,7 @@ dummy_func( * involving _RESUME_CHECK */ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; + LOAD_NEXT_OP_F_OPCODE(); oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { @@ -2968,6 +2985,7 @@ dummy_func( inst(GET_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + LOAD_NEXT_OP_F(); DECREF_INPUTS(); ERROR_IF(iter_o == NULL, error); iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -3021,7 +3039,7 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_ForIter(iter, next_instr, oparg); + _Py_Specialize_ForIter(iter, next_instr, oparg); // <- this can modify next_instr! DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -3316,6 +3334,7 @@ dummy_func( (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } pseudo(SETUP_FINALLY, (-- unused), (HAS_ARG)) = { @@ -3574,6 +3593,8 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); + // when vectorcall calls into eg sys_settrace, our code can be modified + LOAD_NEXT_OP_F(); } op(_MONITOR_CALL, (func[1], maybe_self[1], args[oparg] -- func[1], maybe_self[1], args[oparg])) { @@ -3710,6 +3731,7 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } macro(CALL_NON_PY_GENERAL) = @@ -4026,6 +4048,7 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } macro(CALL_BUILTIN_FAST) = @@ -4303,6 +4326,7 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } macro(CALL_METHOD_DESCRIPTOR_FAST) = @@ -4667,6 +4691,7 @@ dummy_func( PyStackRef_CLOSE(func_st); ERROR_IF(result_o == NULL, error); result = PyStackRef_FromPyObjectSteal(result_o); + LOAD_NEXT_OP_F(); } macro(CALL_FUNCTION_EX) = @@ -4828,6 +4853,7 @@ dummy_func( } next_instr = frame->instr_ptr; if (next_instr != this_instr) { + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -4838,6 +4864,7 @@ dummy_func( PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; + LOAD_NEXT_OP_F_OPCODE(); DISPATCH_GOTO(); } @@ -4851,6 +4878,7 @@ dummy_func( } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; + LOAD_NEXT_OP_F_OPCODE(); DISPATCH_GOTO(); } @@ -4917,6 +4945,7 @@ dummy_func( tier1 inst(EXTENDED_ARG, ( -- )) { assert(oparg); opcode = next_instr->op.code; + LOAD_NEXT_OP_F_OPCODE(); oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); @@ -5238,6 +5267,7 @@ dummy_func( #ifdef Py_TAIL_CALL_INTERP int opcode; #endif + py_tail_call_funcptr next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; DISPATCH(); } @@ -5280,6 +5310,7 @@ dummy_func( #ifdef Py_TAIL_CALL_INTERP int opcode; #endif + py_tail_call_funcptr next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; DISPATCH(); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 0a4f65feb3b512..34cf50d12652ab 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -73,7 +73,28 @@ #define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg #define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg +// 1st version looks like this where we load directly +// next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; +// 2nd version is like NEXTOPARG which does this atomic thing +// we also need a version that loads from opcode directly b/c a few things write to it directly + + /*if (next_op_f != INSTRUCTION_TABLE[opcode]) { \*/ + /* int myopcode=-1; \*/ + /* for (int i = 0; i < 256; i++) { \*/ + /* if (next_op_f == INSTRUCTION_TABLE[i]) { myopcode = i; break; } \*/ + /* } \*/ + /* printf("yo expected %p but got %p used opcode=%d but is really opcode=%d\n", INSTRUCTION_TABLE[opcode], next_op_f, myopcode, opcode); \*/ + /*} \*/ #ifdef Py_TAIL_CALL_INTERP +# define LOAD_NEXT_OP_F() \ + do { \ + _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ + next_op_f = INSTRUCTION_TABLE[word.op.code]; \ + } while (0) +# define LOAD_NEXT_OP_F_OPCODE() \ + do { \ + next_op_f = INSTRUCTION_TABLE[opcode]; \ + } while (0) // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. # define Py_MUSTTAIL [[clang::musttail]] # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) @@ -82,7 +103,8 @@ # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) # define DISPATCH_GOTO() \ do { \ - Py_MUSTTAIL return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ + assert(next_op_f == INSTRUCTION_TABLE[opcode]); \ + Py_MUSTTAIL return next_op_f(TAIL_CALL_ARGS); \ } while (0) # define JUMP_TO_LABEL(name) \ do { \ @@ -144,9 +166,11 @@ do { \ DISPATCH_GOTO(); \ } +// TODO conditional on tail call #define DISPATCH_SAME_OPARG() \ { \ opcode = next_instr->op.code; \ + next_op_f = INSTRUCTION_TABLE[opcode]; \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ } @@ -189,8 +213,8 @@ GETITEM(PyObject *v, Py_ssize_t i) { * for advancing to the next instruction, taking into account cache entries * and skipped instructions. */ -#define JUMPBY(x) (next_instr += (x)) -#define SKIP_OVER(x) (next_instr += (x)) +#define JUMPBY(x) (next_instr += (x)); LOAD_NEXT_OP_F() +#define SKIP_OVER(x) (next_instr += (x)); LOAD_NEXT_OP_F() /* Stack manipulation macros */ @@ -349,6 +373,7 @@ do { \ JUMP_TO_LABEL(error); \ } \ } \ + LOAD_NEXT_OP_F(); \ } while (0); @@ -368,6 +393,7 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { #define LOAD_IP(OFFSET) do { \ next_instr = frame->instr_ptr + (OFFSET); \ + LOAD_NEXT_OP_F(); \ } while (0) /* There's no STORE_IP(), it's inlined by the code generator. */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f02e13f5e3fbce..26e4596c9c6995 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -22,9 +22,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP); PREDICTED_BINARY_OP:; _Py_CODEUNIT* const this_instr = next_instr - 6; @@ -78,11 +80,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_ADD_FLOAT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -132,11 +136,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_ADD_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -185,11 +191,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_ADD_UNICODE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -238,11 +246,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_EXTEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_EXTEND); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -294,11 +304,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_INPLACE_ADD_UNICODE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -377,11 +389,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_MULTIPLY_FLOAT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -431,11 +445,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_MULTIPLY_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -484,11 +500,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBSCR_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef dict_st; @@ -531,11 +549,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBSCR_GETITEM; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef container; @@ -616,11 +636,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBSCR_LIST_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef list_st; @@ -686,11 +708,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBSCR_STR_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef str_st; @@ -748,11 +772,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBSCR_TUPLE_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef tuple_st; @@ -806,11 +832,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBTRACT_FLOAT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -860,11 +888,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBTRACT_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -913,9 +943,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_SLICE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_SLICE); _PyStackRef container; _PyStackRef start; @@ -973,9 +1005,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_LIST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_LIST); _PyStackRef *values; _PyStackRef list; @@ -995,9 +1029,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_MAP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_MAP); _PyStackRef *values; _PyStackRef map; @@ -1037,9 +1073,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_SET; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_SET); _PyStackRef *values; _PyStackRef set; @@ -1085,9 +1123,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_SLICE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_SLICE); _PyStackRef *args; _PyStackRef slice; @@ -1115,9 +1155,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_STRING; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_STRING); _PyStackRef *pieces; _PyStackRef str; @@ -1152,9 +1194,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_TUPLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_TUPLE); _PyStackRef *values; _PyStackRef tup; @@ -1174,9 +1218,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CACHE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CACHE); assert(0 && "Executing a cache."); Py_FatalError("Executing a cache."); @@ -1187,9 +1233,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL); PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; @@ -1330,6 +1378,8 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + // when vectorcall calls into eg sys_settrace, our code can be modified + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -1345,6 +1395,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1359,11 +1410,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_ALLOC_AND_ENTER_INIT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1487,11 +1540,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BOUND_METHOD_EXACT_ARGS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1626,11 +1681,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BOUND_METHOD_GENERAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1748,11 +1805,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BUILTIN_CLASS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1826,6 +1885,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1840,11 +1900,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BUILTIN_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1909,6 +1971,7 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -1924,6 +1987,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1938,11 +2002,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2023,6 +2089,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2037,11 +2104,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BUILTIN_O; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2119,6 +2188,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -2133,11 +2203,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_FUNCTION_EX; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_FUNCTION_EX); opcode = CALL_FUNCTION_EX; _PyStackRef func; @@ -2296,6 +2368,7 @@ JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -2311,6 +2384,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -2325,9 +2399,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_INTRINSIC_1; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_INTRINSIC_1); _PyStackRef value; _PyStackRef res; @@ -2349,9 +2425,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_INTRINSIC_2; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_INTRINSIC_2); _PyStackRef value2_st; _PyStackRef value1_st; @@ -2380,11 +2458,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_ISINSTANCE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2441,9 +2521,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_KW; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_KW); PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; @@ -2612,11 +2694,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_KW_BOUND_METHOD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2745,11 +2829,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_KW_NON_PY; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_KW_NON_PY); opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -2844,6 +2930,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2858,11 +2945,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_KW_PY; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2965,11 +3054,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_LEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3032,11 +3123,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_LIST_APPEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -3096,11 +3189,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_METHOD_DESCRIPTOR_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3171,6 +3266,7 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -3186,6 +3282,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -3200,11 +3297,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3290,6 +3389,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -3304,11 +3404,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3394,6 +3496,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -3408,11 +3511,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_METHOD_DESCRIPTOR_O; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3500,6 +3605,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -3514,11 +3620,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_NON_PY_GENERAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_NON_PY_GENERAL); opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3588,6 +3696,7 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -3603,6 +3712,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -3617,11 +3727,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_PY_EXACT_ARGS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3728,11 +3840,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_PY_GENERAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3824,11 +3938,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_STR_1; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -3883,6 +3999,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -3897,11 +4014,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_TUPLE_1; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -3956,6 +4075,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -3970,11 +4090,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_TYPE_1; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -4014,9 +4136,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CHECK_EG_MATCH; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CHECK_EG_MATCH); _PyStackRef exc_value_st; _PyStackRef match_type_st; @@ -4069,9 +4193,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CHECK_EXC_MATCH; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CHECK_EXC_MATCH); _PyStackRef left; _PyStackRef right; @@ -4101,11 +4227,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CLEANUP_THROW; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter_st; _PyStackRef last_sent_val_st; @@ -4149,9 +4277,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COMPARE_OP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COMPARE_OP); PREDICTED_COMPARE_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -4218,11 +4348,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COMPARE_OP_FLOAT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4270,11 +4402,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COMPARE_OP_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4334,11 +4468,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COMPARE_OP_STR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4387,9 +4523,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CONTAINS_OP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CONTAINS_OP); PREDICTED_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -4439,11 +4577,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CONTAINS_OP_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4479,11 +4619,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CONTAINS_OP_SET; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4520,9 +4662,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CONVERT_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CONVERT_VALUE); _PyStackRef value; _PyStackRef result; @@ -4552,9 +4696,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COPY; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COPY); _PyStackRef bottom; _PyStackRef top; @@ -4571,9 +4717,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COPY_FREE_VARS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COPY_FREE_VARS); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); @@ -4593,9 +4741,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_ATTR); _PyStackRef owner; owner = stack_pointer[-1]; @@ -4616,9 +4766,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_DEREF; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_DEREF); PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); // Can't use ERROR_IF here. @@ -4640,9 +4792,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_FAST); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { @@ -4659,6 +4813,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); DISPATCH(); } @@ -4666,9 +4821,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_GLOBAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4692,9 +4849,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_NAME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_NAME); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -4725,9 +4884,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_SUBSCR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_SUBSCR); _PyStackRef container; _PyStackRef sub; @@ -4752,9 +4913,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DICT_MERGE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DICT_MERGE); _PyStackRef callable; _PyStackRef dict; @@ -4785,9 +4948,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DICT_UPDATE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DICT_UPDATE); _PyStackRef dict; _PyStackRef update; @@ -4822,11 +4987,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = END_ASYNC_FOR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(END_ASYNC_FOR); _PyStackRef awaitable_st; _PyStackRef exc_st; @@ -4859,8 +5026,10 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = END_FOR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(END_FOR); _PyStackRef value; value = stack_pointer[-1]; @@ -4881,9 +5050,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = END_SEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(END_SEND); _PyStackRef receiver; _PyStackRef value; @@ -4903,11 +5074,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = ENTER_EXECUTOR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 @@ -4922,6 +5095,7 @@ * involving _RESUME_CHECK */ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; + LOAD_NEXT_OP_F_OPCODE(); oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { @@ -4942,9 +5116,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = EXIT_INIT_CHECK; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(EXIT_INIT_CHECK); _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; @@ -4966,13 +5142,16 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = EXTENDED_ARG; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(EXTENDED_ARG); opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; + LOAD_NEXT_OP_F_OPCODE(); oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); @@ -4982,9 +5161,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FORMAT_SIMPLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FORMAT_SIMPLE); _PyStackRef value; _PyStackRef res; @@ -5021,9 +5202,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FORMAT_WITH_SPEC; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FORMAT_WITH_SPEC); _PyStackRef value; _PyStackRef fmt_spec; @@ -5049,9 +5232,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FOR_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FOR_ITER); PREDICTED_FOR_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -5069,6 +5254,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ForIter(iter, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + // <- this can modify next_instr! DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -5115,11 +5301,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FOR_ITER_GEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5181,11 +5369,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FOR_ITER_LIST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5242,11 +5432,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FOR_ITER_RANGE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FOR_ITER_RANGE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5297,11 +5489,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FOR_ITER_TUPLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5355,9 +5549,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_AITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_AITER); _PyStackRef obj; _PyStackRef iter; @@ -5408,9 +5604,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_ANEXT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_ANEXT); _PyStackRef aiter; _PyStackRef awaitable; @@ -5432,9 +5630,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_AWAITABLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_AWAITABLE); _PyStackRef iterable; _PyStackRef iter; @@ -5455,9 +5655,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_ITER); _PyStackRef iterable; _PyStackRef iter; @@ -5466,6 +5668,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); PyStackRef_CLOSE(iterable); if (iter_o == NULL) { JUMP_TO_LABEL(pop_1_error); @@ -5479,9 +5682,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_LEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_LEN); _PyStackRef obj; _PyStackRef len; @@ -5508,9 +5713,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_YIELD_FROM_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_YIELD_FROM_ITER); _PyStackRef iterable; _PyStackRef iter; @@ -5555,9 +5762,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = IMPORT_FROM; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(IMPORT_FROM); _PyStackRef from; _PyStackRef res; @@ -5580,9 +5789,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = IMPORT_NAME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(IMPORT_NAME); _PyStackRef level; _PyStackRef fromlist; @@ -5611,11 +5822,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_CALL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_CALL); opcode = INSTRUMENTED_CALL; _PyStackRef *callable; @@ -5765,6 +5978,8 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + // when vectorcall calls into eg sys_settrace, our code can be modified + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -5780,6 +5995,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -5794,11 +6010,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; @@ -5957,6 +6175,7 @@ JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -5972,6 +6191,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -5986,11 +6206,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_CALL_KW; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); opcode = INSTRUMENTED_CALL_KW; _PyStackRef *callable; @@ -6166,10 +6388,12 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_END_FOR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_END_FOR); _PyStackRef receiver; _PyStackRef value; @@ -6195,11 +6419,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_END_SEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; _PyStackRef value; @@ -6229,11 +6455,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ _PyStackRef iter_stackref = TOP(); @@ -6271,11 +6499,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6291,6 +6521,7 @@ } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; + LOAD_NEXT_OP_F_OPCODE(); DISPATCH_GOTO(); } @@ -6298,11 +6529,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -6316,6 +6549,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } // _MONITOR_JUMP_BACKWARD @@ -6329,11 +6563,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); @@ -6343,12 +6579,14 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_LINE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_LINE); opcode = INSTRUMENTED_LINE; int original_opcode = 0; @@ -6368,6 +6606,7 @@ } next_instr = frame->instr_ptr; if (next_instr != this_instr) { + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -6378,6 +6617,7 @@ PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; + LOAD_NEXT_OP_F_OPCODE(); DISPATCH_GOTO(); } @@ -6385,11 +6625,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; @@ -6481,12 +6723,14 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); (void)this_instr; // INSTRUMENTED_JUMP requires this_instr INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); @@ -6497,12 +6741,14 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_POP_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); _PyStackRef iter; iter = stack_pointer[-1]; @@ -6519,11 +6765,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ _PyStackRef cond = POP(); @@ -6540,11 +6788,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ _PyStackRef value_stackref = POP(); @@ -6565,11 +6815,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ _PyStackRef value_stackref = POP(); @@ -6588,11 +6840,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); /* Skip 1 cache entry */ _PyStackRef cond = POP(); @@ -6609,11 +6863,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_RESUME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_RESUME); // _LOAD_BYTECODE { @@ -6633,6 +6889,7 @@ // Make sure this_instr gets reset correctley for any uops that // follow next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); DISPATCH(); } #endif @@ -6650,6 +6907,7 @@ JUMP_TO_LABEL(error); } next_instr = this_instr; + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -6666,6 +6924,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } } @@ -6682,6 +6941,7 @@ /* Instrumentation has jumped */ next_instr = frame->instr_ptr; } + LOAD_NEXT_OP_F(); // maybe needs to be unconditional? } DISPATCH(); } @@ -6690,11 +6950,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); _PyStackRef val; _PyStackRef retval; @@ -6740,11 +7002,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); _PyStackRef val; _PyStackRef retval; @@ -6762,6 +7026,7 @@ } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -6812,9 +7077,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INTERPRETER_EXIT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; @@ -6835,9 +7102,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = IS_OP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(IS_OP); _PyStackRef left; _PyStackRef right; @@ -6858,9 +7127,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = JUMP_BACKWARD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(JUMP_BACKWARD); PREDICTED_JUMP_BACKWARD:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -6888,6 +7159,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -6907,11 +7179,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = JUMP_BACKWARD_JIT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(JUMP_BACKWARD_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ @@ -6926,6 +7200,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -6980,9 +7255,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost @@ -6998,9 +7275,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ @@ -7015,6 +7294,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -7034,9 +7314,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = JUMP_FORWARD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(JUMP_FORWARD); JUMPBY(oparg); DISPATCH(); @@ -7046,9 +7328,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LIST_APPEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LIST_APPEND); _PyStackRef list; _PyStackRef v; @@ -7068,9 +7352,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LIST_EXTEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LIST_EXTEND); _PyStackRef list_st; _PyStackRef iterable_st; @@ -7109,9 +7395,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR); PREDICTED_LOAD_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 10; @@ -7176,6 +7464,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); PyStackRef_CLOSE(owner); if (attr_o == NULL) { JUMP_TO_LABEL(pop_1_error); @@ -7193,11 +7482,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_CLASS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7245,11 +7536,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7307,11 +7600,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7365,11 +7660,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_INSTANCE_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7442,11 +7739,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_METHOD_LAZY_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7499,11 +7798,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_METHOD_NO_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7545,11 +7846,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_METHOD_WITH_VALUES; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7613,11 +7916,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_MODULE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7692,11 +7997,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7733,11 +8040,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7795,11 +8104,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_PROPERTY; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7890,11 +8201,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_SLOT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7953,11 +8266,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_WITH_HINT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -8057,9 +8372,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_BUILD_CLASS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_BUILD_CLASS); _PyStackRef bc; PyObject *bc_o; @@ -8087,9 +8404,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_COMMON_CONSTANT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; // Keep in sync with _common_constants in opcode.py @@ -8113,9 +8432,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_CONST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_CONST); PREDICTED_LOAD_CONST:; _Py_CODEUNIT* const this_instr = next_instr - 1; @@ -8150,9 +8471,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_CONST_IMMORTAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); static_assert(0 == 0, "incorrect cache size"); _PyStackRef value; @@ -8169,9 +8492,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_CONST_MORTAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_CONST_MORTAL); static_assert(0 == 0, "incorrect cache size"); _PyStackRef value; @@ -8187,9 +8512,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_DEREF; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_DEREF); _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -8211,9 +8538,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FAST); _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); @@ -8228,9 +8557,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); _PyStackRef value; value = GETLOCAL(oparg); @@ -8245,9 +8576,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FAST_CHECK; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FAST_CHECK); _PyStackRef value; _PyStackRef value_s = GETLOCAL(oparg); @@ -8271,9 +8604,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); _PyStackRef value1; _PyStackRef value2; @@ -8292,9 +8627,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); _PyStackRef class_dict_st; _PyStackRef value; @@ -8337,9 +8674,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); _PyStackRef mod_or_class_dict; _PyStackRef v; @@ -8417,9 +8756,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_GLOBAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 5; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED_LOAD_GLOBAL:; _Py_CODEUNIT* const this_instr = next_instr - 5; @@ -8471,11 +8812,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_GLOBAL_BUILTIN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *builtins_keys; @@ -8555,11 +8898,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_GLOBAL_MODULE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *globals_keys; @@ -8623,9 +8968,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_LOCALS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_LOCALS); _PyStackRef locals; PyObject *l = LOCALS(); @@ -8647,9 +8994,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_NAME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_NAME); _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -8670,9 +9019,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_SMALL_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_SMALL_INT); _PyStackRef value; assert(oparg < _PY_NSMALLPOSINTS); @@ -8688,9 +9039,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_SPECIAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_SPECIAL); _PyStackRef owner; _PyStackRef attr; @@ -8729,9 +9082,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_SUPER_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -8842,11 +9197,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_SUPER_ATTR_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; @@ -8893,11 +9250,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_SUPER_ATTR_METHOD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; @@ -8960,9 +9319,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MAKE_CELL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MAKE_CELL); // "initial" is probably NULL but not if it's an arg (or set // via the f_locals proxy before MAKE_CELL has run). @@ -8983,9 +9344,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MAKE_FUNCTION; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MAKE_FUNCTION); _PyStackRef codeobj_st; _PyStackRef func; @@ -9016,9 +9379,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MAP_ADD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MAP_ADD); _PyStackRef dict_st; _PyStackRef key; @@ -9049,9 +9414,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MATCH_CLASS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MATCH_CLASS); _PyStackRef subject; _PyStackRef type; @@ -9093,9 +9460,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MATCH_KEYS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MATCH_KEYS); _PyStackRef subject; _PyStackRef keys; @@ -9121,9 +9490,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MATCH_MAPPING; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MATCH_MAPPING); _PyStackRef subject; _PyStackRef res; @@ -9140,9 +9511,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MATCH_SEQUENCE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MATCH_SEQUENCE); _PyStackRef subject; _PyStackRef res; @@ -9159,9 +9532,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = NOP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(NOP); DISPATCH(); } @@ -9170,9 +9545,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = NOT_TAKEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(NOT_TAKEN); DISPATCH(); } @@ -9181,9 +9558,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_EXCEPT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_EXCEPT); _PyStackRef exc_value; exc_value = stack_pointer[-1]; @@ -9202,13 +9581,16 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_ITER); _PyStackRef value; value = stack_pointer[-1]; PyStackRef_CLOSE(value); + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -9218,11 +9600,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_JUMP_IF_FALSE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_JUMP_IF_FALSE); _PyStackRef cond; /* Skip 1 cache entry */ @@ -9240,11 +9624,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_JUMP_IF_NONE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_JUMP_IF_NONE); _PyStackRef value; _PyStackRef b; @@ -9278,11 +9664,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); _PyStackRef value; _PyStackRef b; @@ -9316,11 +9704,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_JUMP_IF_TRUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_JUMP_IF_TRUE); _PyStackRef cond; /* Skip 1 cache entry */ @@ -9338,13 +9728,16 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_TOP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_TOP); _PyStackRef value; value = stack_pointer[-1]; PyStackRef_CLOSE(value); + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -9354,9 +9747,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = PUSH_EXC_INFO; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(PUSH_EXC_INFO); _PyStackRef exc; _PyStackRef prev_exc; @@ -9383,9 +9778,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = PUSH_NULL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(PUSH_NULL); _PyStackRef res; res = PyStackRef_NULL; @@ -9399,11 +9796,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RAISE_VARARGS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RAISE_VARARGS); _PyStackRef *args; args = &stack_pointer[-oparg]; @@ -9430,11 +9829,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RERAISE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RERAISE); _PyStackRef *values; _PyStackRef exc_st; @@ -9473,9 +9874,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RESERVED; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RESERVED); assert(0 && "Executing RESERVED instruction."); Py_FatalError("Executing RESERVED instruction."); @@ -9486,9 +9889,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RESUME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RESUME); PREDICTED_RESUME:; _Py_CODEUNIT* const this_instr = next_instr - 1; @@ -9511,6 +9916,7 @@ // Make sure this_instr gets reset correctley for any uops that // follow next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); DISPATCH(); } #endif @@ -9528,6 +9934,7 @@ JUMP_TO_LABEL(error); } next_instr = this_instr; + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -9552,6 +9959,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } } @@ -9562,11 +9970,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RESUME_CHECK; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) @@ -9600,9 +10010,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RETURN_GENERATOR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); @@ -9639,9 +10051,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RETURN_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RETURN_VALUE); _PyStackRef retval; _PyStackRef res; @@ -9671,9 +10085,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SEND); PREDICTED_SEND:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -9760,6 +10176,7 @@ PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); retval = PyStackRef_FromPyObjectSteal(retval_o); + LOAD_NEXT_OP_F(); } stack_pointer[0] = retval; stack_pointer += 1; @@ -9771,11 +10188,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SEND_GEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; @@ -9841,9 +10260,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SETUP_ANNOTATIONS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SETUP_ANNOTATIONS); PyObject *ann_dict; if (LOCALS() == NULL) { @@ -9870,6 +10291,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { @@ -9888,9 +10312,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SET_ADD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SET_ADD); _PyStackRef set; _PyStackRef v; @@ -9913,9 +10339,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); _PyStackRef attr_st; _PyStackRef func_in; @@ -9941,9 +10369,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SET_UPDATE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SET_UPDATE); _PyStackRef set; _PyStackRef iterable; @@ -9966,9 +10396,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 5; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_ATTR); PREDICTED_STORE_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 5; @@ -10002,6 +10434,8 @@ int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); + // SetAttr can call into specialize + LOAD_NEXT_OP_F(); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); if (err) { @@ -10017,11 +10451,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_ATTR_INSTANCE_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; @@ -10093,11 +10529,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_ATTR_SLOT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; @@ -10144,11 +10582,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_ATTR_WITH_HINT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; @@ -10244,9 +10684,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_DEREF; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_DEREF); _PyStackRef v; v = stack_pointer[-1]; @@ -10263,9 +10705,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_FAST); _PyStackRef value; value = stack_pointer[-1]; @@ -10283,9 +10727,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_FAST_LOAD_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); _PyStackRef value1; _PyStackRef value2; @@ -10306,9 +10752,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_FAST_STORE_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_FAST_STORE_FAST); _PyStackRef value2; _PyStackRef value1; @@ -10337,9 +10785,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_GLOBAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_GLOBAL); _PyStackRef v; v = stack_pointer[-1]; @@ -10360,9 +10810,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_NAME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_NAME); _PyStackRef v; v = stack_pointer[-1]; @@ -10386,6 +10838,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); } PyStackRef_CLOSE(v); if (err) { @@ -10400,9 +10853,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_SLICE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_SLICE); _PyStackRef v; _PyStackRef container; @@ -10434,6 +10889,9 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(slice); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += 2; @@ -10454,9 +10912,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_SUBSCR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED_STORE_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -10489,6 +10949,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); @@ -10505,11 +10966,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_SUBSCR_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; @@ -10546,11 +11009,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_SUBSCR_LIST_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; @@ -10612,9 +11077,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SWAP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SWAP); _PyStackRef *bottom; _PyStackRef *top; @@ -10631,9 +11098,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL); PREDICTED_TO_BOOL:; _Py_CODEUNIT* const this_instr = next_instr - 4; @@ -10677,11 +11146,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_ALWAYS_TRUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef owner; @@ -10714,11 +11185,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_BOOL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -10738,11 +11211,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_INT); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -10773,11 +11248,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_LIST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -10802,11 +11279,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_NONE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_NONE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -10830,11 +11309,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_STR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_STR); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -10866,9 +11347,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNARY_INVERT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNARY_INVERT); _PyStackRef value; _PyStackRef res; @@ -10889,9 +11372,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNARY_NEGATIVE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNARY_NEGATIVE); _PyStackRef value; _PyStackRef res; @@ -10912,9 +11397,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNARY_NOT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNARY_NOT); _PyStackRef value; _PyStackRef res; @@ -10930,9 +11417,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNPACK_EX; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNPACK_EX); _PyStackRef seq; _PyStackRef *right; @@ -10955,9 +11444,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNPACK_SEQUENCE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED_UNPACK_SEQUENCE:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -11004,11 +11495,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNPACK_SEQUENCE_LIST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef seq; @@ -11051,11 +11544,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNPACK_SEQUENCE_TUPLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef seq; @@ -11089,11 +11584,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNPACK_SEQUENCE_TWO_TUPLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef seq; @@ -11128,9 +11625,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = WITH_EXCEPT_START; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(WITH_EXCEPT_START); _PyStackRef exit_func; _PyStackRef exit_self; @@ -11176,6 +11675,7 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -11186,9 +11686,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = YIELD_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(YIELD_VALUE); _PyStackRef retval; _PyStackRef value; @@ -11370,6 +11872,7 @@ JUMP_TO_LABEL(error); #ifdef Py_TAIL_CALL_INTERP int opcode; #endif + py_tail_call_funcptr next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; DISPATCH(); } @@ -11414,6 +11917,7 @@ JUMP_TO_LABEL(error); #ifdef Py_TAIL_CALL_INTERP int opcode; #endif + py_tail_call_funcptr next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; DISPATCH(); } diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 02012fb4172260..9ee213f62a6629 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -232,6 +232,7 @@ def generate_tier1_cases( out.emit(f"#if defined(Py_TAIL_CALL_INTERP)\n") out.emit(f"int opcode = {name};\n") out.emit(f"(void)(opcode);\n") + out.emit(f"py_tail_call_funcptr next_op_f;\n") out.emit(f"#endif\n") needs_this = uses_this(inst) unused_guard = "(void)this_instr;\n" @@ -245,6 +246,7 @@ def generate_tier1_cases( out.emit(f"frame->instr_ptr = next_instr;\n") out.emit(f"next_instr += {inst.size};\n") + out.emit(f"LOAD_NEXT_OP_F();\n") out.emit(f"INSTRUCTION_STATS({name});\n") if inst.is_target: out.emit(f"PREDICTED_{name}:;\n")