From 25da04537e112982714b2080cc8cc9d78f7f2250 Mon Sep 17 00:00:00 2001 From: Andrew Consroe Date: Mon, 10 Feb 2025 12:03:03 -0600 Subject: [PATCH 1/7] okay earliest assignment, compiles but crashes --- Python/bytecodes.c | 2 + Python/ceval_macros.h | 2 +- Python/generated_cases.c.h | 444 +++++++++++++++++++++++ Tools/cases_generator/tier1_generator.py | 2 + 4 files changed, 449 insertions(+), 1 deletion(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 703d7ec61ebab3c..908a07dad37dc54 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5238,6 +5238,7 @@ dummy_func( #ifdef Py_TAIL_CALL_INTERP int opcode; #endif + py_tail_call_funcptr next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; DISPATCH(); } @@ -5280,6 +5281,7 @@ dummy_func( #ifdef Py_TAIL_CALL_INTERP int opcode; #endif + py_tail_call_funcptr next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; DISPATCH(); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 0a4f65feb3b512a..f878c9414f53bf6 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -82,7 +82,7 @@ # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) # define DISPATCH_GOTO() \ do { \ - Py_MUSTTAIL return (INSTRUCTION_TABLE[opcode])(TAIL_CALL_ARGS); \ + Py_MUSTTAIL return next_op_f(TAIL_CALL_ARGS); \ } while (0) # define JUMP_TO_LABEL(name) \ do { \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f02e13f5e3fbce1..f21e20be7e335ad 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -22,9 +22,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP); PREDICTED_BINARY_OP:; _Py_CODEUNIT* const this_instr = next_instr - 6; @@ -78,11 +80,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_ADD_FLOAT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -132,11 +136,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_ADD_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -185,11 +191,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_ADD_UNICODE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -238,11 +246,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_EXTEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_EXTEND); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -294,11 +304,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_INPLACE_ADD_UNICODE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -377,11 +389,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_MULTIPLY_FLOAT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -431,11 +445,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_MULTIPLY_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -484,11 +500,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBSCR_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef dict_st; @@ -531,11 +549,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBSCR_GETITEM; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef container; @@ -616,11 +636,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBSCR_LIST_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef list_st; @@ -686,11 +708,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBSCR_STR_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef str_st; @@ -748,11 +772,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBSCR_TUPLE_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef tuple_st; @@ -806,11 +832,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBTRACT_FLOAT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -860,11 +888,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_OP_SUBTRACT_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -913,9 +943,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BINARY_SLICE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BINARY_SLICE); _PyStackRef container; _PyStackRef start; @@ -973,9 +1005,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_LIST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BUILD_LIST); _PyStackRef *values; _PyStackRef list; @@ -995,9 +1029,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_MAP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BUILD_MAP); _PyStackRef *values; _PyStackRef map; @@ -1037,9 +1073,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_SET; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BUILD_SET); _PyStackRef *values; _PyStackRef set; @@ -1085,9 +1123,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_SLICE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BUILD_SLICE); _PyStackRef *args; _PyStackRef slice; @@ -1115,9 +1155,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_STRING; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BUILD_STRING); _PyStackRef *pieces; _PyStackRef str; @@ -1152,9 +1194,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = BUILD_TUPLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(BUILD_TUPLE); _PyStackRef *values; _PyStackRef tup; @@ -1174,9 +1218,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CACHE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CACHE); assert(0 && "Executing a cache."); Py_FatalError("Executing a cache."); @@ -1187,9 +1233,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL); PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; @@ -1359,11 +1407,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_ALLOC_AND_ENTER_INIT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1487,11 +1537,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BOUND_METHOD_EXACT_ARGS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1626,11 +1678,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BOUND_METHOD_GENERAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1748,11 +1802,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BUILTIN_CLASS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1840,11 +1896,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BUILTIN_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1938,11 +1996,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2037,11 +2097,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_BUILTIN_O; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2133,11 +2195,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_FUNCTION_EX; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_FUNCTION_EX); opcode = CALL_FUNCTION_EX; _PyStackRef func; @@ -2325,9 +2389,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_INTRINSIC_1; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_INTRINSIC_1); _PyStackRef value; _PyStackRef res; @@ -2349,9 +2415,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_INTRINSIC_2; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_INTRINSIC_2); _PyStackRef value2_st; _PyStackRef value1_st; @@ -2380,11 +2448,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_ISINSTANCE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2441,9 +2511,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_KW; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_KW); PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; @@ -2612,11 +2684,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_KW_BOUND_METHOD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2745,11 +2819,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_KW_NON_PY; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_KW_NON_PY); opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -2858,11 +2934,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_KW_PY; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2965,11 +3043,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_LEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3032,11 +3112,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_LIST_APPEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -3096,11 +3178,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_METHOD_DESCRIPTOR_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3200,11 +3284,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3304,11 +3390,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3408,11 +3496,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_METHOD_DESCRIPTOR_O; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3514,11 +3604,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_NON_PY_GENERAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3617,11 +3709,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_PY_EXACT_ARGS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3728,11 +3822,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_PY_GENERAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3824,11 +3920,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_STR_1; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -3897,11 +3995,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_TUPLE_1; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -3970,11 +4070,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CALL_TYPE_1; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -4014,9 +4116,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CHECK_EG_MATCH; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CHECK_EG_MATCH); _PyStackRef exc_value_st; _PyStackRef match_type_st; @@ -4069,9 +4173,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CHECK_EXC_MATCH; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CHECK_EXC_MATCH); _PyStackRef left; _PyStackRef right; @@ -4101,11 +4207,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CLEANUP_THROW; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter_st; _PyStackRef last_sent_val_st; @@ -4149,9 +4257,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COMPARE_OP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(COMPARE_OP); PREDICTED_COMPARE_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -4218,11 +4328,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COMPARE_OP_FLOAT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4270,11 +4382,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COMPARE_OP_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4334,11 +4448,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COMPARE_OP_STR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4387,9 +4503,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CONTAINS_OP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CONTAINS_OP); PREDICTED_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -4439,11 +4557,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CONTAINS_OP_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4479,11 +4599,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CONTAINS_OP_SET; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4520,9 +4642,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = CONVERT_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(CONVERT_VALUE); _PyStackRef value; _PyStackRef result; @@ -4552,9 +4676,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COPY; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(COPY); _PyStackRef bottom; _PyStackRef top; @@ -4571,9 +4697,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = COPY_FREE_VARS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(COPY_FREE_VARS); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); @@ -4593,9 +4721,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(DELETE_ATTR); _PyStackRef owner; owner = stack_pointer[-1]; @@ -4616,9 +4746,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_DEREF; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(DELETE_DEREF); PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); // Can't use ERROR_IF here. @@ -4640,9 +4772,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(DELETE_FAST); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { @@ -4666,9 +4800,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_GLOBAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4692,9 +4828,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_NAME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(DELETE_NAME); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -4725,9 +4863,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DELETE_SUBSCR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(DELETE_SUBSCR); _PyStackRef container; _PyStackRef sub; @@ -4752,9 +4892,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DICT_MERGE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(DICT_MERGE); _PyStackRef callable; _PyStackRef dict; @@ -4785,9 +4927,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = DICT_UPDATE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(DICT_UPDATE); _PyStackRef dict; _PyStackRef update; @@ -4822,11 +4966,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = END_ASYNC_FOR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(END_ASYNC_FOR); _PyStackRef awaitable_st; _PyStackRef exc_st; @@ -4859,8 +5005,10 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = END_FOR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(END_FOR); _PyStackRef value; value = stack_pointer[-1]; @@ -4881,9 +5029,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = END_SEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(END_SEND); _PyStackRef receiver; _PyStackRef value; @@ -4903,11 +5053,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = ENTER_EXECUTOR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 @@ -4942,9 +5094,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = EXIT_INIT_CHECK; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(EXIT_INIT_CHECK); _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; @@ -4966,9 +5120,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = EXTENDED_ARG; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(EXTENDED_ARG); opcode = EXTENDED_ARG; assert(oparg); @@ -4982,9 +5138,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FORMAT_SIMPLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(FORMAT_SIMPLE); _PyStackRef value; _PyStackRef res; @@ -5021,9 +5179,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FORMAT_WITH_SPEC; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(FORMAT_WITH_SPEC); _PyStackRef value; _PyStackRef fmt_spec; @@ -5049,9 +5209,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FOR_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(FOR_ITER); PREDICTED_FOR_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -5115,11 +5277,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FOR_ITER_GEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5181,11 +5345,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FOR_ITER_LIST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5242,11 +5408,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FOR_ITER_RANGE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(FOR_ITER_RANGE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5297,11 +5465,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = FOR_ITER_TUPLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5355,9 +5525,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_AITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(GET_AITER); _PyStackRef obj; _PyStackRef iter; @@ -5408,9 +5580,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_ANEXT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(GET_ANEXT); _PyStackRef aiter; _PyStackRef awaitable; @@ -5432,9 +5606,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_AWAITABLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(GET_AWAITABLE); _PyStackRef iterable; _PyStackRef iter; @@ -5455,9 +5631,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(GET_ITER); _PyStackRef iterable; _PyStackRef iter; @@ -5479,9 +5657,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_LEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(GET_LEN); _PyStackRef obj; _PyStackRef len; @@ -5508,9 +5688,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = GET_YIELD_FROM_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); _PyStackRef iterable; _PyStackRef iter; @@ -5555,9 +5737,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = IMPORT_FROM; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(IMPORT_FROM); _PyStackRef from; _PyStackRef res; @@ -5580,9 +5764,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = IMPORT_NAME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(IMPORT_NAME); _PyStackRef level; _PyStackRef fromlist; @@ -5611,11 +5797,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_CALL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_CALL); opcode = INSTRUMENTED_CALL; _PyStackRef *callable; @@ -5794,11 +5982,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; @@ -5986,11 +6176,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_CALL_KW; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); opcode = INSTRUMENTED_CALL_KW; _PyStackRef *callable; @@ -6166,10 +6358,12 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_END_FOR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); _PyStackRef receiver; _PyStackRef value; @@ -6195,11 +6389,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_END_SEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; _PyStackRef value; @@ -6229,11 +6425,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ _PyStackRef iter_stackref = TOP(); @@ -6271,11 +6469,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6298,11 +6498,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -6329,11 +6531,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); @@ -6343,12 +6547,14 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_LINE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_LINE); opcode = INSTRUMENTED_LINE; int original_opcode = 0; @@ -6385,11 +6591,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; @@ -6481,12 +6689,14 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); (void)this_instr; // INSTRUMENTED_JUMP requires this_instr INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); @@ -6497,12 +6707,14 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_POP_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); _PyStackRef iter; iter = stack_pointer[-1]; @@ -6519,11 +6731,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ _PyStackRef cond = POP(); @@ -6540,11 +6754,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ _PyStackRef value_stackref = POP(); @@ -6565,11 +6781,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ _PyStackRef value_stackref = POP(); @@ -6588,11 +6806,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); /* Skip 1 cache entry */ _PyStackRef cond = POP(); @@ -6609,11 +6829,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_RESUME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_RESUME); // _LOAD_BYTECODE { @@ -6690,11 +6912,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); _PyStackRef val; _PyStackRef retval; @@ -6740,11 +6964,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); _PyStackRef val; _PyStackRef retval; @@ -6812,9 +7038,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = INTERPRETER_EXIT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; @@ -6835,9 +7063,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = IS_OP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(IS_OP); _PyStackRef left; _PyStackRef right; @@ -6858,9 +7088,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = JUMP_BACKWARD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(JUMP_BACKWARD); PREDICTED_JUMP_BACKWARD:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -6907,11 +7139,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = JUMP_BACKWARD_JIT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(JUMP_BACKWARD_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ @@ -6980,9 +7214,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost @@ -6998,9 +7234,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ @@ -7034,9 +7272,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = JUMP_FORWARD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(JUMP_FORWARD); JUMPBY(oparg); DISPATCH(); @@ -7046,9 +7286,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LIST_APPEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LIST_APPEND); _PyStackRef list; _PyStackRef v; @@ -7068,9 +7310,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LIST_EXTEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LIST_EXTEND); _PyStackRef list_st; _PyStackRef iterable_st; @@ -7109,9 +7353,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR); PREDICTED_LOAD_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 10; @@ -7193,11 +7439,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_CLASS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7245,11 +7493,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7307,11 +7557,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7365,11 +7617,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_INSTANCE_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7442,11 +7696,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_METHOD_LAZY_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7499,11 +7755,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_METHOD_NO_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7545,11 +7803,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_METHOD_WITH_VALUES; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7613,11 +7873,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_MODULE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7692,11 +7954,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7733,11 +7997,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7795,11 +8061,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_PROPERTY; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7890,11 +8158,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_SLOT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7953,11 +8223,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_ATTR_WITH_HINT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -8057,9 +8329,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_BUILD_CLASS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_BUILD_CLASS); _PyStackRef bc; PyObject *bc_o; @@ -8087,9 +8361,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_COMMON_CONSTANT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; // Keep in sync with _common_constants in opcode.py @@ -8113,9 +8389,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_CONST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_CONST); PREDICTED_LOAD_CONST:; _Py_CODEUNIT* const this_instr = next_instr - 1; @@ -8150,9 +8428,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_CONST_IMMORTAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); static_assert(0 == 0, "incorrect cache size"); _PyStackRef value; @@ -8169,9 +8449,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_CONST_MORTAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_CONST_MORTAL); static_assert(0 == 0, "incorrect cache size"); _PyStackRef value; @@ -8187,9 +8469,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_DEREF; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_DEREF); _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -8211,9 +8495,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_FAST); _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); @@ -8228,9 +8514,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); _PyStackRef value; value = GETLOCAL(oparg); @@ -8245,9 +8533,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FAST_CHECK; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_FAST_CHECK); _PyStackRef value; _PyStackRef value_s = GETLOCAL(oparg); @@ -8271,9 +8561,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); _PyStackRef value1; _PyStackRef value2; @@ -8292,9 +8584,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); _PyStackRef class_dict_st; _PyStackRef value; @@ -8337,9 +8631,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); _PyStackRef mod_or_class_dict; _PyStackRef v; @@ -8417,9 +8713,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_GLOBAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 5; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED_LOAD_GLOBAL:; _Py_CODEUNIT* const this_instr = next_instr - 5; @@ -8471,11 +8769,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_GLOBAL_BUILTIN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *builtins_keys; @@ -8555,11 +8855,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_GLOBAL_MODULE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *globals_keys; @@ -8623,9 +8925,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_LOCALS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_LOCALS); _PyStackRef locals; PyObject *l = LOCALS(); @@ -8647,9 +8951,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_NAME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_NAME); _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -8670,9 +8976,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_SMALL_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_SMALL_INT); _PyStackRef value; assert(oparg < _PY_NSMALLPOSINTS); @@ -8688,9 +8996,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_SPECIAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_SPECIAL); _PyStackRef owner; _PyStackRef attr; @@ -8729,9 +9039,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_SUPER_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -8842,11 +9154,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_SUPER_ATTR_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; @@ -8893,11 +9207,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = LOAD_SUPER_ATTR_METHOD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; @@ -8960,9 +9276,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MAKE_CELL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(MAKE_CELL); // "initial" is probably NULL but not if it's an arg (or set // via the f_locals proxy before MAKE_CELL has run). @@ -8983,9 +9301,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MAKE_FUNCTION; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(MAKE_FUNCTION); _PyStackRef codeobj_st; _PyStackRef func; @@ -9016,9 +9336,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MAP_ADD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(MAP_ADD); _PyStackRef dict_st; _PyStackRef key; @@ -9049,9 +9371,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MATCH_CLASS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(MATCH_CLASS); _PyStackRef subject; _PyStackRef type; @@ -9093,9 +9417,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MATCH_KEYS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(MATCH_KEYS); _PyStackRef subject; _PyStackRef keys; @@ -9121,9 +9447,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MATCH_MAPPING; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(MATCH_MAPPING); _PyStackRef subject; _PyStackRef res; @@ -9140,9 +9468,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = MATCH_SEQUENCE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(MATCH_SEQUENCE); _PyStackRef subject; _PyStackRef res; @@ -9159,9 +9489,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = NOP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(NOP); DISPATCH(); } @@ -9170,9 +9502,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = NOT_TAKEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(NOT_TAKEN); DISPATCH(); } @@ -9181,9 +9515,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_EXCEPT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(POP_EXCEPT); _PyStackRef exc_value; exc_value = stack_pointer[-1]; @@ -9202,9 +9538,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_ITER; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(POP_ITER); _PyStackRef value; value = stack_pointer[-1]; @@ -9218,11 +9556,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_JUMP_IF_FALSE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); _PyStackRef cond; /* Skip 1 cache entry */ @@ -9240,11 +9580,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_JUMP_IF_NONE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(POP_JUMP_IF_NONE); _PyStackRef value; _PyStackRef b; @@ -9278,11 +9620,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); _PyStackRef value; _PyStackRef b; @@ -9316,11 +9660,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_JUMP_IF_TRUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); _PyStackRef cond; /* Skip 1 cache entry */ @@ -9338,9 +9684,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = POP_TOP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(POP_TOP); _PyStackRef value; value = stack_pointer[-1]; @@ -9354,9 +9702,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = PUSH_EXC_INFO; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(PUSH_EXC_INFO); _PyStackRef exc; _PyStackRef prev_exc; @@ -9383,9 +9733,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = PUSH_NULL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(PUSH_NULL); _PyStackRef res; res = PyStackRef_NULL; @@ -9399,11 +9751,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RAISE_VARARGS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(RAISE_VARARGS); _PyStackRef *args; args = &stack_pointer[-oparg]; @@ -9430,11 +9784,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RERAISE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(RERAISE); _PyStackRef *values; _PyStackRef exc_st; @@ -9473,9 +9829,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RESERVED; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(RESERVED); assert(0 && "Executing RESERVED instruction."); Py_FatalError("Executing RESERVED instruction."); @@ -9486,9 +9844,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RESUME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(RESUME); PREDICTED_RESUME:; _Py_CODEUNIT* const this_instr = next_instr - 1; @@ -9562,11 +9922,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RESUME_CHECK; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) @@ -9600,9 +9962,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RETURN_GENERATOR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); @@ -9639,9 +10003,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = RETURN_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(RETURN_VALUE); _PyStackRef retval; _PyStackRef res; @@ -9671,9 +10037,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SEND; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(SEND); PREDICTED_SEND:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -9771,11 +10139,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SEND_GEN; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; @@ -9841,9 +10211,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SETUP_ANNOTATIONS; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(SETUP_ANNOTATIONS); PyObject *ann_dict; if (LOCALS() == NULL) { @@ -9888,9 +10260,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SET_ADD; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(SET_ADD); _PyStackRef set; _PyStackRef v; @@ -9913,9 +10287,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); _PyStackRef attr_st; _PyStackRef func_in; @@ -9941,9 +10317,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SET_UPDATE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(SET_UPDATE); _PyStackRef set; _PyStackRef iterable; @@ -9966,9 +10344,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_ATTR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 5; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_ATTR); PREDICTED_STORE_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 5; @@ -10017,11 +10397,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_ATTR_INSTANCE_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; @@ -10093,11 +10475,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_ATTR_SLOT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; @@ -10144,11 +10528,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_ATTR_WITH_HINT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; @@ -10244,9 +10630,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_DEREF; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_DEREF); _PyStackRef v; v = stack_pointer[-1]; @@ -10263,9 +10651,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_FAST); _PyStackRef value; value = stack_pointer[-1]; @@ -10283,9 +10673,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_FAST_LOAD_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); _PyStackRef value1; _PyStackRef value2; @@ -10306,9 +10698,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_FAST_STORE_FAST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); _PyStackRef value2; _PyStackRef value1; @@ -10337,9 +10731,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_GLOBAL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_GLOBAL); _PyStackRef v; v = stack_pointer[-1]; @@ -10360,9 +10756,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_NAME; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_NAME); _PyStackRef v; v = stack_pointer[-1]; @@ -10400,9 +10798,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_SLICE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_SLICE); _PyStackRef v; _PyStackRef container; @@ -10454,9 +10854,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_SUBSCR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED_STORE_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -10505,11 +10907,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_SUBSCR_DICT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; @@ -10546,11 +10950,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = STORE_SUBSCR_LIST_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; @@ -10612,9 +11018,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = SWAP; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(SWAP); _PyStackRef *bottom; _PyStackRef *top; @@ -10631,9 +11039,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(TO_BOOL); PREDICTED_TO_BOOL:; _Py_CODEUNIT* const this_instr = next_instr - 4; @@ -10677,11 +11087,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_ALWAYS_TRUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef owner; @@ -10714,11 +11126,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_BOOL; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(TO_BOOL_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -10738,11 +11152,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_INT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(TO_BOOL_INT); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -10773,11 +11189,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_LIST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -10802,11 +11220,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_NONE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(TO_BOOL_NONE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -10830,11 +11250,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = TO_BOOL_STR; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(TO_BOOL_STR); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -10866,9 +11288,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNARY_INVERT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(UNARY_INVERT); _PyStackRef value; _PyStackRef res; @@ -10889,9 +11313,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNARY_NEGATIVE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(UNARY_NEGATIVE); _PyStackRef value; _PyStackRef res; @@ -10912,9 +11338,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNARY_NOT; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(UNARY_NOT); _PyStackRef value; _PyStackRef res; @@ -10930,9 +11358,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNPACK_EX; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(UNPACK_EX); _PyStackRef seq; _PyStackRef *right; @@ -10955,9 +11385,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNPACK_SEQUENCE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED_UNPACK_SEQUENCE:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -11004,11 +11436,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNPACK_SEQUENCE_LIST; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef seq; @@ -11051,11 +11485,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNPACK_SEQUENCE_TUPLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef seq; @@ -11089,11 +11525,13 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = UNPACK_SEQUENCE_TWO_TUPLE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef seq; @@ -11128,9 +11566,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = WITH_EXCEPT_START; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(WITH_EXCEPT_START); _PyStackRef exit_func; _PyStackRef exit_self; @@ -11186,9 +11626,11 @@ #if defined(Py_TAIL_CALL_INTERP) int opcode = YIELD_VALUE; (void)(opcode); + py_tail_call_funcptr next_op_f; #endif frame->instr_ptr = next_instr; next_instr += 1; + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; INSTRUCTION_STATS(YIELD_VALUE); _PyStackRef retval; _PyStackRef value; @@ -11370,6 +11812,7 @@ JUMP_TO_LABEL(error); #ifdef Py_TAIL_CALL_INTERP int opcode; #endif + py_tail_call_funcptr next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; DISPATCH(); } @@ -11414,6 +11857,7 @@ JUMP_TO_LABEL(error); #ifdef Py_TAIL_CALL_INTERP int opcode; #endif + py_tail_call_funcptr next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; DISPATCH(); } diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 02012fb41722607..d240b6a338b9415 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -232,6 +232,7 @@ def generate_tier1_cases( out.emit(f"#if defined(Py_TAIL_CALL_INTERP)\n") out.emit(f"int opcode = {name};\n") out.emit(f"(void)(opcode);\n") + out.emit(f"py_tail_call_funcptr next_op_f;\n") out.emit(f"#endif\n") needs_this = uses_this(inst) unused_guard = "(void)this_instr;\n" @@ -245,6 +246,7 @@ def generate_tier1_cases( out.emit(f"frame->instr_ptr = next_instr;\n") out.emit(f"next_instr += {inst.size};\n") + out.emit(f"next_op_f = INSTRUCTION_TABLE[next_instr->op.code];\n") out.emit(f"INSTRUCTION_STATS({name});\n") if inst.is_target: out.emit(f"PREDICTED_{name}:;\n") From 1979297b67f6d8ad5149413deb9415e1f8fba8ef Mon Sep 17 00:00:00 2001 From: Andrew Consroe Date: Mon, 10 Feb 2025 12:07:16 -0600 Subject: [PATCH 2/7] okay earliest assignment, compiles but crashes --- Python/ceval_macros.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index f878c9414f53bf6..0aafded99f731b1 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -74,6 +74,10 @@ #define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg #ifdef Py_TAIL_CALL_INTERP +# define LOAD_NEXT_OP_F() \ + do { \ + next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; \ + } while (0) // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. # define Py_MUSTTAIL [[clang::musttail]] # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) @@ -189,8 +193,8 @@ GETITEM(PyObject *v, Py_ssize_t i) { * for advancing to the next instruction, taking into account cache entries * and skipped instructions. */ -#define JUMPBY(x) (next_instr += (x)) -#define SKIP_OVER(x) (next_instr += (x)) +#define JUMPBY(x) (next_instr += (x)) LOAD_NEXT_OP_F() +#define SKIP_OVER(x) (next_instr += (x)) LOAD_NEXT_OP_F() /* Stack manipulation macros */ From 92aef83c571f9c6f977a90fb053c381c1fd93f4d Mon Sep 17 00:00:00 2001 From: Andrew Consroe Date: Mon, 10 Feb 2025 12:12:26 -0600 Subject: [PATCH 3/7] okay that worked --- Python/bytecodes.c | 22 ++++++++++++++++++++++ Python/ceval_macros.h | 6 ++++-- Python/generated_cases.c.h | 23 +++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 908a07dad37dc54..70c3b85d49f6411 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -207,6 +207,7 @@ dummy_func( // Make sure this_instr gets reset correctley for any uops that // follow next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); DISPATCH(); } #endif @@ -240,6 +241,7 @@ dummy_func( if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); } } @@ -443,6 +445,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } @@ -980,6 +983,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } @@ -1179,6 +1183,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } @@ -1304,6 +1309,7 @@ dummy_func( } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -1452,6 +1458,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -1529,6 +1536,7 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -1646,6 +1654,7 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } @@ -2018,6 +2027,7 @@ dummy_func( int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); DISPATCH_SAME_OPARG(); } @@ -2141,6 +2151,7 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -2547,6 +2558,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -2668,6 +2680,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_ContainsOp(right, next_instr); DISPATCH_SAME_OPARG(); } @@ -2777,6 +2790,7 @@ dummy_func( this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; // Need to re-dispatch so the warmup counter isn't off by one: next_instr = this_instr; + LOAD_NEXT_OP_F(); DISPATCH_SAME_OPARG(); } #endif @@ -2861,6 +2875,7 @@ dummy_func( opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; + LOAD_NEXT_OP_F(); if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } @@ -3021,6 +3036,7 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -3489,6 +3505,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); DISPATCH_SAME_OPARG(); } @@ -4499,6 +4516,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); DISPATCH_SAME_OPARG(); } @@ -4781,6 +4799,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } @@ -4828,6 +4847,7 @@ dummy_func( } next_instr = frame->instr_ptr; if (next_instr != this_instr) { + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -4838,6 +4858,7 @@ dummy_func( PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; + LOAD_NEXT_OP_F(); DISPATCH_GOTO(); } @@ -4846,6 +4867,7 @@ dummy_func( tstate, frame, this_instr); ERROR_IF(next_opcode < 0, error); next_instr = this_instr; + LOAD_NEXT_OP_F(); if (_PyOpcode_Caches[next_opcode]) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 0aafded99f731b1..9e6f4b2d0d09466 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -193,8 +193,8 @@ GETITEM(PyObject *v, Py_ssize_t i) { * for advancing to the next instruction, taking into account cache entries * and skipped instructions. */ -#define JUMPBY(x) (next_instr += (x)) LOAD_NEXT_OP_F() -#define SKIP_OVER(x) (next_instr += (x)) LOAD_NEXT_OP_F() +#define JUMPBY(x) (next_instr += (x)); LOAD_NEXT_OP_F() +#define SKIP_OVER(x) (next_instr += (x)); LOAD_NEXT_OP_F() /* Stack manipulation macros */ @@ -347,6 +347,7 @@ do { \ } else { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ next_instr = _Py_call_instrumentation_jump(this_instr, tstate, event, frame, src, dest); \ + LOAD_NEXT_OP_F(); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ next_instr = (dest)+1; \ @@ -372,6 +373,7 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { #define LOAD_IP(OFFSET) do { \ next_instr = frame->instr_ptr + (OFFSET); \ + LOAD_NEXT_OP_F(); \ } while (0) /* There's no STORE_IP(), it's inlined by the code generator. */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f21e20be7e335ad..f292b4819b87f0e 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -43,6 +43,7 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1258,6 +1259,7 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2539,6 +2541,7 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4278,6 +4281,7 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CompareOp(left, right, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4523,6 +4527,7 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ContainsOp(right, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5076,6 +5081,7 @@ opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; + LOAD_NEXT_OP_F(); if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } @@ -5228,6 +5234,7 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ForIter(iter, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6486,6 +6493,7 @@ JUMP_TO_LABEL(error); } next_instr = this_instr; + LOAD_NEXT_OP_F(); if (_PyOpcode_Caches[next_opcode]) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } @@ -6574,6 +6582,7 @@ } next_instr = frame->instr_ptr; if (next_instr != this_instr) { + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -6584,6 +6593,7 @@ PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; + LOAD_NEXT_OP_F(); DISPATCH_GOTO(); } @@ -6855,6 +6865,7 @@ // Make sure this_instr gets reset correctley for any uops that // follow next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); DISPATCH(); } #endif @@ -6903,6 +6914,7 @@ if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); } } DISPATCH(); @@ -6988,6 +7000,7 @@ } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -7105,6 +7118,7 @@ this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; // Need to re-dispatch so the warmup counter isn't off by one: next_instr = this_instr; + LOAD_NEXT_OP_F(); DISPATCH_SAME_OPARG(); } #endif @@ -7374,6 +7388,7 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8732,6 +8747,7 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9064,6 +9080,7 @@ int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9871,6 +9888,7 @@ // Make sure this_instr gets reset correctley for any uops that // follow next_instr = frame->instr_ptr; + LOAD_NEXT_OP_F(); DISPATCH(); } #endif @@ -10057,6 +10075,7 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Send(receiver, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10364,6 +10383,7 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10875,6 +10895,7 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreSubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11058,6 +11079,7 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ToBool(value, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11404,6 +11426,7 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; + LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); From 59962ada2e0272325206d1a4de10c4f87bfdc068 Mon Sep 17 00:00:00 2001 From: Andrew Consroe Date: Tue, 11 Feb 2025 15:55:40 -0600 Subject: [PATCH 4/7] wip, hitting bugs --- Python/bytecodes.c | 3 +- Python/ceval_macros.h | 13 +- Python/generated_cases.c.h | 442 +++++++++++------------ Tools/cases_generator/tier1_generator.py | 2 +- 4 files changed, 234 insertions(+), 226 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 70c3b85d49f6411..4e527676c387c66 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3036,7 +3036,8 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); + LOAD_NEXT_OP_F(); // TODO this is redundand with DISPATCH_SAME_ARG now, + // though maybe is good to load it before the following func call _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 9e6f4b2d0d09466..8c4c846cdacbf4f 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -73,10 +73,14 @@ #define TAIL_CALL_PARAMS _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate, _Py_CODEUNIT *next_instr, int oparg #define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, oparg +// 1st version looks like this where we load directly +// next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; +// 2nd version is like NEXTOPARG which does this atomic thing #ifdef Py_TAIL_CALL_INTERP # define LOAD_NEXT_OP_F() \ do { \ - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; \ + _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ + next_op_f = INSTRUCTION_TABLE[word.op.code]; \ } while (0) // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. # define Py_MUSTTAIL [[clang::musttail]] @@ -86,6 +90,7 @@ # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) # define DISPATCH_GOTO() \ do { \ + assert(next_op_f == INSTRUCTION_TABLE[opcode]); \ Py_MUSTTAIL return next_op_f(TAIL_CALL_ARGS); \ } while (0) # define JUMP_TO_LABEL(name) \ @@ -148,9 +153,11 @@ do { \ DISPATCH_GOTO(); \ } +// TODO better #define DISPATCH_SAME_OPARG() \ { \ opcode = next_instr->op.code; \ + next_op_f = INSTRUCTION_TABLE[opcode]; \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ } @@ -347,13 +354,13 @@ do { \ } else { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ next_instr = _Py_call_instrumentation_jump(this_instr, tstate, event, frame, src, dest); \ - LOAD_NEXT_OP_F(); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ next_instr = (dest)+1; \ JUMP_TO_LABEL(error); \ } \ } \ + LOAD_NEXT_OP_F(); \ } while (0); @@ -373,7 +380,7 @@ static inline void _Py_LeaveRecursiveCallPy(PyThreadState *tstate) { #define LOAD_IP(OFFSET) do { \ next_instr = frame->instr_ptr + (OFFSET); \ - LOAD_NEXT_OP_F(); \ + LOAD_NEXT_OP_F(); \ } while (0) /* There's no STORE_IP(), it's inlined by the code generator. */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f292b4819b87f0e..3a492220769e0ed 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -26,7 +26,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP); PREDICTED_BINARY_OP:; _Py_CODEUNIT* const this_instr = next_instr - 6; @@ -87,7 +87,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -143,7 +143,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -198,7 +198,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -253,7 +253,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_EXTEND); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -311,7 +311,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -396,7 +396,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -452,7 +452,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -507,7 +507,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef dict_st; @@ -556,7 +556,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef container; @@ -643,7 +643,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef list_st; @@ -715,7 +715,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef str_st; @@ -779,7 +779,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef tuple_st; @@ -839,7 +839,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -895,7 +895,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; @@ -948,7 +948,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BINARY_SLICE); _PyStackRef container; _PyStackRef start; @@ -1010,7 +1010,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_LIST); _PyStackRef *values; _PyStackRef list; @@ -1034,7 +1034,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_MAP); _PyStackRef *values; _PyStackRef map; @@ -1078,7 +1078,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_SET); _PyStackRef *values; _PyStackRef set; @@ -1128,7 +1128,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_SLICE); _PyStackRef *args; _PyStackRef slice; @@ -1160,7 +1160,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_STRING); _PyStackRef *pieces; _PyStackRef str; @@ -1199,7 +1199,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(BUILD_TUPLE); _PyStackRef *values; _PyStackRef tup; @@ -1223,7 +1223,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CACHE); assert(0 && "Executing a cache."); Py_FatalError("Executing a cache."); @@ -1238,7 +1238,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL); PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; @@ -1415,7 +1415,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1545,7 +1545,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1686,7 +1686,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1810,7 +1810,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -1904,7 +1904,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2004,7 +2004,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2105,7 +2105,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2203,7 +2203,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_FUNCTION_EX); opcode = CALL_FUNCTION_EX; _PyStackRef func; @@ -2395,7 +2395,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_INTRINSIC_1); _PyStackRef value; _PyStackRef res; @@ -2421,7 +2421,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_INTRINSIC_2); _PyStackRef value2_st; _PyStackRef value1_st; @@ -2456,7 +2456,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2517,7 +2517,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_KW); PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; @@ -2693,7 +2693,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; @@ -2828,7 +2828,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_KW_NON_PY); opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); @@ -2943,7 +2943,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3052,7 +3052,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3121,7 +3121,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -3187,7 +3187,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3293,7 +3293,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3399,7 +3399,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3505,7 +3505,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3613,7 +3613,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_NON_PY_GENERAL); opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); @@ -3718,7 +3718,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3831,7 +3831,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef *callable; @@ -3929,7 +3929,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -4004,7 +4004,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -4079,7 +4079,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -4123,7 +4123,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CHECK_EG_MATCH); _PyStackRef exc_value_st; _PyStackRef match_type_st; @@ -4180,7 +4180,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CHECK_EXC_MATCH); _PyStackRef left; _PyStackRef right; @@ -4216,7 +4216,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter_st; _PyStackRef last_sent_val_st; @@ -4264,7 +4264,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COMPARE_OP); PREDICTED_COMPARE_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -4338,7 +4338,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4392,7 +4392,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4458,7 +4458,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4511,7 +4511,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CONTAINS_OP); PREDICTED_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -4568,7 +4568,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4610,7 +4610,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef left; @@ -4651,7 +4651,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(CONVERT_VALUE); _PyStackRef value; _PyStackRef result; @@ -4685,7 +4685,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COPY); _PyStackRef bottom; _PyStackRef top; @@ -4706,7 +4706,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(COPY_FREE_VARS); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); @@ -4730,7 +4730,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_ATTR); _PyStackRef owner; owner = stack_pointer[-1]; @@ -4755,7 +4755,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_DEREF); PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); // Can't use ERROR_IF here. @@ -4781,7 +4781,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_FAST); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { @@ -4809,7 +4809,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4837,7 +4837,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_NAME); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); @@ -4872,7 +4872,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DELETE_SUBSCR); _PyStackRef container; _PyStackRef sub; @@ -4901,7 +4901,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DICT_MERGE); _PyStackRef callable; _PyStackRef dict; @@ -4936,7 +4936,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(DICT_UPDATE); _PyStackRef dict; _PyStackRef update; @@ -4977,7 +4977,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(END_ASYNC_FOR); _PyStackRef awaitable_st; _PyStackRef exc_st; @@ -5013,7 +5013,7 @@ py_tail_call_funcptr next_op_f; #endif next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(END_FOR); _PyStackRef value; value = stack_pointer[-1]; @@ -5038,7 +5038,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(END_SEND); _PyStackRef receiver; _PyStackRef value; @@ -5064,7 +5064,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 @@ -5104,7 +5104,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(EXIT_INIT_CHECK); _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; @@ -5130,7 +5130,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(EXTENDED_ARG); opcode = EXTENDED_ARG; assert(oparg); @@ -5148,7 +5148,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FORMAT_SIMPLE); _PyStackRef value; _PyStackRef res; @@ -5189,7 +5189,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FORMAT_WITH_SPEC); _PyStackRef value; _PyStackRef fmt_spec; @@ -5219,7 +5219,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FOR_ITER); PREDICTED_FOR_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -5290,7 +5290,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5358,7 +5358,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5421,7 +5421,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FOR_ITER_RANGE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5478,7 +5478,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; @@ -5536,7 +5536,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_AITER); _PyStackRef obj; _PyStackRef iter; @@ -5591,7 +5591,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_ANEXT); _PyStackRef aiter; _PyStackRef awaitable; @@ -5617,7 +5617,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_AWAITABLE); _PyStackRef iterable; _PyStackRef iter; @@ -5642,7 +5642,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_ITER); _PyStackRef iterable; _PyStackRef iter; @@ -5668,7 +5668,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_LEN); _PyStackRef obj; _PyStackRef len; @@ -5699,7 +5699,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(GET_YIELD_FROM_ITER); _PyStackRef iterable; _PyStackRef iter; @@ -5748,7 +5748,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(IMPORT_FROM); _PyStackRef from; _PyStackRef res; @@ -5775,7 +5775,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(IMPORT_NAME); _PyStackRef level; _PyStackRef fromlist; @@ -5810,7 +5810,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_CALL); opcode = INSTRUMENTED_CALL; _PyStackRef *callable; @@ -5995,7 +5995,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; @@ -6189,7 +6189,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); opcode = INSTRUMENTED_CALL_KW; _PyStackRef *callable; @@ -6370,7 +6370,7 @@ _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_END_FOR); _PyStackRef receiver; _PyStackRef value; @@ -6402,7 +6402,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; _PyStackRef value; @@ -6438,7 +6438,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); /* Skip 1 cache entry */ _PyStackRef iter_stackref = TOP(); @@ -6482,7 +6482,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6512,7 +6512,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -6545,7 +6545,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); @@ -6562,7 +6562,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_LINE); opcode = INSTRUMENTED_LINE; int original_opcode = 0; @@ -6607,7 +6607,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; @@ -6706,7 +6706,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); (void)this_instr; // INSTRUMENTED_JUMP requires this_instr INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); @@ -6724,7 +6724,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); _PyStackRef iter; iter = stack_pointer[-1]; @@ -6747,7 +6747,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); /* Skip 1 cache entry */ _PyStackRef cond = POP(); @@ -6770,7 +6770,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); /* Skip 1 cache entry */ _PyStackRef value_stackref = POP(); @@ -6797,7 +6797,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); /* Skip 1 cache entry */ _PyStackRef value_stackref = POP(); @@ -6822,7 +6822,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); /* Skip 1 cache entry */ _PyStackRef cond = POP(); @@ -6845,7 +6845,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_RESUME); // _LOAD_BYTECODE { @@ -6930,7 +6930,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); _PyStackRef val; _PyStackRef retval; @@ -6982,7 +6982,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); _PyStackRef val; _PyStackRef retval; @@ -7055,7 +7055,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; @@ -7080,7 +7080,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(IS_OP); _PyStackRef left; _PyStackRef right; @@ -7105,7 +7105,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(JUMP_BACKWARD); PREDICTED_JUMP_BACKWARD:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -7159,7 +7159,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(JUMP_BACKWARD_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ @@ -7232,7 +7232,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost @@ -7252,7 +7252,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ @@ -7290,7 +7290,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(JUMP_FORWARD); JUMPBY(oparg); DISPATCH(); @@ -7304,7 +7304,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LIST_APPEND); _PyStackRef list; _PyStackRef v; @@ -7328,7 +7328,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LIST_EXTEND); _PyStackRef list_st; _PyStackRef iterable_st; @@ -7371,7 +7371,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR); PREDICTED_LOAD_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 10; @@ -7460,7 +7460,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7514,7 +7514,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7578,7 +7578,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7638,7 +7638,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7717,7 +7717,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7776,7 +7776,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7824,7 +7824,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7894,7 +7894,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -7975,7 +7975,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -8018,7 +8018,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -8082,7 +8082,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -8179,7 +8179,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -8244,7 +8244,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; @@ -8348,7 +8348,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_BUILD_CLASS); _PyStackRef bc; PyObject *bc_o; @@ -8380,7 +8380,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; // Keep in sync with _common_constants in opcode.py @@ -8408,7 +8408,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_CONST); PREDICTED_LOAD_CONST:; _Py_CODEUNIT* const this_instr = next_instr - 1; @@ -8447,7 +8447,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_CONST_IMMORTAL); static_assert(0 == 0, "incorrect cache size"); _PyStackRef value; @@ -8468,7 +8468,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_CONST_MORTAL); static_assert(0 == 0, "incorrect cache size"); _PyStackRef value; @@ -8488,7 +8488,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_DEREF); _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -8514,7 +8514,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FAST); _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); @@ -8533,7 +8533,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); _PyStackRef value; value = GETLOCAL(oparg); @@ -8552,7 +8552,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FAST_CHECK); _PyStackRef value; _PyStackRef value_s = GETLOCAL(oparg); @@ -8580,7 +8580,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); _PyStackRef value1; _PyStackRef value2; @@ -8603,7 +8603,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); _PyStackRef class_dict_st; _PyStackRef value; @@ -8650,7 +8650,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); _PyStackRef mod_or_class_dict; _PyStackRef v; @@ -8732,7 +8732,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 5; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED_LOAD_GLOBAL:; _Py_CODEUNIT* const this_instr = next_instr - 5; @@ -8791,7 +8791,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *builtins_keys; @@ -8877,7 +8877,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyDictKeysObject *globals_keys; @@ -8945,7 +8945,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_LOCALS); _PyStackRef locals; PyObject *l = LOCALS(); @@ -8971,7 +8971,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_NAME); _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -8996,7 +8996,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_SMALL_INT); _PyStackRef value; assert(oparg < _PY_NSMALLPOSINTS); @@ -9016,7 +9016,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_SPECIAL); _PyStackRef owner; _PyStackRef attr; @@ -9059,7 +9059,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -9177,7 +9177,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; @@ -9230,7 +9230,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; @@ -9297,7 +9297,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MAKE_CELL); // "initial" is probably NULL but not if it's an arg (or set // via the f_locals proxy before MAKE_CELL has run). @@ -9322,7 +9322,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MAKE_FUNCTION); _PyStackRef codeobj_st; _PyStackRef func; @@ -9357,7 +9357,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MAP_ADD); _PyStackRef dict_st; _PyStackRef key; @@ -9392,7 +9392,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MATCH_CLASS); _PyStackRef subject; _PyStackRef type; @@ -9438,7 +9438,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MATCH_KEYS); _PyStackRef subject; _PyStackRef keys; @@ -9468,7 +9468,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MATCH_MAPPING); _PyStackRef subject; _PyStackRef res; @@ -9489,7 +9489,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(MATCH_SEQUENCE); _PyStackRef subject; _PyStackRef res; @@ -9510,7 +9510,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(NOP); DISPATCH(); } @@ -9523,7 +9523,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(NOT_TAKEN); DISPATCH(); } @@ -9536,7 +9536,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_EXCEPT); _PyStackRef exc_value; exc_value = stack_pointer[-1]; @@ -9559,7 +9559,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_ITER); _PyStackRef value; value = stack_pointer[-1]; @@ -9579,7 +9579,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_JUMP_IF_FALSE); _PyStackRef cond; /* Skip 1 cache entry */ @@ -9603,7 +9603,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_JUMP_IF_NONE); _PyStackRef value; _PyStackRef b; @@ -9643,7 +9643,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); _PyStackRef value; _PyStackRef b; @@ -9683,7 +9683,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_JUMP_IF_TRUE); _PyStackRef cond; /* Skip 1 cache entry */ @@ -9705,7 +9705,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(POP_TOP); _PyStackRef value; value = stack_pointer[-1]; @@ -9723,7 +9723,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(PUSH_EXC_INFO); _PyStackRef exc; _PyStackRef prev_exc; @@ -9754,7 +9754,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(PUSH_NULL); _PyStackRef res; res = PyStackRef_NULL; @@ -9774,7 +9774,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RAISE_VARARGS); _PyStackRef *args; args = &stack_pointer[-oparg]; @@ -9807,7 +9807,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RERAISE); _PyStackRef *values; _PyStackRef exc_st; @@ -9850,7 +9850,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RESERVED); assert(0 && "Executing RESERVED instruction."); Py_FatalError("Executing RESERVED instruction."); @@ -9865,7 +9865,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RESUME); PREDICTED_RESUME:; _Py_CODEUNIT* const this_instr = next_instr - 1; @@ -9946,7 +9946,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) @@ -9984,7 +9984,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); @@ -10025,7 +10025,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(RETURN_VALUE); _PyStackRef retval; _PyStackRef res; @@ -10059,7 +10059,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SEND); PREDICTED_SEND:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -10164,7 +10164,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; @@ -10234,7 +10234,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SETUP_ANNOTATIONS); PyObject *ann_dict; if (LOCALS() == NULL) { @@ -10283,7 +10283,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SET_ADD); _PyStackRef set; _PyStackRef v; @@ -10310,7 +10310,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); _PyStackRef attr_st; _PyStackRef func_in; @@ -10340,7 +10340,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SET_UPDATE); _PyStackRef set; _PyStackRef iterable; @@ -10367,7 +10367,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 5; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_ATTR); PREDICTED_STORE_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 5; @@ -10423,7 +10423,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; @@ -10501,7 +10501,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; @@ -10554,7 +10554,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; @@ -10654,7 +10654,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_DEREF); _PyStackRef v; v = stack_pointer[-1]; @@ -10675,7 +10675,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_FAST); _PyStackRef value; value = stack_pointer[-1]; @@ -10697,7 +10697,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); _PyStackRef value1; _PyStackRef value2; @@ -10722,7 +10722,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_FAST_STORE_FAST); _PyStackRef value2; _PyStackRef value1; @@ -10755,7 +10755,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_GLOBAL); _PyStackRef v; v = stack_pointer[-1]; @@ -10780,7 +10780,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_NAME); _PyStackRef v; v = stack_pointer[-1]; @@ -10822,7 +10822,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_SLICE); _PyStackRef v; _PyStackRef container; @@ -10878,7 +10878,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED_STORE_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -10934,7 +10934,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; @@ -10977,7 +10977,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; @@ -11043,7 +11043,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(SWAP); _PyStackRef *bottom; _PyStackRef *top; @@ -11064,7 +11064,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL); PREDICTED_TO_BOOL:; _Py_CODEUNIT* const this_instr = next_instr - 4; @@ -11115,7 +11115,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef owner; @@ -11154,7 +11154,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -11180,7 +11180,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_INT); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -11217,7 +11217,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -11248,7 +11248,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_NONE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -11278,7 +11278,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(TO_BOOL_STR); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; @@ -11314,7 +11314,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNARY_INVERT); _PyStackRef value; _PyStackRef res; @@ -11339,7 +11339,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNARY_NEGATIVE); _PyStackRef value; _PyStackRef res; @@ -11364,7 +11364,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNARY_NOT); _PyStackRef value; _PyStackRef res; @@ -11384,7 +11384,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNPACK_EX); _PyStackRef seq; _PyStackRef *right; @@ -11411,7 +11411,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED_UNPACK_SEQUENCE:; _Py_CODEUNIT* const this_instr = next_instr - 2; @@ -11465,7 +11465,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef seq; @@ -11514,7 +11514,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef seq; @@ -11554,7 +11554,7 @@ (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef seq; @@ -11593,7 +11593,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(WITH_EXCEPT_START); _PyStackRef exit_func; _PyStackRef exit_self; @@ -11653,7 +11653,7 @@ #endif frame->instr_ptr = next_instr; next_instr += 1; - next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; + LOAD_NEXT_OP_F(); INSTRUCTION_STATS(YIELD_VALUE); _PyStackRef retval; _PyStackRef value; diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index d240b6a338b9415..9ee213f62a6629b 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -246,7 +246,7 @@ def generate_tier1_cases( out.emit(f"frame->instr_ptr = next_instr;\n") out.emit(f"next_instr += {inst.size};\n") - out.emit(f"next_op_f = INSTRUCTION_TABLE[next_instr->op.code];\n") + out.emit(f"LOAD_NEXT_OP_F();\n") out.emit(f"INSTRUCTION_STATS({name});\n") if inst.is_target: out.emit(f"PREDICTED_{name}:;\n") From e9dd9c9102823112425a46cb9c41b550ea9f7188 Mon Sep 17 00:00:00 2001 From: Andrew Consroe Date: Wed, 12 Feb 2025 16:41:49 -0600 Subject: [PATCH 5/7] wip; trying to get LOAD_NEXT_OP_F where needed --- Python/bytecodes.c | 46 +++++++++++++----------- Python/ceval_macros.h | 14 +++++++- Python/generated_cases.c.h | 72 ++++++++++++++++++++++++++++---------- 3 files changed, 91 insertions(+), 41 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4e527676c387c66..66cfd9d995fcb4f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -157,6 +157,7 @@ dummy_func( if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { int err = _Py_HandlePending(tstate); ERROR_IF(err != 0, error); + LOAD_NEXT_OP_F(); } } @@ -167,6 +168,7 @@ dummy_func( if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { int err = _Py_HandlePending(tstate); ERROR_IF(err != 0, error); + LOAD_NEXT_OP_F(); } } } @@ -189,6 +191,7 @@ dummy_func( ERROR_NO_POP(); } next_instr = this_instr; + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -241,8 +244,8 @@ dummy_func( if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; - LOAD_NEXT_OP_F(); } + LOAD_NEXT_OP_F(); // maybe needs to be unconditional? } macro(INSTRUMENTED_RESUME) = @@ -364,6 +367,7 @@ dummy_func( pure inst(POP_TOP, (value --)) { DECREF_INPUTS(); + LOAD_NEXT_OP_F(); } pure inst(PUSH_NULL, (-- res)) { @@ -445,7 +449,6 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); } @@ -838,6 +841,7 @@ dummy_func( } else { err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + LOAD_NEXT_OP_F(); Py_DECREF(slice); } DECREF_INPUTS(); @@ -983,7 +987,6 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); } @@ -995,6 +998,7 @@ dummy_func( op(_STORE_SUBSCR, (v, container, sub -- )) { /* container[sub] = v */ int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + LOAD_NEXT_OP_F(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1183,7 +1187,6 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); } @@ -1239,6 +1242,7 @@ dummy_func( } PyStackRef_CLOSE(v); retval = PyStackRef_FromPyObjectSteal(retval_o); + LOAD_NEXT_OP_F(); } macro(SEND) = _SPECIALIZE_SEND + _SEND; @@ -1424,6 +1428,7 @@ dummy_func( } else { err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + LOAD_NEXT_OP_F(); } DECREF_INPUTS(); ERROR_IF(err, error); @@ -1458,7 +1463,6 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -1536,7 +1540,6 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_StoreAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -1549,6 +1552,8 @@ dummy_func( PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + // SetAttr can call into specialize + LOAD_NEXT_OP_F(); DECREF_INPUTS(); ERROR_IF(err, error); } @@ -1654,7 +1659,6 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); DISPATCH_SAME_OPARG(); } @@ -1768,6 +1772,7 @@ dummy_func( _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; PyStackRef_XCLOSE(tmp); + LOAD_NEXT_OP_F(); } inst(MAKE_CELL, (--)) { @@ -1959,6 +1964,7 @@ dummy_func( ERROR_IF(ann_dict == NULL, error); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + LOAD_NEXT_OP_F(); Py_DECREF(ann_dict); ERROR_IF(err, error); } @@ -2027,7 +2033,6 @@ dummy_func( int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); DISPATCH_SAME_OPARG(); } @@ -2151,7 +2156,6 @@ dummy_func( if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_LoadAttr(owner, next_instr, name); DISPATCH_SAME_OPARG(); } @@ -2191,6 +2195,7 @@ dummy_func( else { /* Classic, pushes one value. */ attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + LOAD_NEXT_OP_F(); DECREF_INPUTS(); ERROR_IF(attr_o == NULL, error); } @@ -2558,7 +2563,6 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); } @@ -2680,7 +2684,6 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_ContainsOp(right, next_instr); DISPATCH_SAME_OPARG(); } @@ -2790,7 +2793,6 @@ dummy_func( this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; // Need to re-dispatch so the warmup counter isn't off by one: next_instr = this_instr; - LOAD_NEXT_OP_F(); DISPATCH_SAME_OPARG(); } #endif @@ -2873,9 +2875,9 @@ dummy_func( * involving _RESUME_CHECK */ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; + LOAD_NEXT_OP_F_OPCODE(); oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; - LOAD_NEXT_OP_F(); if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } @@ -2983,6 +2985,7 @@ dummy_func( inst(GET_ITER, (iterable -- iter)) { /* before: [obj]; after [getiter(obj)] */ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + LOAD_NEXT_OP_F(); DECREF_INPUTS(); ERROR_IF(iter_o == NULL, error); iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -3036,9 +3039,7 @@ dummy_func( #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); // TODO this is redundand with DISPATCH_SAME_ARG now, - // though maybe is good to load it before the following func call - _Py_Specialize_ForIter(iter, next_instr, oparg); + _Py_Specialize_ForIter(iter, next_instr, oparg); // <- this can modify next_instr! DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -3333,6 +3334,7 @@ dummy_func( (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } pseudo(SETUP_FINALLY, (-- unused), (HAS_ARG)) = { @@ -3506,7 +3508,6 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); DISPATCH_SAME_OPARG(); } @@ -3592,6 +3593,8 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); + // when vectorcall calls into eg sys_settrace, our code can be modified + LOAD_NEXT_OP_F(); } op(_MONITOR_CALL, (func[1], maybe_self[1], args[oparg] -- func[1], maybe_self[1], args[oparg])) { @@ -3728,6 +3731,7 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } macro(CALL_NON_PY_GENERAL) = @@ -4044,6 +4048,7 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } macro(CALL_BUILTIN_FAST) = @@ -4517,7 +4522,6 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); DISPATCH_SAME_OPARG(); } @@ -4800,7 +4804,6 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); } @@ -4859,7 +4862,7 @@ dummy_func( PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; - LOAD_NEXT_OP_F(); + LOAD_NEXT_OP_F_OPCODE(); DISPATCH_GOTO(); } @@ -4868,12 +4871,12 @@ dummy_func( tstate, frame, this_instr); ERROR_IF(next_opcode < 0, error); next_instr = this_instr; - LOAD_NEXT_OP_F(); if (_PyOpcode_Caches[next_opcode]) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; + LOAD_NEXT_OP_F_OPCODE(); DISPATCH_GOTO(); } @@ -4940,6 +4943,7 @@ dummy_func( tier1 inst(EXTENDED_ARG, ( -- )) { assert(oparg); opcode = next_instr->op.code; + LOAD_NEXT_OP_F_OPCODE(); oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 8c4c846cdacbf4f..835c3fab227aa9e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -76,12 +76,17 @@ // 1st version looks like this where we load directly // next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; // 2nd version is like NEXTOPARG which does this atomic thing +// we also need a version that loads from opcode directly b/c a few things write to it directly #ifdef Py_TAIL_CALL_INTERP # define LOAD_NEXT_OP_F() \ do { \ _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ next_op_f = INSTRUCTION_TABLE[word.op.code]; \ } while (0) +# define LOAD_NEXT_OP_F_OPCODE() \ + do { \ + next_op_f = INSTRUCTION_TABLE[opcode]; \ + } while (0) // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. # define Py_MUSTTAIL [[clang::musttail]] # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) @@ -90,6 +95,13 @@ # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) # define DISPATCH_GOTO() \ do { \ + if (next_op_f != INSTRUCTION_TABLE[opcode]) { \ + int myopcode=-1; \ + for (int i = 0; i < 256; i++) { \ + if (next_op_f == INSTRUCTION_TABLE[i]) { myopcode = i; break; } \ + } \ + printf("yo expected %p but got %p used opcode=%d but is really opcode=%d\n", INSTRUCTION_TABLE[opcode], next_op_f, myopcode, opcode); \ + } \ assert(next_op_f == INSTRUCTION_TABLE[opcode]); \ Py_MUSTTAIL return next_op_f(TAIL_CALL_ARGS); \ } while (0) @@ -153,7 +165,7 @@ do { \ DISPATCH_GOTO(); \ } -// TODO better +// TODO conditional on tail call #define DISPATCH_SAME_OPARG() \ { \ opcode = next_instr->op.code; \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3a492220769e0ed..6f193c2ecd4f4bf 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -43,7 +43,6 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1259,7 +1258,6 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1380,6 +1378,8 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + // when vectorcall calls into eg sys_settrace, our code can be modified + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -1395,6 +1395,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1884,6 +1885,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -1969,6 +1971,7 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -1984,6 +1987,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2085,6 +2089,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -2183,6 +2188,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -2377,6 +2383,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -2541,7 +2548,6 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2923,6 +2929,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -3273,6 +3280,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -3379,6 +3387,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -3485,6 +3494,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -3593,6 +3603,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -3683,6 +3694,7 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -3698,6 +3710,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -3984,6 +3997,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -4059,6 +4073,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -4281,7 +4296,6 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CompareOp(left, right, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4527,7 +4541,6 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ContainsOp(right, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4798,6 +4811,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); DISPATCH(); } @@ -5079,9 +5093,9 @@ * involving _RESUME_CHECK */ if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; + LOAD_NEXT_OP_F_OPCODE(); oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; - LOAD_NEXT_OP_F(); if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } @@ -5135,6 +5149,7 @@ opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; + LOAD_NEXT_OP_F_OPCODE(); oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); @@ -5234,10 +5249,10 @@ #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ForIter(iter, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + // <- this can modify next_instr! DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -5651,6 +5666,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); PyStackRef_CLOSE(iterable); if (iter_o == NULL) { JUMP_TO_LABEL(pop_1_error); @@ -5960,6 +5976,8 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + // when vectorcall calls into eg sys_settrace, our code can be modified + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -5975,6 +5993,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += 1 + oparg; assert(WITHIN_STACK_BOUNDS()); } @@ -6169,6 +6188,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); } @@ -6493,12 +6513,12 @@ JUMP_TO_LABEL(error); } next_instr = this_instr; - LOAD_NEXT_OP_F(); if (_PyOpcode_Caches[next_opcode]) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; + LOAD_NEXT_OP_F_OPCODE(); DISPATCH_GOTO(); } @@ -6526,6 +6546,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } // _MONITOR_JUMP_BACKWARD @@ -6593,7 +6614,7 @@ PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; - LOAD_NEXT_OP_F(); + LOAD_NEXT_OP_F_OPCODE(); DISPATCH_GOTO(); } @@ -6883,6 +6904,7 @@ JUMP_TO_LABEL(error); } next_instr = this_instr; + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -6899,6 +6921,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } } @@ -6914,8 +6937,8 @@ if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ next_instr = frame->instr_ptr; - LOAD_NEXT_OP_F(); } + LOAD_NEXT_OP_F(); // maybe needs to be unconditional? } DISPATCH(); } @@ -7118,7 +7141,6 @@ this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; // Need to re-dispatch so the warmup counter isn't off by one: next_instr = this_instr; - LOAD_NEXT_OP_F(); DISPATCH_SAME_OPARG(); } #endif @@ -7134,6 +7156,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -7174,6 +7197,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -7267,6 +7291,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -7388,7 +7413,6 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7437,6 +7461,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); PyStackRef_CLOSE(owner); if (attr_o == NULL) { JUMP_TO_LABEL(pop_1_error); @@ -8747,7 +8772,6 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9080,7 +9104,6 @@ int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9564,6 +9587,7 @@ _PyStackRef value; value = stack_pointer[-1]; PyStackRef_CLOSE(value); + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -9710,6 +9734,7 @@ _PyStackRef value; value = stack_pointer[-1]; PyStackRef_CLOSE(value); + LOAD_NEXT_OP_F(); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -9906,6 +9931,7 @@ JUMP_TO_LABEL(error); } next_instr = this_instr; + LOAD_NEXT_OP_F(); DISPATCH(); } } @@ -9930,6 +9956,7 @@ if (err != 0) { JUMP_TO_LABEL(error); } + LOAD_NEXT_OP_F(); } } } @@ -10075,7 +10102,6 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Send(receiver, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10147,6 +10173,7 @@ PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); retval = PyStackRef_FromPyObjectSteal(retval_o); + LOAD_NEXT_OP_F(); } stack_pointer[0] = retval; stack_pointer += 1; @@ -10261,6 +10288,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { @@ -10383,7 +10413,6 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10402,6 +10431,8 @@ int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); + // SetAttr can call into specialize + LOAD_NEXT_OP_F(); PyStackRef_CLOSE(v); PyStackRef_CLOSE(owner); if (err) { @@ -10804,6 +10835,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); } PyStackRef_CLOSE(v); if (err) { @@ -10854,6 +10886,9 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); + _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(slice); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += 2; @@ -10895,7 +10930,6 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreSubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10912,6 +10946,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_NEXT_OP_F(); PyStackRef_CLOSE(v); PyStackRef_CLOSE(container); PyStackRef_CLOSE(sub); @@ -11079,7 +11114,6 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ToBool(value, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11426,7 +11460,6 @@ #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - LOAD_NEXT_OP_F(); _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11639,6 +11672,7 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); From e2d9b28d86ef98b688f1d17b8b32efbfc0e0945b Mon Sep 17 00:00:00 2001 From: Andrew Consroe Date: Wed, 12 Feb 2025 16:44:47 -0600 Subject: [PATCH 6/7] and a few more --- Python/bytecodes.c | 2 ++ Python/generated_cases.c.h | 3 +++ 2 files changed, 5 insertions(+) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 66cfd9d995fcb4f..6429cf8b99a9bff 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4326,6 +4326,7 @@ dummy_func( DECREF_INPUTS(); ERROR_IF(res_o == NULL, error); res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } macro(CALL_METHOD_DESCRIPTOR_FAST) = @@ -4690,6 +4691,7 @@ dummy_func( PyStackRef_CLOSE(func_st); ERROR_IF(result_o == NULL, error); result = PyStackRef_FromPyObjectSteal(result_o); + LOAD_NEXT_OP_F(); } macro(CALL_FUNCTION_EX) = diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6f193c2ecd4f4bf..26e4596c9c69951 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2368,6 +2368,7 @@ JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -3265,6 +3266,7 @@ JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { @@ -6173,6 +6175,7 @@ JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); + LOAD_NEXT_OP_F(); } // _CHECK_PERIODIC { From e9c43a0088c535b4b8c1f72b8ea7ae0e6c46df7d Mon Sep 17 00:00:00 2001 From: Andrew Consroe Date: Wed, 12 Feb 2025 17:09:18 -0600 Subject: [PATCH 7/7] remove deubgging loop --- Python/ceval_macros.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 835c3fab227aa9e..34cf50d12652abd 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -77,6 +77,14 @@ // next_op_f = INSTRUCTION_TABLE[next_instr->op.code]; // 2nd version is like NEXTOPARG which does this atomic thing // we also need a version that loads from opcode directly b/c a few things write to it directly + + /*if (next_op_f != INSTRUCTION_TABLE[opcode]) { \*/ + /* int myopcode=-1; \*/ + /* for (int i = 0; i < 256; i++) { \*/ + /* if (next_op_f == INSTRUCTION_TABLE[i]) { myopcode = i; break; } \*/ + /* } \*/ + /* printf("yo expected %p but got %p used opcode=%d but is really opcode=%d\n", INSTRUCTION_TABLE[opcode], next_op_f, myopcode, opcode); \*/ + /*} \*/ #ifdef Py_TAIL_CALL_INTERP # define LOAD_NEXT_OP_F() \ do { \ @@ -95,13 +103,6 @@ # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) # define DISPATCH_GOTO() \ do { \ - if (next_op_f != INSTRUCTION_TABLE[opcode]) { \ - int myopcode=-1; \ - for (int i = 0; i < 256; i++) { \ - if (next_op_f == INSTRUCTION_TABLE[i]) { myopcode = i; break; } \ - } \ - printf("yo expected %p but got %p used opcode=%d but is really opcode=%d\n", INSTRUCTION_TABLE[opcode], next_op_f, myopcode, opcode); \ - } \ assert(next_op_f == INSTRUCTION_TABLE[opcode]); \ Py_MUSTTAIL return next_op_f(TAIL_CALL_ARGS); \ } while (0)