/
githubmirror
/
cpython
Обзор
Документация
Войти
/
githubmirror
/
cpython
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
main
Modules/_testinternalcapi/test_cases.c.h
13 898 строк
573 KB
Dino Viehland
gh-150490: Raise PyType_Modified for insertion into split dictionary (take 2) (#152914)
06 июл 2026, 22:56
Не верифицирован
06 июл 2026, 22:56
2092192
Код
Авторство
О чём код?
// This file is generated by Tools/cases_generator/tier1_generator.py // from: // Python/bytecodes.c, Modules/_testinternalcapi/testbytecodes.c // Do not edit! #ifdef TIER_TWO #error "This file is for Tier 1 only" #endif #define TIER_ONE 1 #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: switch (dispatch_code) #endif { #endif /* _Py_TAIL_CALL_INTERP */ /* BEGIN INSTRUCTIONS */ TARGET(BINARY_OP) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); PREDICTED_BINARY_OP:; _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; _PyStackRef lhs; _PyStackRef rhs; _PyStackRef res; _PyStackRef l; _PyStackRef r; _PyStackRef value; // _SPECIALIZE_BINARY_OP { rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(BINARY_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_OPARG_LAST); } /* Skip 4 cache entries */ // _BINARY_OP { PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); l = lhs; r = rhs; } // _POP_TOP { value = r; stack_pointer[-2] = res; stack_pointer[-1] = l; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = l; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(BINARY_OP_ADD_FLOAT) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_ADD_FLOAT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; // _GUARD_TOS_FLOAT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_FLOAT { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_FLOAT { right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval + ((PyFloatObject *)right_o)->ob_fval; PyObject *d = PyFloat_FromDouble(dres); if (d == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } // _POP_TOP_FLOAT { value = r; assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); } // _POP_TOP_FLOAT { value = l; assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_ADD_INT) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_ADD_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; // _GUARD_TOS_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_INT { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_INT { right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } l = left; r = right; } // _POP_TOP_INT { value = r; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } // _POP_TOP_INT { value = l; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_ADD_UNICODE) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_ADD_UNICODE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; // _GUARD_TOS_UNICODE { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_UNICODE { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_ADD_UNICODE { right = value; left = nos; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyUnicode_CheckExact(left_o)); assert(PyUnicode_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); PyObject *res_o = PyUnicode_Concat(left_o, right_o); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); l = left; r = right; } // _POP_TOP_UNICODE { value = r; assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); } // _POP_TOP_UNICODE { value = l; assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_EXTEND) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_EXTEND; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_EXTEND); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; _PyStackRef value; /* Skip 1 cache entry */ // _GUARD_BINARY_OP_EXTEND { right = stack_pointer[-1]; left = stack_pointer[-2]; PyObject *descr = read_obj(&this_instr[2].cache); PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); assert(d != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int match = (d->guard != NULL) ? d->guard(left_o, right_o) : (Py_TYPE(left_o) == d->lhs_type && Py_TYPE(right_o) == d->rhs_type); _PyFrame_StackPointerInvalidate(frame); if (!match) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip -4 cache entry */ // _BINARY_OP_EXTEND { PyObject *descr = read_obj(&this_instr[2].cache); PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; STAT_INC(BINARY_OP, hit); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject *res_o = d->action(left_o, right_o); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } assert(d->result_type == NULL || Py_TYPE(res_o) == d->result_type); assert(!d->result_unique || Py_REFCNT(res_o) == 1 || _Py_IsImmortal(res_o)); assert(!PyFloat_CheckExact(res_o) || Py_REFCNT(res_o) == 1); res = PyStackRef_FromPyObjectSteal(res_o); l = left; r = right; } // _POP_TOP { value = r; stack_pointer[-2] = res; stack_pointer[-1] = l; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = l; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_INPLACE_ADD_UNICODE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; _PyStackRef left; _PyStackRef right; _PyStackRef res; // _GUARD_TOS_UNICODE { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_UNICODE { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_INPLACE_ADD_UNICODE { right = value; left = nos; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); assert(PyUnicode_CheckExact(left_o)); assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); int next_oparg; #if TIER_ONE assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else next_oparg = (int)CURRENT_OPERAND0_16(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); assert(PyUnicode_CheckExact(left_o)); if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyObject *right_o = PyStackRef_AsPyObjectSteal(right); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyUnicode_Append(&temp, right_o); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); _PyFrame_StackPointerInvalidate(frame); *target_local = PyStackRef_NULL; if (temp == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(temp); } stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_FLOAT) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_MULTIPLY_FLOAT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; // _GUARD_TOS_FLOAT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_FLOAT { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_FLOAT { right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval * ((PyFloatObject *)right_o)->ob_fval; PyObject *d = PyFloat_FromDouble(dres); if (d == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } // _POP_TOP_FLOAT { value = r; assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); } // _POP_TOP_FLOAT { value = l; assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_INT) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_MULTIPLY_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; // _GUARD_TOS_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_INT { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_MULTIPLY_INT { right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } l = left; r = right; } // _POP_TOP_INT { value = r; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } // _POP_TOP_INT { value = l; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_SUBSCR_DICT) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_DICT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef nos; _PyStackRef dict_st; _PyStackRef sub_st; _PyStackRef res; _PyStackRef ds; _PyStackRef ss; _PyStackRef value; // _GUARD_NOS_DICT_SUBSCRIPT { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!Py_TYPE(o)->tp_as_mapping) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } if (Py_TYPE(o)->tp_as_mapping->mp_subscript != _PyDict_Subscript) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_SUBSCR_DICT { sub_st = stack_pointer[-1]; dict_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(Py_TYPE(dict)->tp_as_mapping->mp_subscript == _PyDict_Subscript); STAT_INC(BINARY_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyDict_Subscript(dict, sub); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); ds = dict_st; ss = sub_st; } // _POP_TOP { value = ss; stack_pointer[-2] = res; stack_pointer[-1] = ds; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = ds; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(BINARY_OP_SUBSCR_GETITEM) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_GETITEM; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef container; _PyStackRef getitem; _PyStackRef sub; _PyStackRef new_frame; /* Skip 5 cache entries */ // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _BINARY_OP_SUBSCR_CHECK_FUNC { container = stack_pointer[-2]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); if (getitem_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } getitem = PyStackRef_FromPyObjectNew(getitem_o); } // _BINARY_OP_SUBSCR_INIT_CALL { sub = stack_pointer[-1]; STAT_INC(BINARY_OP, hit); _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); pushed_frame->localsplus[0] = container; pushed_frame->localsplus[1] = sub; frame->return_offset = 6u ; new_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(BINARY_OP_SUBSCR_LIST_INT) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_LIST_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; _PyStackRef list_st; _PyStackRef sub_st; _PyStackRef res; _PyStackRef ls; _PyStackRef ss; // _GUARD_TOS_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_LIST { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_SUBSCR_LIST_INT { sub_st = value; list_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PyLong_CheckExact(sub)); assert(PyList_CheckExact(list)); Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); if (index < 0) { index += PyList_GET_SIZE(list); } #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } res = PyStackRef_FromPyObjectSteal(res_o); #else if (index < 0 || index >= PyList_GET_SIZE(list)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(BINARY_OP, hit); ls = list_st; ss = sub_st; } // _POP_TOP_INT { value = ss; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } // _POP_TOP { value = ls; stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_LIST_SLICE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef tos; _PyStackRef nos; _PyStackRef list_st; _PyStackRef sub_st; _PyStackRef res; _PyStackRef ls; _PyStackRef ss; _PyStackRef value; // _GUARD_TOS_SLICE { tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PySlice_Check(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_LIST { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_SUBSCR_LIST_SLICE { sub_st = tos; list_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PySlice_Check(sub)); assert(PyList_CheckExact(list)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_SliceSubscript(list, sub); _PyFrame_StackPointerInvalidate(frame); STAT_INC(BINARY_OP, hit); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); ls = list_st; ss = sub_st; } // _POP_TOP { value = ss; stack_pointer[-2] = res; stack_pointer[-1] = ls; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = ls; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(BINARY_OP_SUBSCR_STR_INT) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_STR_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; _PyStackRef str_st; _PyStackRef sub_st; _PyStackRef res; _PyStackRef s; _PyStackRef i; // _GUARD_TOS_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_COMPACT_ASCII { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } if (!PyUnicode_IS_COMPACT_ASCII(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_SUBSCR_STR_INT { sub_st = value; str_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); assert(PyLong_CheckExact(sub)); assert(PyUnicode_CheckExact(str)); if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (PyUnicode_GET_LENGTH(str) <= index) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } uint8_t c = PyUnicode_1BYTE_DATA(str)[index]; assert(c < 128); STAT_INC(BINARY_OP, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; s = str_st; i = sub_st; res = PyStackRef_FromPyObjectBorrow(res_o); } // _POP_TOP_INT { value = i; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } // _POP_TOP_UNICODE { value = s; assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_TUPLE_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; _PyStackRef tuple_st; _PyStackRef sub_st; _PyStackRef res; _PyStackRef ts; _PyStackRef ss; // _GUARD_TOS_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_TUPLE { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS { sub_st = value; tuple_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (index >= PyTuple_GET_SIZE(tuple)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_SUBSCR_TUPLE_INT { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); STAT_INC(BINARY_OP, hit); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); ts = tuple_st; ss = sub_st; } // _POP_TOP_INT { value = ss; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } // _POP_TOP { value = ts; stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(BINARY_OP_SUBSCR_USTR_INT) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_USTR_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_USTR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; _PyStackRef str_st; _PyStackRef sub_st; _PyStackRef res; _PyStackRef s; _PyStackRef i; // _GUARD_TOS_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_UNICODE { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_SUBSCR_USTR_INT { sub_st = value; str_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); assert(PyLong_CheckExact(sub)); assert(PyUnicode_CheckExact(str)); if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (PyUnicode_GET_LENGTH(str) <= index) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } Py_UCS4 c = PyUnicode_READ_CHAR(str, index); if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; s = str_st; i = sub_st; res = PyStackRef_FromPyObjectBorrow(res_o); } // _POP_TOP_INT { value = i; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } // _POP_TOP_UNICODE { value = s; assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_FLOAT) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBTRACT_FLOAT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; // _GUARD_TOS_FLOAT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_FLOAT { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_FLOAT { right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyFloat_CheckExact(left_o)); assert(PyFloat_CheckExact(right_o)); STAT_INC(BINARY_OP, hit); double dres = ((PyFloatObject *)left_o)->ob_fval - ((PyFloatObject *)right_o)->ob_fval; PyObject *d = PyFloat_FromDouble(dres); if (d == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(d); l = left; r = right; } // _POP_TOP_FLOAT { value = r; assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); } // _POP_TOP_FLOAT { value = l; assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_INT) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBTRACT_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; // _GUARD_TOS_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } // _GUARD_NOS_INT { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } } /* Skip 5 cache entries */ // _BINARY_OP_SUBTRACT_INT { right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyLong_CheckExact(left_o)); assert(PyLong_CheckExact(right_o)); assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } l = left; r = right; } // _POP_TOP_INT { value = r; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } // _POP_TOP_INT { value = l; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BINARY_SLICE) { #if _Py_TAIL_CALL_INTERP int opcode = BINARY_SLICE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); _PyStackRef container; _PyStackRef start; _PyStackRef stop; _PyStackRef res; // _SPECIALIZE_BINARY_SLICE { #if ENABLE_SPECIALIZATION OPCODE_DEFERRED_INC(BINARY_SLICE); #endif /* ENABLE_SPECIALIZATION */ } // _BINARY_SLICE { stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; PyObject *container_o = PyStackRef_AsPyObjectBorrow(container); PyObject *start_o = PyStackRef_AsPyObjectBorrow(start); PyObject *stop_o = PyStackRef_AsPyObjectBorrow(stop); PyObject *res_o; if (PyList_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); res_o = _PyList_BinarySlice(container_o, start_o, stop_o); _PyFrame_StackPointerInvalidate(frame); } else if (PyTuple_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); res_o = _PyTuple_BinarySlice(container_o, start_o, stop_o); _PyFrame_StackPointerInvalidate(frame); } else if (PyUnicode_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); res_o = _PyUnicode_BinarySlice(container_o, start_o, stop_o); _PyFrame_StackPointerInvalidate(frame); } else { PyObject *slice = PySlice_New(start_o, stop_o, NULL); if (slice == NULL) { res_o = NULL; } else { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); res_o = PyObject_GetItem(container_o, slice); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); _PyFrame_StackPointerInvalidate(frame); } } _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = stop; stop = PyStackRef_NULL; stack_pointer[-1] = stop; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = start; start = PyStackRef_NULL; stack_pointer[-2] = start; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_INTERPOLATION) { #if _Py_TAIL_CALL_INTERP int opcode = BUILD_INTERPOLATION; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_INTERPOLATION); _PyStackRef value; _PyStackRef str; _PyStackRef *format; _PyStackRef interpolation; format = &stack_pointer[-(oparg & 1)]; str = stack_pointer[-1 - (oparg & 1)]; value = stack_pointer[-2 - (oparg & 1)]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); int conversion = oparg >> 2; PyObject *format_o; if (oparg & 1) { format_o = PyStackRef_AsPyObjectBorrow(format[0]); } else { format_o = &_Py_STR(empty); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); _PyFrame_StackPointerInvalidate(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(format[0]); _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer += -(oparg & 1); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(str); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); _PyFrame_StackPointerInvalidate(frame); if (interpolation_o == NULL) { JUMP_TO_LABEL(error); } interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); stack_pointer[0] = interpolation; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_LIST) { #if _Py_TAIL_CALL_INTERP int opcode = BUILD_LIST; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); _PyStackRef *values; _PyStackRef list; values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); _PyFrame_StackPointerInvalidate(frame); if (list_o == NULL) { JUMP_TO_LABEL(error); } list = PyStackRef_FromPyObjectStealMortal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_MAP) { #if _Py_TAIL_CALL_INTERP int opcode = BUILD_MAP; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); _PyStackRef *values; _PyStackRef map; values = &stack_pointer[-oparg*2]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); _PyFrame_StackPointerInvalidate(frame); if (map_o == NULL) { stack_pointer += -oparg*2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } map = PyStackRef_FromPyObjectStealMortal(map_o); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_SET) { #if _Py_TAIL_CALL_INTERP int opcode = BUILD_SET; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); _PyStackRef *values; _PyStackRef set; values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *set_o = PySet_New(NULL); _PyFrame_StackPointerInvalidate(frame); if (set_o == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = values[_i]; values[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } int err = 0; for (Py_ssize_t i = 0; i < oparg; i++) { _PyStackRef value = values[i]; values[i] = PyStackRef_NULL; if (err == 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); _PyFrame_StackPointerInvalidate(frame); } else { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); _PyFrame_StackPointerInvalidate(frame); } } if (err) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); Py_DECREF(set_o); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectStealMortal(set_o); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_SLICE) { #if _Py_TAIL_CALL_INTERP int opcode = BUILD_SLICE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); _PyStackRef *args; _PyStackRef slice; args = &stack_pointer[-oparg]; PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = args[_i]; args[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { JUMP_TO_LABEL(error); } slice = PyStackRef_FromPyObjectStealMortal(slice_o); stack_pointer[0] = slice; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_STRING) { #if _Py_TAIL_CALL_INTERP int opcode = BUILD_STRING; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); _PyStackRef *pieces; _PyStackRef str; pieces = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); _PyFrame_StackPointerInvalidate(frame); if (str_o == NULL) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_TEMPLATE) { #if _Py_TAIL_CALL_INTERP int opcode = BUILD_TEMPLATE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TEMPLATE); _PyStackRef strings; _PyStackRef interpolations; _PyStackRef template; interpolations = stack_pointer[-1]; strings = stack_pointer[-2]; PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(interpolations); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(strings); _PyFrame_StackPointerInvalidate(frame); if (template_o == NULL) { JUMP_TO_LABEL(error); } template = PyStackRef_FromPyObjectSteal(template_o); stack_pointer[0] = template; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(BUILD_TUPLE) { #if _Py_TAIL_CALL_INTERP int opcode = BUILD_TUPLE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); _PyStackRef *values; _PyStackRef tup; values = &stack_pointer[-oparg]; PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); if (tup_o == NULL) { JUMP_TO_LABEL(error); } tup = PyStackRef_FromPyObjectStealMortal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CACHE) { #if _Py_TAIL_CALL_INTERP int opcode = CACHE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); assert(0 && "Executing a cache."); Py_FatalError("Executing a cache."); DISPATCH(); } TARGET(CALL) { #if _Py_TAIL_CALL_INTERP int opcode = CALL; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; opcode = CALL; _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef res; // _SPECIALIZE_CALL { self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD { if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; self_or_null = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL { args = &stack_pointer[-oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } if (Py_TYPE(callable_o) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, total_args, PyStackRef_NULL, opcode == INSTRUMENTED_CALL, frame, this_instr, tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_ALLOC_AND_ENTER_INIT) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_ALLOC_AND_ENTER_INIT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef init; _PyStackRef self; _PyStackRef *args; _PyStackRef init_frame; _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_OBJECT { self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_RECURSION_REMAINING { if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _ALLOCATE_OBJECT { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyStackRef_IsNull(self_or_null)); assert(PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); assert(tp->tp_alloc == PyType_GenericAlloc); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *self_o = PyType_GenericAlloc(tp, 0); _PyFrame_StackPointerInvalidate(frame); if (self_o == NULL) { JUMP_TO_LABEL(error); } self_or_null = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(init_func); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } // _CREATE_INIT_FRAME { args = &stack_pointer[-oparg]; self = self_or_null; init = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); _PyFrame_StackPointerInvalidate(frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); shim->localsplus[0] = PyStackRef_DUP(self); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init, NULL, args-1, oparg+1, NULL, shim); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, shim); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; tstate->py_recursion_remaining--; init_frame = PyStackRef_Wrap(temp); } // _PUSH_FRAME { new_frame = init_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_BOUND_METHOD_EXACT_ARGS; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS { null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { self_or_null = null; assert(PyStackRef_IsNull(self_or_null)); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); STAT_INC(CALL, hit); self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } // flush // _CHECK_FUNCTION_VERSION { uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_FUNCTION_EXACT_ARGS { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_RECURSION_REMAINING { if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _INIT_CALL_PY_EXACT_ARGS { args = &stack_pointer[-oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); } // _SAVE_RETURN_OFFSET { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; #endif } // _PUSH_FRAME { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(CALL_BOUND_METHOD_GENERAL) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_BOUND_METHOD_GENERAL; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_METHOD_VERSION { null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _EXPAND_METHOD { self_or_null = null; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyStackRef_IsNull(self_or_null)); assert(Py_TYPE(callable_o) == &PyMethod_Type); self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } // flush // _CHECK_RECURSION_REMAINING { if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _PY_FRAME_GENERAL { args = &stack_pointer[-oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; #endif } // _PUSH_FRAME { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(CALL_BUILTIN_CLASS) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_CLASS; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_CLASS { callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; if (tp->tp_vectorcall == NULL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_BUILTIN_CLASS { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_CallBuiltinClass_StackRef( callable, arguments, total_args); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_BUILTIN_FAST) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_FAST; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_FAST { callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_BUILTIN_FAST { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFast_StackRef( callable, arguments, total_args ); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS { callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_BUILTIN_FAST_WITH_KEYWORDS { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRef(callable, arguments, total_args); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_BUILTIN_O) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_O; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef res; _PyStackRef c; _PyStackRef s; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_BUILTIN_O { self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; } if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_RECURSION_LIMIT { if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_BUILTIN_O { args = &stack_pointer[-oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { args--; } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { JUMP_TO_LABEL(error); } c = callable; s = args[0]; res = PyStackRef_FromPyObjectSteal(res_o); } // _POP_TOP { value = s; stack_pointer[-2 - oparg] = res; stack_pointer[-1 - oparg] = c; stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = c; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_EX_NON_PY_GENERAL) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_EX_NON_PY_GENERAL; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CALL_EX_NON_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX == 1, "incorrect cache size"); _PyStackRef func_st; _PyStackRef func; _PyStackRef callargs; _PyStackRef null; _PyStackRef callargs_st; _PyStackRef kwargs_st; _PyStackRef result; /* Skip 1 cache entry */ // _CHECK_IS_NOT_PY_CALLABLE_EX { func_st = stack_pointer[-4]; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); if (Py_TYPE(func) == &PyFunction_Type && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { UPDATE_MISS_STATS(CALL_FUNCTION_EX); assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX)); JUMP_TO_PREDICTED(CALL_FUNCTION_EX); } } // _MAKE_CALLARGS_A_TUPLE { callargs = stack_pointer[-2]; func = func_st; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } } // _CALL_FUNCTION_EX_NON_PY_GENERAL { kwargs_st = stack_pointer[-1]; callargs_st = callargs; null = stack_pointer[-3]; func_st = func; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); (void)null; assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *result_o = PyObject_Call(func, callargs, kwargs); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC_AT_END { stack_pointer[0] = result; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_EX_PY) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_EX_PY; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CALL_EX_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX == 1, "incorrect cache size"); _PyStackRef func; _PyStackRef callargs; _PyStackRef func_st; _PyStackRef callargs_st; _PyStackRef kwargs_st; _PyStackRef ex_frame; _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(CALL_FUNCTION_EX); assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX)); JUMP_TO_PREDICTED(CALL_FUNCTION_EX); } } // _MAKE_CALLARGS_A_TUPLE { callargs = stack_pointer[-2]; func = stack_pointer[-4]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } } // _CHECK_IS_PY_CALLABLE_EX { func_st = func; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); if (Py_TYPE(func) != &PyFunction_Type) { UPDATE_MISS_STATS(CALL_FUNCTION_EX); assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX)); JUMP_TO_PREDICTED(CALL_FUNCTION_EX); } if (((PyFunctionObject *)func)->vectorcall != _PyFunction_Vectorcall) { UPDATE_MISS_STATS(CALL_FUNCTION_EX); assert(_PyOpcode_Deopt[opcode] == (CALL_FUNCTION_EX)); JUMP_TO_PREDICTED(CALL_FUNCTION_EX); } } // _PY_FRAME_EX { kwargs_st = stack_pointer[-1]; callargs_st = callargs; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); assert(PyTuple_CheckExact(callargs)); assert(Py_TYPE(func) == &PyFunction_Type); assert(((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall); PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } ex_frame = PyStackRef_Wrap(new_frame); } // _SAVE_RETURN_OFFSET { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; #endif } // _PUSH_FRAME { new_frame = ex_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(CALL_FUNCTION_EX) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_FUNCTION_EX; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CALL_FUNCTION_EX); PREDICTED_CALL_FUNCTION_EX:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; opcode = CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; _PyStackRef func_st; _PyStackRef null; _PyStackRef callargs_st; _PyStackRef kwargs_st; _PyStackRef result; // _SPECIALIZE_CALL_FUNCTION_EX { func = stack_pointer[-4]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_CallFunctionEx(func, next_instr); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_FUNCTION_EX); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _MAKE_CALLARGS_A_TUPLE { callargs = stack_pointer[-2]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_FUNCTION_EX { kwargs_st = stack_pointer[-1]; callargs_st = callargs; null = stack_pointer[-3]; func_st = func; (void)null; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); PyObject *result_o; assert(!_PyErr_Occurred(tstate)); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); _PyFrame_StackPointerInvalidate(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result_o == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); _PyFrame_StackPointerInvalidate(frame); } else { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_CLEAR(result_o); _PyFrame_StackPointerInvalidate(frame); } } } } else { if (Py_TYPE(func) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); } PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC_AT_END { stack_pointer[0] = result; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_INTRINSIC_1) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_INTRINSIC_1; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); _PyStackRef value; _PyStackRef res; _PyStackRef v; // _CALL_INTRINSIC_1 { value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } v = value; res = PyStackRef_FromPyObjectSteal(res_o); } // _POP_TOP { value = v; stack_pointer[-1] = res; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(CALL_INTRINSIC_2) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_INTRINSIC_2; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); _PyStackRef value2_st; _PyStackRef value1_st; _PyStackRef res; _PyStackRef vs1; _PyStackRef vs2; _PyStackRef value; // _CALL_INTRINSIC_2 { value1_st = stack_pointer[-1]; value2_st = stack_pointer[-2]; assert(oparg <= MAX_INTRINSIC_2); PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); vs1 = value1_st; vs2 = value2_st; } // _POP_TOP { value = vs2; stack_pointer[-2] = res; stack_pointer[-1] = vs1; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = vs1; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(CALL_ISINSTANCE) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_ISINSTANCE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; _PyStackRef instance; _PyStackRef cls; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_THIRD_NULL { null = stack_pointer[-3]; if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _GUARD_CALLABLE_ISINSTANCE { callable = stack_pointer[-4]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.isinstance) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_ISINSTANCE { cls = stack_pointer[-1]; instance = stack_pointer[-2]; STAT_INC(CALL, hit); PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int retval = PyObject_IsInstance(inst_o, cls_o); _PyFrame_StackPointerInvalidate(frame); if (retval < 0) { JUMP_TO_LABEL(error); } (void)null; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(cls); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(instance); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable); _PyFrame_StackPointerInvalidate(frame); res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); } stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CALL_KW) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; opcode = CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; _PyStackRef res; // _SPECIALIZE_CALL_KW { self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD_KW { if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; self_or_null = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_KW { kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); if (Py_TYPE(callable_o) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, total_args, kwnames, opcode == INSTRUMENTED_CALL_KW, frame, this_instr, tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CALL_KW_BOUND_METHOD) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW_BOUND_METHOD; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } } // _CHECK_METHOD_VERSION_KW { null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } } // _EXPAND_METHOD_KW { self_or_null = null; assert(PyStackRef_IsNull(self_or_null)); _PyStackRef callable_s = callable; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(Py_TYPE(callable_o) == &PyMethod_Type); self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); assert(PyStackRef_FunctionCheck(callable)); stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable_s); _PyFrame_StackPointerInvalidate(frame); } // flush // _PY_FRAME_KW { kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; #endif } // _PUSH_FRAME { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(CALL_KW_NON_PY) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW_NON_PY; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE_KW { callable = stack_pointer[-3 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } } // _CALL_KW_NON_PY { kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, kwnames); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_KW_PY) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW_PY; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } } // _CHECK_FUNCTION_VERSION_KW { callable = stack_pointer[-3 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } } // _CHECK_RECURSION_REMAINING { if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); JUMP_TO_PREDICTED(CALL_KW); } } // _PY_FRAME_KW { kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; self_or_null = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; #endif } // _PUSH_FRAME { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(CALL_LEN) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_LEN; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; _PyStackRef arg; _PyStackRef res; _PyStackRef a; _PyStackRef c; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_NOS_NULL { null = stack_pointer[-2]; if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _GUARD_CALLABLE_LEN { callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.len) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_LEN { arg = stack_pointer[-1]; STAT_INC(CALL, hit); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(arg_o); _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { JUMP_TO_LABEL(error); } PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { JUMP_TO_LABEL(error); } a = arg; c = callable; res = PyStackRef_FromPyObjectSteal(res_o); } // _POP_TOP { value = c; stack_pointer[-3] = res; stack_pointer[-2] = a; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = a; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(CALL_LIST_APPEND) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_LIST_APPEND; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef nos; _PyStackRef self; _PyStackRef arg; _PyStackRef none; _PyStackRef c; _PyStackRef s; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_LIST_APPEND { callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyInterpreterState *interp = tstate->interp; if (callable_o != interp->callable_cache.list_append) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _GUARD_NOS_NOT_NULL { nos = stack_pointer[-2]; if (PyStackRef_IsNull(nos)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _GUARD_NOS_LIST { PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_LIST_APPEND { arg = stack_pointer[-1]; self = nos; assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); if (err) { JUMP_TO_LABEL(error); } c = callable; s = self; none = PyStackRef_None; } // _POP_TOP { value = s; stack_pointer[-3] = none; stack_pointer[-2] = c; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = c; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_FAST; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (method->d_method->ml_flags != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; } if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } PyObject *self = PyStackRef_AsPyObjectBorrow( PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_METHOD_DESCRIPTOR_FAST { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc, self, arguments, total_args ); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (method->d_method->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc, self, arguments, total_args ); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = self_or_null; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef res; _PyStackRef c; _PyStackRef s; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (method->d_method->ml_flags != METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; } if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } PyObject *self = PyStackRef_AsPyObjectBorrow( PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_RECURSION_LIMIT { if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_METHOD_DESCRIPTOR_NOARGS { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; assert(oparg == 1 || !PyStackRef_IsNull(self_or_null)); if (!PyStackRef_IsNull(self_or_null)) { args--; } _PyStackRef self_stackref = args[0]; PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); STAT_INC(CALL, hit); PyCFunction cfunc = method->d_method->ml_meth; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { JUMP_TO_LABEL(error); } c = callable; s = args[0]; res = PyStackRef_FromPyObjectSteal(res_o); } // _POP_TOP { value = s; stack_pointer[-2 - oparg] = res; stack_pointer[-1 - oparg] = c; stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = c; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_O) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_O; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef res; _PyStackRef c; _PyStackRef s; _PyStackRef a; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_CALLABLE_METHOD_DESCRIPTOR_O { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (method->d_method->ml_flags != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { total_args++; } if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } PyObject *self = PyStackRef_AsPyObjectBorrow( PyStackRef_IsNull(self_or_null) ? args[0] : self_or_null); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_RECURSION_LIMIT { if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_METHOD_DESCRIPTOR_O { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; } STAT_INC(CALL, hit); PyCFunction cfunc = method->d_method->ml_meth; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { JUMP_TO_LABEL(error); } c = callable; s = arguments[0]; a = arguments[1]; res = PyStackRef_FromPyObjectSteal(res_o); } // _POP_TOP { value = a; stack_pointer[-2 - oparg] = res; stack_pointer[-1 - oparg] = c; stack_pointer[-oparg] = s; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = s; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = c; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_NON_PY_GENERAL) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_NON_PY_GENERAL; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CHECK_IS_NOT_PY_CALLABLE { callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_NON_PY_GENERAL { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, PyStackRef_NULL); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_PY_EXACT_ARGS) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_PY_EXACT_ARGS; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_FUNCTION_VERSION { callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_FUNCTION_EXACT_ARGS { self_or_null = stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_STACK_SPACE { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyFunctionObject *func = (PyFunctionObject *)callable_o; PyCodeObject *code = (PyCodeObject *)func->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_RECURSION_REMAINING { if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _INIT_CALL_PY_EXACT_ARGS { args = &stack_pointer[-oparg]; int has_self = !PyStackRef_IsNull(self_or_null); STAT_INC(CALL, hit); _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; pushed_frame->localsplus[0] = self_or_null; for (int i = 0; i < oparg; i++) { first_non_self_local[i] = args[i]; } new_frame = PyStackRef_Wrap(pushed_frame); } // _SAVE_RETURN_OFFSET { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; #endif } // _PUSH_FRAME { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(CALL_PY_GENERAL) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_PY_GENERAL; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_FUNCTION_VERSION { callable = stack_pointer[-2 - oparg]; uint32_t func_version = read_u32(&this_instr[2].cache); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CHECK_RECURSION_REMAINING { if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _PY_FRAME_GENERAL { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; if (!PyStackRef_IsNull(self_or_null)) { args--; total_args++; } assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } new_frame = PyStackRef_Wrap(temp); } // _SAVE_RETURN_OFFSET { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; #endif } // _PUSH_FRAME { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(CALL_STR_1) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_STR_1; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; _PyStackRef arg; _PyStackRef res; _PyStackRef a; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_NOS_NULL { null = stack_pointer[-2]; if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _GUARD_CALLABLE_STR_1 { callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyUnicode_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_STR_1 { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Str(arg_o); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } a = arg; res = PyStackRef_FromPyObjectSteal(res_o); } // _POP_TOP { value = a; stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_TUPLE_1) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_TUPLE_1; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; _PyStackRef arg; _PyStackRef res; _PyStackRef a; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_NOS_NULL { null = stack_pointer[-2]; if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _GUARD_CALLABLE_TUPLE_1 { callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyTuple_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_TUPLE_1 { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = PySequence_Tuple(arg_o); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } a = arg; res = PyStackRef_FromPyObjectSteal(res_o); } // _POP_TOP { value = a; stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(CALL_TYPE_1) { #if _Py_TAIL_CALL_INTERP int opcode = CALL_TYPE_1; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; _PyStackRef arg; _PyStackRef res; _PyStackRef a; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_NOS_NULL { null = stack_pointer[-2]; if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _GUARD_CALLABLE_TYPE_1 { callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (callable_o != (PyObject *)&PyType_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); } } // _CALL_TYPE_1 { arg = stack_pointer[-1]; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); STAT_INC(CALL, hit); a = arg; res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); } // _POP_TOP { value = a; stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(CHECK_EG_MATCH) { #if _Py_TAIL_CALL_INTERP int opcode = CHECK_EG_MATCH; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); _PyStackRef exc_value_st; _PyStackRef match_type_st; _PyStackRef rest; _PyStackRef match; match_type_st = stack_pointer[-1]; exc_value_st = stack_pointer[-2]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) { JUMP_TO_LABEL(error); } if (!Py_IsNone(match_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyErr_SetHandledException(match_o); _PyFrame_StackPointerInvalidate(frame); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); stack_pointer[0] = rest; stack_pointer[1] = match; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CHECK_EXC_MATCH) { #if _Py_TAIL_CALL_INTERP int opcode = CHECK_EXC_MATCH; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); _PyStackRef left; _PyStackRef right; _PyStackRef b; right = stack_pointer[-1]; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyExceptionInstance_Check(left_o)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int res = PyErr_GivenExceptionMatches(left_o, right_o); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(right); _PyFrame_StackPointerInvalidate(frame); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CLEANUP_THROW) { #if _Py_TAIL_CALL_INTERP int opcode = CLEANUP_THROW; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter; _PyStackRef null_in; _PyStackRef last_sent_val; _PyStackRef exc_value_st; _PyStackRef none; _PyStackRef null_out; _PyStackRef value; exc_value_st = stack_pointer[-1]; last_sent_val = stack_pointer[-2]; null_in = stack_pointer[-3]; sub_iter = stack_pointer[-4]; PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); #if !_Py_TAIL_CALL_INTERP assert(throwflag); #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); _PyFrame_StackPointerInvalidate(frame); if (matches) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub_iter; sub_iter = value; stack_pointer[-4] = sub_iter; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-1] = exc_value_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = last_sent_val; last_sent_val = PyStackRef_NULL; stack_pointer[-2] = last_sent_val; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = null_in; null_in = PyStackRef_NULL; stack_pointer[-3] = null_in; PyStackRef_XCLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); null_out = null_in; none = PyStackRef_None; } else { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } stack_pointer[0] = none; stack_pointer[1] = null_out; stack_pointer[2] = value; stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COMPARE_OP) { #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); PREDICTED_COMPARE_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef left; _PyStackRef right; _PyStackRef res; // _SPECIALIZE_COMPARE_OP { right = stack_pointer[-1]; left = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_CompareOp(left, right, next_instr, oparg); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(COMPARE_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _COMPARE_OP { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = right; right = PyStackRef_NULL; stack_pointer[-1] = right; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = left; left = PyStackRef_NULL; stack_pointer[-2] = left; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } if (oparg & 16) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int res_bool = PyObject_IsTrue(res_o); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(res_o); _PyFrame_StackPointerInvalidate(frame); if (res_bool < 0) { JUMP_TO_LABEL(error); } res = res_bool ? PyStackRef_True : PyStackRef_False; } else { res = PyStackRef_FromPyObjectSteal(res_o); } } stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COMPARE_OP_FLOAT) { #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP_FLOAT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; // _GUARD_TOS_FLOAT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); } } // _GUARD_NOS_FLOAT { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); } } /* Skip 1 cache entry */ // _COMPARE_OP_FLOAT { right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left_o); double dright = PyFloat_AS_DOUBLE(right_o); int sign_ish = COMPARISON_BIT(dleft, dright); l = left; r = right; res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; } // _POP_TOP_FLOAT { value = r; assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); } // _POP_TOP_FLOAT { value = l; assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COMPARE_OP_INT) { #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; // _GUARD_TOS_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); } } // _GUARD_NOS_INT { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); } } /* Skip 1 cache entry */ // _COMPARE_OP_INT { right = value; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(_PyLong_IsCompact((PyLongObject *)left_o)); assert(_PyLong_IsCompact((PyLongObject *)right_o)); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && _PyLong_DigitCount((PyLongObject *)right_o) <= 1); Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); int sign_ish = COMPARISON_BIT(ileft, iright); l = left; r = right; res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; } // _POP_TOP_INT { value = r; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } // _POP_TOP_INT { value = l; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COMPARE_OP_STR) { #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP_STR; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; // _GUARD_TOS_UNICODE { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); } } // _GUARD_NOS_UNICODE { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); JUMP_TO_PREDICTED(COMPARE_OP); } } /* Skip 1 cache entry */ // _COMPARE_OP_STR { right = value; left = nos; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); l = left; r = right; assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; } // _POP_TOP_UNICODE { value = r; assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); } // _POP_TOP_UNICODE { value = l; assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(CONTAINS_OP) { #if _Py_TAIL_CALL_INTERP int opcode = CONTAINS_OP; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); PREDICTED_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef right; _PyStackRef left; _PyStackRef b; _PyStackRef l; _PyStackRef r; _PyStackRef value; // _SPECIALIZE_CONTAINS_OP { right = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_ContainsOp(right, next_instr); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CONTAINS_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _CONTAINS_OP { left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int res = PySequence_Contains(right_o, left_o); _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; l = left; r = right; } // _POP_TOP { value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = l; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(CONTAINS_OP_DICT) { #if _Py_TAIL_CALL_INTERP int opcode = CONTAINS_OP_DICT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; _PyStackRef right; _PyStackRef b; _PyStackRef l; _PyStackRef r; _PyStackRef value; // _GUARD_TOS_ANY_DICT { tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyAnyDict_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); JUMP_TO_PREDICTED(CONTAINS_OP); } } /* Skip 1 cache entry */ // _CONTAINS_OP_DICT { right = tos; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyAnyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int res = PyDict_Contains(right_o, left_o); _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; l = left; r = right; } // _POP_TOP { value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = l; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(CONTAINS_OP_SET) { #if _Py_TAIL_CALL_INTERP int opcode = CONTAINS_OP_SET; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; _PyStackRef right; _PyStackRef b; _PyStackRef l; _PyStackRef r; _PyStackRef value; // _GUARD_TOS_ANY_SET { tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyAnySet_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); JUMP_TO_PREDICTED(CONTAINS_OP); } } /* Skip 1 cache entry */ // _CONTAINS_OP_SET { right = tos; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyAnySet_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int res = _PySet_Contains((PySetObject *)right_o, left_o); _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; l = left; r = right; } // _POP_TOP { value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = l; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(CONVERT_VALUE) { #if _Py_TAIL_CALL_INTERP int opcode = CONVERT_VALUE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); _PyStackRef value; _PyStackRef result; value = stack_pointer[-1]; conversion_func conv_fn; assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[0] = result; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COPY) { #if _Py_TAIL_CALL_INTERP int opcode = COPY; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-1 - (oparg-1)]; top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(COPY_FREE_VARS) { #if _Py_TAIL_CALL_INTERP int opcode = COPY_FREE_VARS; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); PyCodeObject *co = _PyFrame_GetCode(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyObject *closure = func->func_closure; assert(oparg == co->co_nfreevars); int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } DISPATCH(); } TARGET(DELETE_DEREF) { #if _Py_TAIL_CALL_INTERP int opcode = DELETE_DEREF; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); Py_DECREF(oldobj); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(DELETE_FAST) { #if _Py_TAIL_CALL_INTERP int opcode = DELETE_FAST; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(DELETE_SUBSCR) { #if _Py_TAIL_CALL_INTERP int opcode = DELETE_SUBSCR; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); _PyStackRef container; _PyStackRef sub; sub = stack_pointer[-1]; container = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } DISPATCH(); } TARGET(DICT_MERGE) { #if _Py_TAIL_CALL_INTERP int opcode = DICT_MERGE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); _PyStackRef callable; _PyStackRef dict; _PyStackRef update; _PyStackRef u; _PyStackRef value; // _DICT_MERGE { update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; callable = stack_pointer[-5 - (oparg - 1)]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); PyObject *dupkey = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_XDECREF(dupkey); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } u = update; } // _POP_TOP { value = u; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(DICT_UPDATE) { #if _Py_TAIL_CALL_INTERP int opcode = DICT_UPDATE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); _PyStackRef dict; _PyStackRef update; _PyStackRef upd; _PyStackRef value; // _DICT_UPDATE { update = stack_pointer[-1]; dict = stack_pointer[-2 - (oparg - 1)]; PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = PyDict_Update(dict_o, update_o); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); if (matches) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject *exc = _PyErr_GetRaisedException(tstate); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); _PyFrame_StackPointerInvalidate(frame); if (has_keys == 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'%T' object is not a mapping", update_o); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(exc); _PyFrame_StackPointerInvalidate(frame); } else { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyErr_ChainExceptions1(exc); _PyFrame_StackPointerInvalidate(frame); } } JUMP_TO_LABEL(error); } upd = update; } // _POP_TOP { value = upd; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(END_ASYNC_FOR) { #if _Py_TAIL_CALL_INTERP int opcode = END_ASYNC_FOR; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); _PyStackRef awaitable_st; _PyStackRef exc_st; exc_st = stack_pointer[-1]; awaitable_st = stack_pointer[-2]; JUMPBY(0); (void)oparg; PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); _PyFrame_StackPointerInvalidate(frame); if (matches) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = exc_st; exc_st = PyStackRef_NULL; stack_pointer[-1] = exc_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = awaitable_st; awaitable_st = PyStackRef_NULL; stack_pointer[-2] = awaitable_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } DISPATCH(); } TARGET(END_FOR) { #if _Py_TAIL_CALL_INTERP int opcode = END_FOR; (void)(opcode); #endif next_instr += 1; INSTRUCTION_STATS(END_FOR); _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(END_SEND) { #if _Py_TAIL_CALL_INTERP int opcode = END_SEND; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); _PyStackRef receiver; _PyStackRef index_or_null; _PyStackRef value; _PyStackRef val; value = stack_pointer[-1]; index_or_null = stack_pointer[-2]; receiver = stack_pointer[-3]; val = value; (void)index_or_null; stack_pointer[-3] = val; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(ENTER_EXECUTOR) { #if _Py_TAIL_CALL_INTERP int opcode = ENTER_EXECUTOR; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; if (IS_JIT_TRACING()) { int og_opcode = executor->vm_data.opcode; int og_oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyJit_EnterExecutorShouldStopTracing(og_opcode)) { if (_PyOpcode_Caches[_PyOpcode_Deopt[og_opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } opcode = og_opcode; oparg = og_oparg; DISPATCH_GOTO_NON_TRACING(); } JUMP_TO_LABEL(stop_tracing); } assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->current_executor == NULL); uintptr_t iversion = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) != iversion) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } DISPATCH_GOTO(); } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ } TARGET(EXIT_INIT_CHECK) { #if _Py_TAIL_CALL_INTERP int opcode = EXIT_INIT_CHECK; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; if (!PyStackRef_IsNone(should_be_none)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(EXTENDED_ARG) { #if _Py_TAIL_CALL_INTERP int opcode = EXTENDED_ARG; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } TARGET(FORMAT_SIMPLE) { #if _Py_TAIL_CALL_INTERP int opcode = FORMAT_SIMPLE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(value_o, NULL); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } else { res = value; stack_pointer += -1; } stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(FORMAT_WITH_SPEC) { #if _Py_TAIL_CALL_INTERP int opcode = FORMAT_WITH_SPEC; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); _PyStackRef value; _PyStackRef fmt_spec; _PyStackRef res; fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fmt_spec; fmt_spec = PyStackRef_NULL; stack_pointer[-1] = fmt_spec; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = value; value = PyStackRef_NULL; stack_pointer[-2] = value; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(FOR_ITER) { #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); PREDICTED_FOR_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; // _SPECIALIZE_FOR_ITER { null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _FOR_ITER { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); stack_pointer[-1] = null_or_index; DISPATCH(); } next = item; } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(FOR_ITER_GEN) { #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_GEN; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef gen_frame; _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } } // _FOR_ITER_GEN_FRAME { iter = stack_pointer[-2]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(gen) != &PyGen_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } if (!gen_try_set_executing((PyGenObject *)gen)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_None); gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; pushed_frame->previous = frame; frame->return_offset = (uint16_t)( 2u + oparg); gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME { new_frame = gen_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(FOR_ITER_LIST) { #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_LIST; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_LIST { null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyList_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } #endif } // _ITER_JUMP_LIST { #ifdef Py_GIL_DISABLED #else PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(list_o) == &PyList_Type); STAT_INC(FOR_ITER, hit); if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); stack_pointer[-1] = null_or_index; DISPATCH(); } #endif } // _ITER_NEXT_LIST { PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED assert(_Py_IsOwnedByCurrentThread(list_o) || _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); _PyFrame_StackPointerInvalidate(frame); if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } if (result == 0) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); stack_pointer[-1] = null_or_index; DISPATCH(); } #else next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); #endif null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(FOR_ITER_RANGE) { #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_RANGE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_RANGE { iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(r) != &PyRangeIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } #endif } // _ITER_JUMP_RANGE { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); #endif STAT_INC(FOR_ITER, hit); if (r->len <= 0) { JUMPBY(oparg + 1); DISPATCH(); } } // _ITER_NEXT_RANGE { _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(r) == &PyRangeIter_Type); #ifdef Py_GIL_DISABLED assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); #endif assert(r->len > 0); long value = r->start; r->start = value + r->step; r->len--; PyObject *res = PyLong_FromLong(value); if (res == NULL) { JUMP_TO_LABEL(error); } next = PyStackRef_FromPyObjectSteal(res); } stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(FOR_ITER_TUPLE) { #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_TUPLE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ // _ITER_CHECK_TUPLE { null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyTuple_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); } // _ITER_JUMP_TUPLE { PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); (void)tuple_o; assert(Py_TYPE(tuple_o) == &PyTuple_Type); STAT_INC(FOR_ITER, hit); if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); stack_pointer[-1] = null_or_index; DISPATCH(); } } // _ITER_NEXT_TUPLE { PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); assert(Py_TYPE(tuple_o) == &PyTuple_Type); uintptr_t i = PyStackRef_UntagInt(null_or_index); assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(FOR_ITER_VIRTUAL) { #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_VIRTUAL; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_VIRTUAL); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; /* Skip 1 cache entry */ // _GUARD_TOS_NOT_NULL { null_or_index = stack_pointer[-1]; if (PyStackRef_IsNull(null_or_index)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } } // _FOR_ITER_VIRTUAL { iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); _PyFrame_StackPointerInvalidate(frame); PyObject *next_o = next_index.object; index = next_index.index; if (next_o == NULL) { if (index < 0) { JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); DISPATCH(); } null_or_index = PyStackRef_TagInt(index); next = PyStackRef_FromPyObjectSteal(next_o); } stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_AITER) { #if _Py_TAIL_CALL_INTERP int opcode = GET_AITER; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); _PyStackRef obj; _PyStackRef iter; obj = stack_pointer[-1]; unaryfunc getter = NULL; PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); PyObject *iter_o; PyTypeObject *type = Py_TYPE(obj_o); if (type->tp_as_async != NULL) { getter = type->tp_as_async->am_aiter; } if (getter == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); iter_o = (*getter)(obj_o); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(iter_o); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_ANEXT) { #if _Py_TAIL_CALL_INTERP int opcode = GET_ANEXT; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); _PyStackRef aiter; _PyStackRef awaitable; aiter = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); _PyFrame_StackPointerInvalidate(frame); if (awaitable_o == NULL) { JUMP_TO_LABEL(error); } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_AWAITABLE) { #if _Py_TAIL_CALL_INTERP int opcode = GET_AWAITABLE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iterable); _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_ITER) { #if _Py_TAIL_CALL_INTERP int opcode = GET_ITER; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(GET_ITER); PREDICTED_GET_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef iterable; _PyStackRef iter; _PyStackRef index_or_null; // _SPECIALIZE_GET_ITER { iterable = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_GetIter(iterable, next_instr); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(GET_ITER); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _GET_ITER { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsError(result)) { JUMP_TO_LABEL(pop_1_error); } iter = result; } stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_ITER_SELF) { #if _Py_TAIL_CALL_INTERP int opcode = GET_ITER_SELF; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(GET_ITER_SELF); static_assert(INLINE_CACHE_ENTRIES_GET_ITER == 1, "incorrect cache size"); _PyStackRef iterable; _PyStackRef res; /* Skip 1 cache entry */ // _GUARD_ITERATOR { iterable = stack_pointer[-1]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)); if (tp->tp_iter != PyObject_SelfIter) { UPDATE_MISS_STATS(GET_ITER); assert(_PyOpcode_Deopt[opcode] == (GET_ITER)); JUMP_TO_PREDICTED(GET_ITER); } STAT_INC(GET_ITER, hit); } // _PUSH_NULL { res = PyStackRef_NULL; } stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_ITER_VIRTUAL) { #if _Py_TAIL_CALL_INTERP int opcode = GET_ITER_VIRTUAL; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(GET_ITER_VIRTUAL); static_assert(INLINE_CACHE_ENTRIES_GET_ITER == 1, "incorrect cache size"); _PyStackRef iterable; _PyStackRef zero; /* Skip 1 cache entry */ // _GUARD_ITER_VIRTUAL { iterable = stack_pointer[-1]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)); if (tp->_tp_iteritem == NULL) { UPDATE_MISS_STATS(GET_ITER); assert(_PyOpcode_Deopt[opcode] == (GET_ITER)); JUMP_TO_PREDICTED(GET_ITER); } STAT_INC(GET_ITER, hit); } // _PUSH_TAGGED_ZERO { zero = PyStackRef_TagInt(0); } stack_pointer[0] = zero; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(GET_LEN) { #if _Py_TAIL_CALL_INTERP int opcode = GET_LEN; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); _PyStackRef obj; _PyStackRef len; obj = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { JUMP_TO_LABEL(error); } PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) { JUMP_TO_LABEL(error); } len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(IMPORT_FROM) { #if _Py_TAIL_CALL_INTERP int opcode = IMPORT_FROM; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); _PyStackRef from; _PyStackRef res; from = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *res_o; if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportFrom( tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportFrom( tstate, PyStackRef_AsPyObjectBorrow(from), name); _PyFrame_StackPointerInvalidate(frame); } if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(IMPORT_NAME) { #if _Py_TAIL_CALL_INTERP int opcode = IMPORT_NAME; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); _PyStackRef level; _PyStackRef fromlist; _PyStackRef res; fromlist = stack_pointer[-1]; level = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyObject *res_o; if (!(oparg & 0x02)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level), oparg & 0x01); _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); _PyFrame_StackPointerInvalidate(frame); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fromlist; fromlist = PyStackRef_NULL; stack_pointer[-1] = fromlist; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = level; level = PyStackRef_NULL; stack_pointer[-2] = level; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_CALL) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_CALL; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); opcode = INSTRUMENTED_CALL; _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef func; _PyStackRef maybe_self; _PyStackRef *args; _PyStackRef res; /* Skip 3 cache entries */ // _MAYBE_EXPAND_METHOD { self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; self_or_null = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } } // _MONITOR_CALL { args = &stack_pointer[-oparg]; maybe_self = self_or_null; func = callable; int is_meth = !PyStackRef_IsNull(maybe_self); PyObject *function = PyStackRef_AsPyObjectBorrow(func); PyObject *arg0; if (is_meth) { arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); } else if (oparg) { arg0 = PyStackRef_AsPyObjectBorrow(args[0]); } else { arg0 = &_PyInstrumentation_MISSING; } stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = maybe_self; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg0 ); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } } // _DO_CALL { args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } if (Py_TYPE(callable_o) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, total_args, NULL, frame ); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, total_args, PyStackRef_NULL, opcode == INSTRUMENTED_CALL, frame, this_instr, tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; _PyStackRef func_st; _PyStackRef null; _PyStackRef callargs_st; _PyStackRef kwargs_st; _PyStackRef result; /* Skip 1 cache entry */ // _MAKE_CALLARGS_A_TUPLE { callargs = stack_pointer[-2]; func = stack_pointer[-4]; PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_FUNCTION_EX { kwargs_st = stack_pointer[-1]; callargs_st = callargs; null = stack_pointer[-3]; func_st = func; (void)null; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); PyObject *result_o; assert(!_PyErr_Occurred(tstate)); if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); assert(PyTuple_CheckExact(callargs)); PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); _PyFrame_StackPointerInvalidate(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result_o == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); _PyFrame_StackPointerInvalidate(frame); } else { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_CLEAR(result_o); _PyFrame_StackPointerInvalidate(frame); } } } } else { if (Py_TYPE(func) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } assert( 2u == 1 + INLINE_CACHE_ENTRIES_CALL_FUNCTION_EX); frame->return_offset = 2u ; DISPATCH_INLINED(new_frame); } PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); assert(PyTuple_CheckExact(callargs)); PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); } // _CHECK_PERIODIC_AT_END { stack_pointer[0] = result; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics_at_end(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(INSTRUMENTED_CALL_KW) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_CALL_KW; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); opcode = INSTRUMENTED_CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef kwnames; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD_KW { self_or_null = stack_pointer[-2 - oparg]; callable = stack_pointer[-3 - oparg]; if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *self = ((PyMethodObject *)callable_o)->im_self; self_or_null = PyStackRef_FromPyObjectNew(self); PyObject *method = ((PyMethodObject *)callable_o)->im_func; _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectNew(method); stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); _PyFrame_StackPointerInvalidate(frame); } } // _MONITOR_CALL_KW { args = &stack_pointer[-1 - oparg]; int is_meth = !PyStackRef_IsNull(self_or_null); PyObject *arg; if (is_meth) { arg = PyStackRef_AsPyObjectBorrow(self_or_null); } else if (args) { arg = PyStackRef_AsPyObjectBorrow(args[0]); } else { arg = &_PyInstrumentation_MISSING; } PyObject *function = PyStackRef_AsPyObjectBorrow(callable); stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } } // _DO_CALL_KW { kwnames = stack_pointer[-1]; args = &stack_pointer[-1 - oparg]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); if (Py_TYPE(callable_o) == &PyFunction_Type && !IS_PEP523_HOOKED(tstate) && ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); _PyFrame_StackPointerInvalidate(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, total_args, kwnames, opcode == INSTRUMENTED_CALL_KW, frame, this_instr, tstate); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_END_ASYNC_FOR) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_END_ASYNC_FOR; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); _PyStackRef awaitable_st; _PyStackRef exc_st; // _MONITOR_END_ASYNC_FOR { assert((next_instr-oparg)->op.code == END_SEND || (next_instr-oparg)->op.code >= MIN_INSTRUMENTED_OPCODE); INSTRUMENTED_JUMP(next_instr-oparg, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); } // _END_ASYNC_FOR { exc_st = stack_pointer[-1]; awaitable_st = stack_pointer[-2]; JUMPBY(0); (void)oparg; PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); _PyFrame_StackPointerInvalidate(frame); if (matches) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = exc_st; exc_st = PyStackRef_NULL; stack_pointer[-1] = exc_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = awaitable_st; awaitable_st = PyStackRef_NULL; stack_pointer[-2] = awaitable_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } } DISPATCH(); } TARGET(INSTRUMENTED_END_FOR) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_END_FOR; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); _PyStackRef receiver; _PyStackRef value; value = stack_pointer[-1]; receiver = stack_pointer[-3]; if (PyStackRef_GenCheck(receiver)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(INSTRUMENTED_END_SEND) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_END_SEND; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; _PyStackRef index_or_null; _PyStackRef value; _PyStackRef val; value = stack_pointer[-1]; index_or_null = stack_pointer[-2]; receiver = stack_pointer[-3]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } } val = value; (void)index_or_null; stack_pointer[-3] = val; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(INSTRUMENTED_FOR_ITER) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; /* Skip 1 cache entry */ null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); stack_pointer[-1] = null_or_index; DISPATCH(); } next = item; INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_INSTRUCTION) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); _PyFrame_StackPointerInvalidate(frame); if (next_opcode < 0) { JUMP_TO_LABEL(error); } next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); } assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } // _MONITOR_JUMP_BACKWARD { INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); } DISPATCH(); } TARGET(INSTRUMENTED_JUMP_FORWARD) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } TARGET(INSTRUMENTED_LINE) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_LINE; (void)(opcode); #endif _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { PyCodeObject *code = _PyFrame_GetCode(frame); int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; next_instr = this_instr; } else { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); original_opcode = _Py_call_instrumentation_line( tstate, frame, this_instr, prev_instr); _PyFrame_StackPointerInvalidate(frame); // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackAssertInvalid(frame); if (original_opcode < 0) { next_instr = this_instr+1; JUMP_TO_LABEL(error); } next_instr = frame->instr_ptr; if (next_instr != this_instr) { DISPATCH(); } } if (_PyOpcode_Caches[original_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; _PyStackRef attr; _PyStackRef *null; /* Skip 1 cache entry */ // _LOAD_SUPER_ATTR { self_st = stack_pointer[-1]; class_st = stack_pointer[-2]; global_super_st = stack_pointer[-3]; PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); _PyFrame_StackPointerInvalidate(frame); if (err) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } } PyObject *super; { PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); _PyFrame_StackPointerInvalidate(frame); } if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); _PyFrame_StackPointerInvalidate(frame); } else { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_CLEAR(super); _PyFrame_StackPointerInvalidate(frame); } } } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { JUMP_TO_LABEL(error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *attr_o = PyObject_GetAttr(super, name); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(super); _PyFrame_StackPointerInvalidate(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[1]; if (oparg & 1) { null[0] = PyStackRef_NULL; } } stack_pointer[0] = attr; stack_pointer += 1 + (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_NOT_TAKEN) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); #endif _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); (void)this_instr; INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); DISPATCH(); } TARGET(INSTRUMENTED_POP_ITER) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_ITER; (void)(opcode); #endif _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; iter = stack_pointer[-2]; (void)index_or_null; INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int jump = PyStackRef_IsFalse(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); if (jump) { INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; int jump = PyStackRef_IsNone(value); RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); if (jump) { INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); _PyFrame_StackPointerInvalidate(frame); stack_pointer += 1; } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; int jump = !PyStackRef_IsNone(value); RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); if (jump) { stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); _PyFrame_StackPointerInvalidate(frame); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { stack_pointer += -1; } DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int jump = PyStackRef_IsTrue(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); if (jump) { INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_RESUME) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_RESUME; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_RESUME); /* Skip 1 cache entry */ // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); _PyFrame_StackPointerInvalidate(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; frame->instr_ptr = bytecode + off; next_instr = frame->instr_ptr; DISPATCH(); } #endif } // _MAYBE_INSTRUMENT { #ifdef Py_GIL_DISABLED int check_instrumentation = 1; #else int check_instrumentation = (tstate->tracing == 0); #endif if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } next_instr = this_instr; DISPATCH(); } } } // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { Test_EvalFrame_Resumes++; if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } } // _MONITOR_RESUME { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation( tstate, oparg == 0 ? PY_MONITORING_EVENT_PY_START : PY_MONITORING_EVENT_PY_RESUME, frame, this_instr); _PyFrame_StackPointerInvalidate(frame); // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackAssertInvalid(frame); if (err) { JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; } } DISPATCH(); } TARGET(INSTRUMENTED_RETURN_VALUE) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); _PyStackRef val; _PyStackRef value; _PyStackRef retval; _PyStackRef res; // _RETURN_VALUE_EVENT { val = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } } // _MAKE_HEAP_SAFE { value = val; value = PyStackRef_MakeHeapSafe(value); } // _RETURN_VALUE { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INSTRUMENTED_YIELD_VALUE) { #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); opcode = INSTRUMENTED_YIELD_VALUE; _PyStackRef val; _PyStackRef value; _PyStackRef retval; // _YIELD_VALUE_EVENT { val = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); _PyFrame_StackPointerInvalidate(frame); if (err) { // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackAssertInvalid(frame); JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackAssertInvalid(frame); next_instr = frame->instr_ptr; DISPATCH(); } } // _MAKE_HEAP_SAFE { value = val; value = PyStackRef_MakeHeapSafe(value); } // _YIELD_VALUE { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(INTERPRETER_EXIT) { #if _Py_TAIL_CALL_INTERP int opcode = INTERPRETER_EXIT; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); assert(_PyFrame_IsIncomplete(frame)); tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); #if !_Py_TAIL_CALL_INTERP assert(frame == &entry.frame); #endif #ifdef _Py_TIER2 _PyStackRef executor = frame->localsplus[0]; assert(tstate->current_executor == NULL); if (!PyStackRef_IsNull(executor)) { assert(PyStackRef_TYPE(executor) == &_PyUOpExecutor_Type); tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(executor); _PyFrame_StackPointerInvalidate(frame); stack_pointer += 1; } #endif LLTRACE_RESUME_FRAME(); return result; } TARGET(IS_OP) { #if _Py_TAIL_CALL_INTERP int opcode = IS_OP; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); _PyStackRef left; _PyStackRef right; _PyStackRef b; _PyStackRef l; _PyStackRef r; _PyStackRef value; // _IS_OP { right = stack_pointer[-1]; left = stack_pointer[-2]; int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; b = res ? PyStackRef_True : PyStackRef_False; l = left; r = right; } // _POP_TOP { value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = l; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(JUMP_BACKWARD) { #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); PREDICTED_JUMP_BACKWARD:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; /* Skip 1 cache entry */ // _SPECIALIZE_JUMP_BACKWARD { #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); next_instr = this_instr; DISPATCH_SAME_OPARG(); } #endif } // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT { assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); } DISPATCH(); } TARGET(JUMP_BACKWARD_JIT) { #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD_JIT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT { assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); } // _JIT { #ifdef _Py_TIER2 bool is_resume = this_instr->op.code == RESUME_CHECK_JIT; _Py_BackoffCounter counter = this_instr[1].counter; if ((backoff_counter_triggers(counter) && !IS_JIT_TRACING() && (this_instr->op.code == JUMP_BACKWARD_JIT || is_resume)) && next_instr->op.code != ENTER_EXECUTOR) { _Py_CODEUNIT *insert_exec_at = this_instr; for (int tmp = oparg; tmp > 255; tmp >>= 8) { insert_exec_at--; } int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, is_resume ? insert_exec_at : next_instr, stack_pointer, 0, NULL, oparg, NULL); if (succ) { ENTER_TRACING(); } else { this_instr[1].counter = restart_backoff_counter(counter); } } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } #endif } DISPATCH(); } TARGET(JUMP_BACKWARD_NO_INTERRUPT) { #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); DISPATCH(); } TARGET(JUMP_BACKWARD_NO_JIT) { #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT { assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); } DISPATCH(); } TARGET(JUMP_FORWARD) { #if _Py_TAIL_CALL_INTERP int opcode = JUMP_FORWARD; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); JUMPBY(oparg); DISPATCH(); } TARGET(LIST_APPEND) { #if _Py_TAIL_CALL_INTERP int opcode = LIST_APPEND; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); _PyStackRef list; _PyStackRef v; v = stack_pointer[-1]; list = stack_pointer[-2 - (oparg-1)]; int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); if (err < 0) { JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LIST_EXTEND) { #if _Py_TAIL_CALL_INTERP int opcode = LIST_EXTEND; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); _PyStackRef list_st; _PyStackRef iterable_st; _PyStackRef i; _PyStackRef value; // _LIST_EXTEND { iterable_st = stack_pointer[-1]; list_st = stack_pointer[-2 - (oparg-1)]; PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); _PyFrame_StackPointerInvalidate(frame); if (none_val == NULL) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyErr_Clear(tstate); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } assert(Py_IsNone(none_val)); i = iterable_st; } // _POP_TOP { value = i; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(LOAD_ATTR) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); PREDICTED_LOAD_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; _PyStackRef owner; _PyStackRef attr; _PyStackRef *self_or_null; // _SPECIALIZE_LOAD_ATTR { owner = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadAttr(owner, next_instr, name); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } /* Skip 8 cache entries */ // _LOAD_ATTR { self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { JUMP_TO_LABEL(pop_1_error); } } else { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); attr = _PyObject_GetAttrStackRef(PyStackRef_AsPyObjectBorrow(owner), name); _PyFrame_StackPointerInvalidate(frame); stack_pointer[-1] = attr; stack_pointer += (oparg&1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { JUMP_TO_LABEL(error); } stack_pointer += -(oparg&1); } } stack_pointer[-1] = attr; stack_pointer += (oparg&1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_CLASS; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef *null; /* Skip 1 cache entry */ // _CHECK_ATTR_CLASS { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } /* Skip 2 cache entries */ // _LOAD_ATTR_CLASS { PyObject *descr = read_obj(&this_instr[6].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); } // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; if (oparg & 1) { null[0] = PyStackRef_NULL; } } stack_pointer += (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef *null; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _CHECK_ATTR_CLASS { uint32_t type_version = read_u32(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _LOAD_ATTR_CLASS { PyObject *descr = read_obj(&this_instr[6].cache); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); } // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; if (oparg & 1) { null[0] = PyStackRef_NULL; } } stack_pointer += (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef new_frame; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME { uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *getattribute = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); if (f->func_version != func_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked( tstate, PyStackRef_FromPyObjectNew(f), 2, frame); pushed_frame->localsplus[0] = owner; pushed_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); new_frame = PyStackRef_Wrap(pushed_frame); } // _SAVE_RETURN_OFFSET { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; #endif } // _PUSH_FRAME { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(LOAD_ATTR_INSTANCE_VALUE) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_INSTANCE_VALUE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; _PyStackRef value; _PyStackRef *null; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); if (!increfed) { if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); o = owner; } // _POP_TOP { value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; if (oparg & 1) { null[0] = PyStackRef_NULL; } } stack_pointer += (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_METHOD_LAZY_DICT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _CHECK_ATTR_METHOD_LAZY_DICT { uint16_t dictoffset = read_u16(&this_instr[4].cache); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } /* Skip 1 cache entry */ // _LOAD_ATTR_METHOD_LAZY_DICT { PyObject *descr = read_obj(&this_instr[6].cache); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; } stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_METHOD_NO_DICT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; } stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_METHOD_WITH_VALUES; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); attr = PyStackRef_FromPyObjectNew(descr); self = owner; } stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_MODULE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; _PyStackRef value; _PyStackRef *null; /* Skip 1 cache entry */ // _LOAD_ATTR_MODULE { owner = stack_pointer[-1]; uint32_t dict_version = read_u32(&this_instr[2].cache); uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } assert(keys->dk_kind == DICT_KEYS_UNICODE); assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; PyObject *attr_o = FT_ATOMIC_LOAD_PTR_CONSUME(ep->me_value); if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); o = owner; } // _POP_TOP { value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; if (oparg & 1) { null[0] = PyStackRef_NULL; } } stack_pointer += (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT { PyObject *descr = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[0] = attr; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _CHECK_MANAGED_OBJECT_HAS_VALUES { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[0] = attr; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_PROPERTY; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef new_frame; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _LOAD_ATTR_PROPERTY_FRAME { uint32_t func_version = read_u32(&this_instr[4].cache); PyObject *fget = read_obj(&this_instr[6].cache); assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; if (f->func_version != func_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } PyCodeObject *code = (PyCodeObject *)f->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); pushed_frame->localsplus[0] = owner; new_frame = PyStackRef_Wrap(pushed_frame); } // _SAVE_RETURN_OFFSET { #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); #endif #if TIER_TWO frame->return_offset = oparg; #endif } // _PUSH_FRAME { assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_SLOT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; _PyStackRef value; _PyStackRef *null; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _LOAD_ATTR_SLOT { uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); if (!increfed) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); o = owner; } // _POP_TOP { value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; if (oparg & 1) { null[0] = PyStackRef_NULL; } } stack_pointer += (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_WITH_HINT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; _PyStackRef value; _PyStackRef *null; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } // _LOAD_ATTR_WITH_HINT { uint16_t hint = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); if (dict == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); assert(PyDict_CheckExact((PyObject *)dict)); #ifdef Py_GIL_DISABLED if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } #endif PyObject *attr_o; if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (dk->dk_kind != DICT_KEYS_UNICODE) { if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); if (attr_o == NULL) { if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } STAT_INC(LOAD_ATTR, hit); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); if (!increfed) { if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); JUMP_TO_PREDICTED(LOAD_ATTR); } } #else attr = PyStackRef_FromPyObjectNew(attr_o); #endif o = owner; } // _POP_TOP { value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[0]; if (oparg & 1) { null[0] = PyStackRef_NULL; } } stack_pointer += (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_BUILD_CLASS) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_BUILD_CLASS; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); _PyStackRef bc; int err; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *bc_o = _PyMapping_GetOptionalItem2(BUILTINS(), &_Py_ID(__build_class__), &err); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (bc_o == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_COMMON_CONSTANT) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_COMMON_CONSTANT; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_CONST) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_CONST; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); _PyStackRef value; Test_EvalFrame_Loads++; PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_DEREF) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_DEREF; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); value = _PyCell_GetStackRef(cell); _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST_AND_CLEAR) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); _PyStackRef value; value = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST_BORROW) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_BORROW; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW); _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; value1 = PyStackRef_Borrow(GETLOCAL(oparg1)); value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); stack_pointer[0] = value1; stack_pointer[1] = value2; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST_CHECK) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_CHECK; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); _PyStackRef value; _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } value = PyStackRef_DUP(value_s); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FAST_LOAD_FAST) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; value1 = PyStackRef_DUP(GETLOCAL(oparg1)); value2 = PyStackRef_DUP(GETLOCAL(oparg2)); stack_pointer[0] = value1; stack_pointer[1] = value2; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FROM_DICT_OR_DEREF) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); _PyStackRef class_dict_st; _PyStackRef value; class_dict_st = stack_pointer[-1]; PyObject *name; PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); assert(class_dict); assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (!value_o) { PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value_o = PyCell_GetRef(cell); if (value_o == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(class_dict_st); _PyFrame_StackPointerInvalidate(frame); value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_FROM_DICT_OR_GLOBALS) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); _PyStackRef mod_or_class_dict; _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(mod_or_class_dict); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { if (!_PyErr_Occurred(tstate)) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } if (PyLazyImport_CheckExact(v_o)) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } } } else { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } } if (PyLazyImport_CheckExact(v_o)) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } } } } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_GLOBAL) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_GLOBAL; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED_LOAD_GLOBAL:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef *res; _PyStackRef *null; // _SPECIALIZE_LOAD_GLOBAL { uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_GLOBAL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } /* Skip 1 cache entry */ /* Skip 1 cache entry */ /* Skip 1 cache entry */ // _LOAD_GLOBAL { res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(*res)) { JUMP_TO_LABEL(error); } } // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[1]; if (oparg & 1) { null[0] = PyStackRef_NULL; } } stack_pointer += 1 + (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_GLOBAL_BUILTIN) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_GLOBAL_BUILTIN; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; /* Skip 1 cache entry */ // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&this_instr[2].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(keys->dk_kind == DICT_KEYS_UNICODE); } // _LOAD_GLOBAL_BUILTINS { uint16_t version = read_u16(&this_instr[3].cache); uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); } // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[1]; if (oparg & 1) { null[0] = PyStackRef_NULL; } } stack_pointer[0] = res; stack_pointer += 1 + (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_GLOBAL_MODULE) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_GLOBAL_MODULE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; /* Skip 1 cache entry */ // _NOP { } // _LOAD_GLOBAL_MODULE { uint16_t version = read_u16(&this_instr[2].cache); uint16_t index = read_u16(&this_instr[4].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); assert(index < DK_SIZE(keys)); PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else res = PyStackRef_FromPyObjectNew(res_o); #endif STAT_INC(LOAD_GLOBAL, hit); } // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[1]; if (oparg & 1) { null[0] = PyStackRef_NULL; } } stack_pointer[0] = res; stack_pointer += 1 + (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_LOCALS) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_LOCALS; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); _PyStackRef locals; PyObject *l = LOCALS(); if (l == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_NAME) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_NAME; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } if (PyLazyImport_CheckExact(v_o)) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); _PyFrame_StackPointerInvalidate(frame); if (l_v == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, l_v); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(l_v); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); _PyFrame_StackPointerInvalidate(frame); } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_SMALL_INT) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SMALL_INT; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); _PyStackRef value; assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_SPECIAL) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SPECIAL; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); _PyStackRef self; _PyStackRef *method_and_self; // _INSERT_NULL { self = stack_pointer[-1]; method_and_self = &stack_pointer[-1]; method_and_self[1] = self; method_and_self[0] = PyStackRef_NULL; } // _LOAD_SPECIAL { method_and_self = &stack_pointer[-1]; PyObject *name = _Py_SpecialMethods[oparg].name; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _PyObject_LookupSpecialMethod(name, method_and_self); _PyFrame_StackPointerInvalidate(frame); if (err <= 0) { if (err == 0) { PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) ? _Py_SpecialMethods[oparg].error_suggestion : _Py_SpecialMethods[oparg].error; _PyFrame_StackPointerInvalidate(frame); assert(!_PyErr_Occurred(tstate)); assert(errfmt != NULL); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(LOAD_SUPER_ATTR) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SUPER_ATTR; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; opcode = LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; _PyStackRef attr; _PyStackRef *null; // _SPECIALIZE_LOAD_SUPER_ATTR { class_st = stack_pointer[-2]; global_super_st = stack_pointer[-3]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _LOAD_SUPER_ATTR { self_st = stack_pointer[-1]; PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); _PyFrame_StackPointerInvalidate(frame); if (err) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } } PyObject *super; { PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); _PyFrame_StackPointerInvalidate(frame); } if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); _PyFrame_StackPointerInvalidate(frame); } else { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_CLEAR(super); _PyFrame_StackPointerInvalidate(frame); } } } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { JUMP_TO_LABEL(error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *attr_o = PyObject_GetAttr(super, name); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(super); _PyFrame_StackPointerInvalidate(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } // _PUSH_NULL_CONDITIONAL { null = &stack_pointer[1]; if (oparg & 1) { null[0] = PyStackRef_NULL; } } stack_pointer[0] = attr; stack_pointer += 1 + (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_SUPER_ATTR_ATTR) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SUPER_ATTR_ATTR; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; _PyStackRef attr_st; /* Skip 1 cache entry */ self_st = stack_pointer[-1]; class_st = stack_pointer[-2]; global_super_st = stack_pointer[-3]; PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); assert(!(oparg & 1)); if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { JUMP_TO_LABEL(error); } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[0] = attr_st; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(LOAD_SUPER_ATTR_METHOD) { #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SUPER_ATTR_METHOD; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; _PyStackRef attr; _PyStackRef self_or_null; /* Skip 1 cache entry */ // _GUARD_LOAD_SUPER_ATTR_METHOD { class_st = stack_pointer[-2]; global_super_st = stack_pointer[-3]; PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); assert(oparg & 1); if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } } // _LOAD_SUPER_ATTR_METHOD { self_st = stack_pointer[-1]; PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; int method_found = 0; PyObject *attr_o; { int *method_found_ptr = &method_found; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); _PyFrame_StackPointerInvalidate(frame); } if (attr_o == NULL) { JUMP_TO_LABEL(error); } if (method_found) { self_or_null = self_st; } else { stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(self_st); _PyFrame_StackPointerInvalidate(frame); self_or_null = PyStackRef_NULL; stack_pointer += 1; } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; stack_pointer[-2] = global_super_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-1] = class_st; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); } stack_pointer[0] = attr; stack_pointer[1] = self_or_null; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(MAKE_CELL) { #if _Py_TAIL_CALL_INTERP int opcode = MAKE_CELL; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(MAKE_FUNCTION) { #if _Py_TAIL_CALL_INTERP int opcode = MAKE_FUNCTION; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); _PyStackRef codeobj_st; _PyStackRef func; _PyStackRef co; _PyStackRef value; // _MAKE_FUNCTION { codeobj_st = stack_pointer[-1]; PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); _PyFrame_StackPointerInvalidate(frame); if (func_obj == NULL) { JUMP_TO_LABEL(error); } co = codeobj_st; _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); } // _POP_TOP { value = co; stack_pointer[-1] = func; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(MAP_ADD) { #if _Py_TAIL_CALL_INTERP int opcode = MAP_ADD; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); _PyStackRef dict_st; _PyStackRef key; _PyStackRef value; value = stack_pointer[-1]; key = stack_pointer[-2]; dict_st = stack_pointer[-3 - (oparg - 1)]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2( (PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value) ); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(pop_2_error); } stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(MATCH_CLASS) { #if _Py_TAIL_CALL_INTERP int opcode = MATCH_CLASS; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); _PyStackRef subject; _PyStackRef type; _PyStackRef names; _PyStackRef attrs; _PyStackRef s; _PyStackRef tp; _PyStackRef n; _PyStackRef value; // _MATCH_CLASS { names = stack_pointer[-1]; type = stack_pointer[-2]; subject = stack_pointer[-3]; assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *attrs_o = _PyEval_MatchClass(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); _PyFrame_StackPointerInvalidate(frame); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); } else { if (_PyErr_Occurred(tstate)) { JUMP_TO_LABEL(error); } attrs = PyStackRef_None; } s = subject; tp = type; n = names; } // _POP_TOP { value = n; stack_pointer[-3] = attrs; stack_pointer[-2] = s; stack_pointer[-1] = tp; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = tp; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = s; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(MATCH_KEYS) { #if _Py_TAIL_CALL_INTERP int opcode = MATCH_KEYS; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); _PyStackRef subject; _PyStackRef keys; _PyStackRef values_or_none; keys = stack_pointer[-1]; subject = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); _PyFrame_StackPointerInvalidate(frame); if (values_or_none_o == NULL) { JUMP_TO_LABEL(error); } values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(MATCH_MAPPING) { #if _Py_TAIL_CALL_INTERP int opcode = MATCH_MAPPING; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(MATCH_SEQUENCE) { #if _Py_TAIL_CALL_INTERP int opcode = MATCH_SEQUENCE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(NOP) { #if _Py_TAIL_CALL_INTERP int opcode = NOP; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); DISPATCH(); } TARGET(NOT_TAKEN) { #if _Py_TAIL_CALL_INTERP int opcode = NOT_TAKEN; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); DISPATCH(); } TARGET(POP_EXCEPT) { #if _Py_TAIL_CALL_INTERP int opcode = POP_EXCEPT; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); Py_XSETREF(exc_info->exc_value, PyStackRef_IsNone(exc_value) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(POP_ITER) { #if _Py_TAIL_CALL_INTERP int opcode = POP_ITER; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; iter = stack_pointer[-2]; (void)index_or_null; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(POP_JUMP_IF_FALSE) { #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_FALSE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(POP_JUMP_IF_NONE) { #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_NONE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); _PyStackRef value; _PyStackRef b; _PyStackRef cond; /* Skip 1 cache entry */ // _IS_NONE { value = stack_pointer[-1]; if (PyStackRef_IsNone(value)) { b = PyStackRef_True; } else { b = PyStackRef_False; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); } } // _POP_JUMP_IF_TRUE { cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(POP_JUMP_IF_NOT_NONE) { #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); _PyStackRef value; _PyStackRef b; _PyStackRef cond; /* Skip 1 cache entry */ // _IS_NONE { value = stack_pointer[-1]; if (PyStackRef_IsNone(value)) { b = PyStackRef_True; } else { b = PyStackRef_False; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); } } // _POP_JUMP_IF_FALSE { cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(POP_JUMP_IF_TRUE) { #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_TRUE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(POP_TOP) { #if _Py_TAIL_CALL_INTERP int opcode = POP_TOP; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(PUSH_EXC_INFO) { #if _Py_TAIL_CALL_INTERP int opcode = PUSH_EXC_INFO; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); _PyStackRef exc; _PyStackRef prev_exc; _PyStackRef new_exc; exc = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); } else { prev_exc = PyStackRef_None; } assert(PyStackRef_ExceptionInstanceCheck(exc)); exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); new_exc = exc; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(PUSH_NULL) { #if _Py_TAIL_CALL_INTERP int opcode = PUSH_NULL; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); _PyStackRef res; res = PyStackRef_NULL; stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(RAISE_VARARGS) { #if _Py_TAIL_CALL_INTERP int opcode = RAISE_VARARGS; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); _PyStackRef *args; args = &stack_pointer[-oparg]; assert(oparg < 3); PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = do_raise(tstate, exc, cause); _PyFrame_StackPointerInvalidate(frame); if (err) { assert(oparg == 0); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } JUMP_TO_LABEL(error); } TARGET(RERAISE) { #if _Py_TAIL_CALL_INTERP int opcode = RERAISE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); _PyStackRef *values; _PyStackRef exc_st; exc_st = stack_pointer[-1]; values = &stack_pointer[-1 - oparg]; PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); assert(oparg >= 0 && oparg <= 2); if (oparg) { frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); } assert(exc && PyExceptionInstance_Check(exc)); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } TARGET(RESERVED) { #if _Py_TAIL_CALL_INTERP int opcode = RESERVED; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); assert(0 && "Executing RESERVED instruction."); Py_FatalError("Executing RESERVED instruction."); DISPATCH(); } TARGET(RESUME) { #if _Py_TAIL_CALL_INTERP int opcode = RESUME; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(RESUME); PREDICTED_RESUME:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); _PyFrame_StackPointerInvalidate(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; frame->instr_ptr = bytecode + off; next_instr = frame->instr_ptr; DISPATCH(); } #endif } // _MAYBE_INSTRUMENT { #ifdef Py_GIL_DISABLED int check_instrumentation = 1; #else int check_instrumentation = (tstate->tracing == 0); #endif if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } next_instr = this_instr; DISPATCH(); } } } // _QUICKEN_RESUME { uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_Resume(this_instr, tstate, frame); _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { Test_EvalFrame_Resumes++; if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } } } DISPATCH(); } TARGET(RESUME_CHECK) { #if _Py_TAIL_CALL_INTERP int opcode = RESUME_CHECK; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(RESUME_CHECK); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); JUMP_TO_PREDICTED(RESUME); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); JUMP_TO_PREDICTED(RESUME); } #ifdef Py_GIL_DISABLED if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); JUMP_TO_PREDICTED(RESUME); } #endif DISPATCH(); } TARGET(RESUME_CHECK_JIT) { #if _Py_TAIL_CALL_INTERP int opcode = RESUME_CHECK_JIT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(RESUME_CHECK_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _RESUME_CHECK { #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); JUMP_TO_PREDICTED(RESUME); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); assert((version & _PY_EVAL_EVENTS_MASK) == 0); if (eval_breaker != version) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); JUMP_TO_PREDICTED(RESUME); } #ifdef Py_GIL_DISABLED if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); JUMP_TO_PREDICTED(RESUME); } #endif } // _JIT { #ifdef _Py_TIER2 bool is_resume = this_instr->op.code == RESUME_CHECK_JIT; _Py_BackoffCounter counter = this_instr[1].counter; if ((backoff_counter_triggers(counter) && !IS_JIT_TRACING() && (this_instr->op.code == JUMP_BACKWARD_JIT || is_resume)) && next_instr->op.code != ENTER_EXECUTOR) { _Py_CODEUNIT *insert_exec_at = this_instr; for (int tmp = oparg; tmp > 255; tmp >>= 8) { insert_exec_at--; } int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, is_resume ? insert_exec_at : next_instr, stack_pointer, 0, NULL, oparg, NULL); if (succ) { ENTER_TRACING(); } else { this_instr[1].counter = restart_backoff_counter(counter); } } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } #endif } DISPATCH(); } TARGET(RETURN_GENERATOR) { #if _Py_TAIL_CALL_INTERP int opcode = RETURN_GENERATOR; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { JUMP_TO_LABEL(error); } assert(STACK_LEVEL() <= 2); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(RETURN_VALUE) { #if _Py_TAIL_CALL_INTERP int opcode = RETURN_VALUE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); _PyStackRef value; _PyStackRef retval; _PyStackRef res; // _MAKE_HEAP_SAFE { value = stack_pointer[-1]; value = PyStackRef_MakeHeapSafe(value); } // _RETURN_VALUE { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(SEND) { #if _Py_TAIL_CALL_INTERP int opcode = SEND; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); PREDICTED_SEND:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef receiver; _PyStackRef null_or_index; _PyStackRef v; _PyStackRef retval; // _SPECIALIZE_SEND { receiver = stack_pointer[-3]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_Send(receiver, next_instr); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(SEND); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _SEND { v = stack_pointer[-1]; null_or_index = stack_pointer[-2]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && gen_try_set_executing((PyGenObject *)receiver_o)) { PyGenObject *gen = (PyGenObject *)receiver_o; _PyInterpreterFrame *gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)( 2u + oparg); assert(gen_frame->previous == NULL); gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } if (!PyStackRef_IsNull(null_or_index) && PyStackRef_IsNone(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index); _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); } JUMPBY(oparg); stack_pointer[-2] = null_or_index; DISPATCH(); } retval = item; } else { PyObject *v_o = PyStackRef_AsPyObjectBorrow(v); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PySendResultPair res = _PyIter_Send(receiver_o, v_o); _PyFrame_StackPointerInvalidate(frame); if (res.kind == PYGEN_ERROR) { JUMP_TO_LABEL(error); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); _PyFrame_StackPointerInvalidate(frame); retval = PyStackRef_FromPyObjectSteal(res.object); if (res.kind == PYGEN_RETURN) { JUMPBY(oparg); } stack_pointer += 1; } } stack_pointer[-2] = null_or_index; stack_pointer[-1] = retval; DISPATCH(); } TARGET(SEND_ASYNC_GEN) { #if _Py_TAIL_CALL_INTERP int opcode = SEND_ASYNC_GEN; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_ASYNC_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_in; _PyStackRef v; _PyStackRef asend; _PyStackRef null_out; _PyStackRef retval; /* Skip 1 cache entry */ // _GUARD_3OS_ASYNC_GEN_ASEND { iter = stack_pointer[-3]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (!PyAsyncGenASend_CheckExact(iter_o)) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); JUMP_TO_PREDICTED(SEND); } } // _SEND_ASYNC_GEN { v = stack_pointer[-1]; null_in = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); assert(PyAsyncGenASend_CheckExact(iter_o)); PyObject *val = PyStackRef_AsPyObjectBorrow(v); PyObject *retval_o; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o); _PyFrame_StackPointerInvalidate(frame); if (what == PYGEN_ERROR) { JUMP_TO_LABEL(error); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); _PyFrame_StackPointerInvalidate(frame); asend = iter; null_out = null_in; retval = PyStackRef_FromPyObjectSteal(retval_o); if (what == PYGEN_RETURN) { JUMPBY(oparg); } } stack_pointer[-2] = asend; stack_pointer[-1] = null_out; stack_pointer[0] = retval; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(SEND_GEN) { #if _Py_TAIL_CALL_INTERP int opcode = SEND_GEN; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; _PyStackRef v; _PyStackRef gen_frame; _PyStackRef new_frame; /* Skip 1 cache entry */ // _CHECK_PEP_523 { if (IS_PEP523_HOOKED(tstate)) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); JUMP_TO_PREDICTED(SEND); } } // _SEND_GEN_FRAME { v = stack_pointer[-1]; receiver = stack_pointer[-3]; PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); JUMP_TO_PREDICTED(SEND); } if (!gen_try_set_executing((PyGenObject *)gen)) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); JUMP_TO_PREDICTED(SEND); } STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; assert( 2u + oparg <= UINT16_MAX); frame->return_offset = (uint16_t)( 2u + oparg); pushed_frame->previous = frame; gen_frame = PyStackRef_Wrap(pushed_frame); } // _PUSH_FRAME { new_frame = gen_frame; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); #ifdef Py_DEBUG assert(frame->previous->stackpointer_valid == 1); #endif DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } TARGET(SEND_VIRTUAL) { #if _Py_TAIL_CALL_INTERP int opcode = SEND_VIRTUAL; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_VIRTUAL); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef val; _PyStackRef nos; _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef none; _PyStackRef next; /* Skip 1 cache entry */ // _GUARD_TOS_IS_NONE { val = stack_pointer[-1]; if (!PyStackRef_IsNone(val)) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); JUMP_TO_PREDICTED(SEND); } } // _GUARD_NOS_NOT_NULL { nos = stack_pointer[-2]; if (PyStackRef_IsNull(nos)) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); JUMP_TO_PREDICTED(SEND); } } // _SEND_VIRTUAL { none = val; null_or_index = nos; iter = stack_pointer[-3]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); _PyFrame_StackPointerInvalidate(frame); PyObject *next_o = next_index.object; index = next_index.index; if (next_o == NULL) { if (index < 0) { JUMP_TO_LABEL(error); } next = none; JUMPBY(oparg); DISPATCH(); } next = PyStackRef_FromPyObjectSteal(next_o); null_or_index = PyStackRef_TagInt(index); } stack_pointer[-2] = null_or_index; stack_pointer[-1] = next; DISPATCH(); } TARGET(SETUP_ANNOTATIONS) { #if _Py_TAIL_CALL_INTERP int opcode = SETUP_ANNOTATIONS; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } int err; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject* ann_dict = _PyMapping_GetOptionalItem2(LOCALS(), &_Py_ID(__annotations__), &err); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (ann_dict == NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); ann_dict = PyDict_New(); _PyFrame_StackPointerInvalidate(frame); if (ann_dict == NULL) { JUMP_TO_LABEL(error); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } } else { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(SET_ADD) { #if _Py_TAIL_CALL_INTERP int opcode = SET_ADD; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); _PyStackRef set; _PyStackRef v; v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectSteal(v)); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(SET_FUNCTION_ATTRIBUTE) { #if _Py_TAIL_CALL_INTERP int opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); _PyStackRef attr_st; _PyStackRef func_in; _PyStackRef func_out; func_in = stack_pointer[-1]; attr_st = stack_pointer[-2]; PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); func_out = func_in; assert(PyFunction_Check(func)); size_t offset = _Py_FunctionAttributeOffsets[oparg]; assert(offset != 0); PyObject **ptr = (PyObject **)(((char *)func) + offset); assert(*ptr == NULL); *ptr = attr; stack_pointer[-2] = func_out; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(SET_UPDATE) { #if _Py_TAIL_CALL_INTERP int opcode = SET_UPDATE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); _PyStackRef set; _PyStackRef iterable; _PyStackRef i; _PyStackRef value; // _SET_UPDATE { iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } i = iterable; } // _POP_TOP { value = i; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(STORE_ATTR) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); PREDICTED_STORE_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; _PyStackRef v; _PyStackRef owner; // _SPECIALIZE_STORE_ATTR { owner = stack_pointer[-1]; v = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { if (!PyStackRef_IsNull(v)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_StoreAttr(owner, next_instr, name); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } } OPCODE_DEFERRED_INC(STORE_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } /* Skip 3 cache entries */ // _STORE_ATTR { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(v); _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(STORE_ATTR_INSTANCE_VALUE) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR_INSTANCE_VALUE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef value; _PyStackRef owner; _PyStackRef o; /* Skip 1 cache entry */ // _LOCK_OBJECT { value = stack_pointer[-1]; if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } } // _GUARD_TYPE_VERSION_LOCKED { owner = value; uint32_t type_version = read_u32(&this_instr[2].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UNLOCK_OBJECT(owner_o); if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } } } // _GUARD_DORV_NO_DICT { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_dictoffset < 0); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); if (_PyObject_GetManagedDict(owner_o) || !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UNLOCK_OBJECT(owner_o); if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } } } // _STORE_ATTR_INSTANCE_VALUE { value = stack_pointer[-2]; uint16_t offset = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); assert(_PyObject_GetManagedDict(owner_o) == NULL); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *old_value = *value_ptr; FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); if (old_value == NULL) { PyDictValues *values = _PyObject_InlineValues(owner_o); Py_ssize_t index = value_ptr - values->values; _PyDictValues_AddToInsertionOrder(values, index); } UNLOCK_OBJECT(owner_o); o = owner; stack_pointer[-2] = o; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = o; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(STORE_ATTR_SLOT) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR_SLOT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; _PyStackRef o; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } } // _STORE_ATTR_SLOT { value = stack_pointer[-2]; uint16_t index = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); PyObject *old_value = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); o = owner; stack_pointer[-2] = o; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = o; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(STORE_ATTR_WITH_HINT) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR_WITH_HINT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; _PyStackRef o; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } } // _STORE_ATTR_WITH_HINT { value = stack_pointer[-2]; uint16_t hint = read_u16(&this_instr[4].cache); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); if (dict == NULL) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } if (!LOCK_OBJECT(dict)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); if (hint >= (size_t)dict->ma_keys->dk_nentries || dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) { UNLOCK_OBJECT(dict); if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; if (ep->me_key != name) { UNLOCK_OBJECT(dict); if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } } PyObject *old_value = ep->me_value; if (old_value == NULL) { UNLOCK_OBJECT(dict); if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); JUMP_TO_PREDICTED(STORE_ATTR); } } _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); _PyFrame_StackPointerInvalidate(frame); FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); o = owner; stack_pointer[-2] = o; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { value = o; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(STORE_DEREF) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_DEREF; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); _PyStackRef v; v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(STORE_FAST) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_FAST; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); _PyStackRef value; _PyStackRef trash; // _SWAP_FAST { value = stack_pointer[-1]; _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; trash = tmp; } // _POP_TOP { value = trash; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(STORE_FAST_LOAD_FAST) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_FAST_LOAD_FAST; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); _PyStackRef value1; _PyStackRef value2; value1 = stack_pointer[-1]; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); GETLOCAL(oparg1) = value1; value2 = PyStackRef_DUP(GETLOCAL(oparg2)); stack_pointer[-1] = value2; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(STORE_FAST_STORE_FAST) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_FAST_STORE_FAST; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); _PyStackRef value2; _PyStackRef value1; value1 = stack_pointer[-1]; value2 = stack_pointer[-2]; uint32_t oparg1 = oparg >> 4; uint32_t oparg2 = oparg & 15; _PyStackRef tmp = GETLOCAL(oparg1); GETLOCAL(oparg1) = value1; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); tmp = GETLOCAL(oparg2); GETLOCAL(oparg2) = value2; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } TARGET(STORE_GLOBAL) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_GLOBAL; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; if (PyStackRef_IsNull(v)) { stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); err = PyDict_Pop(GLOBALS(), name, NULL); _PyFrame_StackPointerInvalidate(frame); if (err == 0) { err = -1; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); _PyFrame_StackPointerInvalidate(frame); } } else { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); _PyFrame_StackPointerInvalidate(frame); } if (err < 0) { JUMP_TO_LABEL(error); } DISPATCH(); } TARGET(STORE_NAME) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_NAME; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int error = _PyEval_StoreName(tstate, v, name, ns); _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(v)) { } else { stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); _PyFrame_StackPointerInvalidate(frame); stack_pointer += 1; } if (error) { JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(STORE_SLICE) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_SLICE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); _PyStackRef v; _PyStackRef container; _PyStackRef start; _PyStackRef stop; // _SPECIALIZE_STORE_SLICE { #if ENABLE_SPECIALIZATION OPCODE_DEFERRED_INC(STORE_SLICE); #endif /* ENABLE_SPECIALIZATION */ } // _STORE_SLICE { stop = stack_pointer[-1]; start = stack_pointer[-2]; container = stack_pointer[-3]; v = stack_pointer[-4]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), PyStackRef_AsPyObjectSteal(stop), Py_None); _PyFrame_StackPointerInvalidate(frame); int err; if (slice == NULL) { err = 1; } else { stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); _PyFrame_StackPointerInvalidate(frame); stack_pointer += 2; } _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-4] = v; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(STORE_SUBSCR) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_SUBSCR; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED_STORE_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef container; _PyStackRef sub; _PyStackRef v; // _SPECIALIZE_STORE_SUBSCR { sub = stack_pointer[-1]; container = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_StoreSubscr(container, sub, next_instr); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_SUBSCR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } // _STORE_SUBSCR { v = stack_pointer[-3]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-3] = v; PyStackRef_CLOSE(tmp); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } } DISPATCH(); } TARGET(STORE_SUBSCR_DICT) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_SUBSCR_DICT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef nos; _PyStackRef value; _PyStackRef dict_st; _PyStackRef sub; _PyStackRef st; // _GUARD_NOS_DICT_STORE_SUBSCRIPT { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!Py_TYPE(o)->tp_as_mapping) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); } if (Py_TYPE(o)->tp_as_mapping->mp_ass_subscript != _PyDict_StoreSubscript) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); } } /* Skip 1 cache entry */ // _STORE_SUBSCR_DICT { sub = stack_pointer[-1]; dict_st = nos; value = stack_pointer[-3]; PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(Py_TYPE(dict)->tp_as_mapping->mp_ass_subscript == _PyDict_StoreSubscript); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); _PyFrame_StackPointerInvalidate(frame); if (err) { stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(dict_st); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } st = dict_st; } // _POP_TOP { value = st; stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(STORE_SUBSCR_LIST_INT) { #if _Py_TAIL_CALL_INTERP int opcode = STORE_SUBSCR_LIST_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; _PyStackRef list_st; _PyStackRef sub_st; _PyStackRef ls; _PyStackRef ss; // _GUARD_TOS_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); } } // _GUARD_NOS_LIST { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); } } /* Skip 1 cache entry */ // _STORE_SUBSCR_LIST_INT { sub_st = value; list_st = nos; value = stack_pointer[-3]; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); assert(PyLong_CheckExact(sub)); assert(PyList_CheckExact(list)); Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); if (!LOCK_OBJECT(list)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); } Py_ssize_t len = PyList_GET_SIZE(list); if (index < 0) { index += len; } if (index < 0 || index >= len) { UNLOCK_OBJECT(list); if (true) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); } } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], PyStackRef_AsPyObjectSteal(value)); assert(old_value != NULL); UNLOCK_OBJECT(list); ls = list_st; ss = sub_st; stack_pointer[-3] = ls; stack_pointer[-2] = ss; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); Py_DECREF(old_value); _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_INT { value = ss; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } // _POP_TOP { value = ls; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(SWAP) { #if _Py_TAIL_CALL_INTERP int opcode = SWAP; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); _PyStackRef bottom; _PyStackRef top; top = stack_pointer[-1]; bottom = stack_pointer[-2 - (oparg-2)]; _PyStackRef temp = bottom; bottom = top; top = temp; stack_pointer[-2 - (oparg-2)] = bottom; stack_pointer[-1] = top; DISPATCH(); } TARGET(TO_BOOL) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); PREDICTED_TO_BOOL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; _PyStackRef value; _PyStackRef res; // _SPECIALIZE_TO_BOOL { value = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_ToBool(value, next_instr); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(TO_BOOL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ } /* Skip 2 cache entries */ // _TO_BOOL { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } res = err ? PyStackRef_True : PyStackRef_False; } stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(TO_BOOL_ALWAYS_TRUE) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_ALWAYS_TRUE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; _PyStackRef res; _PyStackRef v; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION { owner = stack_pointer[-1]; uint32_t type_version = read_u32(&this_instr[2].cache); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); JUMP_TO_PREDICTED(TO_BOOL); } } // _REPLACE_WITH_TRUE { value = owner; res = PyStackRef_True; v = value; } // _POP_TOP { value = v; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(TO_BOOL_BOOL) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_BOOL; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; if (!PyStackRef_BoolCheck(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); DISPATCH(); } TARGET(TO_BOOL_INT) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_INT; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; _PyStackRef v; // _GUARD_TOS_INT { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); JUMP_TO_PREDICTED(TO_BOOL); } } /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _TO_BOOL_INT { STAT_INC(TO_BOOL, hit); PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); res = (_PyLong_IsZero((PyLongObject *)value_o)) ? PyStackRef_False : PyStackRef_True; v = value; } // _POP_TOP_INT { value = v; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); } stack_pointer[-1] = res; DISPATCH(); } TARGET(TO_BOOL_LIST) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_LIST; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef tos; _PyStackRef value; _PyStackRef res; _PyStackRef v; // _GUARD_TOS_LIST { tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); JUMP_TO_PREDICTED(TO_BOOL); } } /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _TO_BOOL_LIST { value = tos; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); assert(PyList_CheckExact(value_o)); STAT_INC(TO_BOOL, hit); res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; v = value; } // _POP_TOP { value = v; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(TO_BOOL_NONE) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_NONE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; /* Skip 1 cache entry */ /* Skip 2 cache entries */ value = stack_pointer[-1]; if (!PyStackRef_IsNone(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); res = PyStackRef_False; stack_pointer[-1] = res; DISPATCH(); } TARGET(TO_BOOL_STR) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_STR; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; _PyStackRef v; // _GUARD_TOS_UNICODE { value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); JUMP_TO_PREDICTED(TO_BOOL); } } /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _TO_BOOL_STR { STAT_INC(TO_BOOL, hit); PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); res = value_o == &_Py_STR(empty) ? PyStackRef_False : PyStackRef_True; v = value; } // _POP_TOP_UNICODE { value = v; assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); } stack_pointer[-1] = res; DISPATCH(); } TARGET(TRACE_RECORD) { #if _Py_TAIL_CALL_INTERP int opcode = TRACE_RECORD; (void)(opcode); #endif _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(TRACE_RECORD); opcode = TRACE_RECORD; #if _Py_TIER2 assert(IS_JIT_TRACING()); next_instr = this_instr; frame->instr_ptr = prev_instr; opcode = next_instr->op.code; bool stop_tracing = ( opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT || (opcode >= MIN_INSTRUMENTED_OPCODE && opcode != ENTER_EXECUTOR) ); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; _PyJitTracerState *tracer = _tstate->jit_tracer_state; assert(tracer != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); _PyFrame_StackPointerInvalidate(frame); if (full) { LEAVE_TRACING(); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = stop_tracing_and_jit(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } DISPATCH(); } for (int i = 0; i < tracer->prev_state.recorded_count; i++) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_CLEAR(tracer->prev_state.recorded_values[i]); _PyFrame_StackPointerInvalidate(frame); } tracer->prev_state.recorded_count = 0; tracer->prev_state.instr = next_instr; PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_SETREF(tracer->prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); _PyFrame_StackPointerInvalidate(frame); } tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; tracer->prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]] // Branch opcodes use the cache for branch history, not // specialization counters. Don't reset it. && !IS_CONDITIONAL_JUMP_OPCODE(opcode)) { (&next_instr[1])->counter = trigger_backoff_counter(); } const _PyOpcodeRecordEntry *record_entry = &_PyOpcode_RecordEntries[opcode]; for (int i = 0; i < record_entry->count; i++) { _Py_RecordFuncPtr doesnt_escape = _PyOpcode_RecordFunctions[record_entry->indices[i]]; doesnt_escape(frame, stack_pointer, oparg, &tracer->prev_state.recorded_values[i]); } tracer->prev_state.recorded_count = record_entry->count; DISPATCH_GOTO_NON_TRACING(); #else (void)prev_instr; Py_FatalError("JIT instruction executed in non-jit build."); #endif } TARGET(UNARY_INVERT) { #if _Py_TAIL_CALL_INTERP int opcode = UNARY_INVERT; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); _PyStackRef value; _PyStackRef res; _PyStackRef v; // _UNARY_INVERT { value = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); v = value; } // _POP_TOP { value = v; stack_pointer[-1] = res; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(UNARY_NEGATIVE) { #if _Py_TAIL_CALL_INTERP int opcode = UNARY_NEGATIVE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); _PyStackRef value; _PyStackRef res; _PyStackRef v; // _UNARY_NEGATIVE { value = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); v = value; } // _POP_TOP { value = v; stack_pointer[-1] = res; assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(UNARY_NOT) { #if _Py_TAIL_CALL_INTERP int opcode = UNARY_NOT; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; assert(PyStackRef_BoolCheck(value)); res = PyStackRef_IsFalse(value) ? PyStackRef_True : PyStackRef_False; stack_pointer[-1] = res; DISPATCH(); } TARGET(UNPACK_EX) { #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_EX; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); _PyStackRef seq; _PyStackRef *top; seq = stack_pointer[-1]; top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); _PyFrame_StackPointerInvalidate(frame); if (res == 0) { JUMP_TO_LABEL(error); } stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(UNPACK_SEQUENCE) { #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED_UNPACK_SEQUENCE:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; _PyStackRef seq; _PyStackRef *top; // _SPECIALIZE_UNPACK_SEQUENCE { seq = stack_pointer[-1]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); #endif /* ENABLE_SPECIALIZATION */ (void)seq; (void)counter; } // _UNPACK_SEQUENCE { top = &stack_pointer[-1 + oparg]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); _PyFrame_StackPointerInvalidate(frame); if (res == 0) { JUMP_TO_LABEL(error); } } stack_pointer += oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(UNPACK_SEQUENCE_LIST) { #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE_LIST; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; _PyStackRef *values; // _GUARD_TOS_LIST { tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } /* Skip 1 cache entry */ // _UNPACK_SEQUENCE_LIST { seq = tos; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); assert(PyList_CheckExact(seq_o)); if (!LOCK_OBJECT(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); if (true) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } UNLOCK_OBJECT(seq_o); stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(UNPACK_SEQUENCE_TUPLE) { #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE_TUPLE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; _PyStackRef *values; // _GUARD_TOS_TUPLE { tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } /* Skip 1 cache entry */ // _UNPACK_SEQUENCE_TUPLE { seq = tos; values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); assert(PyTuple_CheckExact(seq_o)); if (PyTuple_GET_SIZE(seq_o) != oparg) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE_TWO_TUPLE; (void)(opcode); #endif _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; _PyStackRef val1; _PyStackRef val0; // _GUARD_TOS_TUPLE { tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } /* Skip 1 cache entry */ // _UNPACK_SEQUENCE_TWO_TUPLE { seq = tos; assert(oparg == 2); PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); assert(PyTuple_CheckExact(seq_o)); if (PyTuple_GET_SIZE(seq_o) != 2) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } TARGET(WITH_EXCEPT_START) { #if _Py_TAIL_CALL_INTERP int opcode = WITH_EXCEPT_START; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); _PyStackRef exit_func; _PyStackRef exit_self; _PyStackRef lasti; _PyStackRef val; _PyStackRef res; val = stack_pointer[-1]; lasti = stack_pointer[-3]; exit_self = stack_pointer[-4]; exit_func = stack_pointer[-5]; PyObject *exc, *tb; PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); assert(val_o && PyExceptionInstance_Check(val_o)); exc = PyExceptionInstance_Class(val_o); PyObject *original_tb = tb = PyException_GetTraceback(val_o); if (tb == NULL) { tb = Py_None; } assert(PyStackRef_IsTaggedInt(lasti)); (void)lasti; PyObject* res_o; { PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); _PyFrame_StackPointerInvalidate(frame); } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); Py_XDECREF(original_tb); _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } TARGET(YIELD_VALUE) { #if _Py_TAIL_CALL_INTERP int opcode = YIELD_VALUE; (void)(opcode); #endif frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); opcode = YIELD_VALUE; _PyStackRef value; _PyStackRef retval; // _MAKE_HEAP_SAFE { value = stack_pointer[-1]; value = PyStackRef_MakeHeapSafe(value); } // _YIELD_VALUE { retval = value; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); frame->instr_ptr++; PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); _PyStackRef temp = retval; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); assert(i >= 0 && i <= INT_MAX); int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = temp; LLTRACE_RESUME_FRAME(); } stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } /* END INSTRUCTIONS */ #if !_Py_TAIL_CALL_INTERP #if USE_COMPUTED_GOTOS _unknown_opcode: #else EXTRA_CASES // From pycore_opcode_metadata.h, a 'case' for each unused opcode #endif /* Tell C compilers not to hold the opcode variable in the loop. next_instr points the current instruction without TARGET(). */ opcode = next_instr->op.code; _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", _PyFrame_GetCode(frame)->co_filename, PyUnstable_InterpreterFrame_GetLine(frame), opcode); JUMP_TO_LABEL(error); } /* This should never be reached. Every opcode should end with DISPATCH() or goto error. */ Py_UNREACHABLE(); #endif /* _Py_TAIL_CALL_INTERP */ /* BEGIN LABELS */ LABEL(pop_2_error) { stack_pointer -= 2; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } LABEL(pop_1_error) { stack_pointer -= 1; assert(WITHIN_STACK_BOUNDS()); JUMP_TO_LABEL(error); } LABEL(error) { _PyFrame_StackAssertInvalid(frame); #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "error return without exception set"); _PyFrame_StackPointerInvalidate(frame); } #else assert(_PyErr_Occurred(tstate)); #endif _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); STOP_TRACING(); stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!_PyFrame_IsIncomplete(frame)) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyFrameObject *f = _PyFrame_GetFrameObject(frame); _PyFrame_StackPointerInvalidate(frame); if (f != NULL) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); PyTraceBack_Here(f); _PyFrame_StackPointerInvalidate(frame); } } assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); _PyEval_MonitorRaise(tstate, frame, next_instr-1); _PyFrame_StackPointerInvalidate(frame); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } LABEL(exception_unwind) { STOP_TRACING(); int offset = INSTR_OFFSET()-1; int level, handler, lasti; int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); if (handled == 0) { assert(_PyErr_Occurred(tstate)); _PyStackRef *stackbase = _PyFrame_Stackbase(frame); while (frame->stackpointer > stackbase) { _PyStackRef ref = _PyFrame_StackPop(frame); PyStackRef_XCLOSE(ref); } monitor_unwind(tstate, frame, next_instr-1); JUMP_TO_LABEL(exit_unwind); } assert(STACK_LEVEL() >= level); _PyStackRef *new_top = _PyFrame_Stackbase(frame) + level; assert(frame->stackpointer >= new_top); while (frame->stackpointer > new_top) { _PyStackRef ref = _PyFrame_StackPop(frame); PyStackRef_XCLOSE(ref); } if (lasti) { int frame_lasti = _PyInterpreterFrame_LASTI(frame); _PyStackRef lasti = PyStackRef_TagInt(frame_lasti); _PyFrame_StackPush(frame, lasti); } PyObject *exc = _PyErr_GetRaisedException(tstate); _PyFrame_StackPush(frame, PyStackRef_FromPyObjectSteal(exc)); next_instr = _PyFrame_GetBytecode(frame) + handler; int err = monitor_handled(tstate, frame, next_instr, exc); if (err < 0) { JUMP_TO_LABEL(exception_unwind); } #ifdef Py_DEBUG if (frame->lltrace >= 5) { lltrace_resume_frame(frame); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP int opcode; #endif DISPATCH(); } LABEL(exit_unwind) { assert(_PyErr_Occurred(tstate)); DTRACE_FUNCTION_RETURN(); JUMP_TO_LABEL(exit_unwind_notrace); } LABEL(exit_unwind_notrace) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); frame->return_offset = 0; if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { tstate->current_frame = frame->previous; #if !_Py_TAIL_CALL_INTERP assert(frame == &entry.frame); #endif #ifdef _Py_TIER2 _PyStackRef executor = frame->localsplus[0]; assert(tstate->current_executor == NULL); if (!PyStackRef_IsNull(executor)) { tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); PyStackRef_CLOSE(executor); } #endif return NULL; } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } LABEL(start_frame) { #ifdef Py_DEBUG assert(frame->stackpointer_valid == 1); #endif int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { JUMP_TO_LABEL(exit_unwind_notrace); } DTRACE_FUNCTION_ENTRY(); next_instr = frame->instr_ptr; #ifdef Py_DEBUG int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); if (lltrace < 0) { JUMP_TO_LABEL(exit_unwind); } frame->lltrace = lltrace; assert(!_PyErr_Occurred(tstate)); #endif stack_pointer = _PyFrame_GetStackPointer(frame); _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP int opcode; #endif DISPATCH(); } #if _Py_TAIL_CALL_INTERP && !defined(_Py_TIER2) Py_GCC_ATTRIBUTE((unused)) #endif LABEL(stop_tracing) { #if _Py_TIER2 assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); _PyFrame_StackPointerInvalidate(frame); LEAVE_TRACING(); assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); int err = stop_tracing_and_jit(tstate, frame); _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } DISPATCH_GOTO_NON_TRACING(); #else Py_FatalError("JIT label executed in non-jit build."); #endif } /* END LABELS */ #undef TIER_ONE