From 0edde64a41c2c3eeb4fd495efe7fff3d631cae4b Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 4 Apr 2024 18:49:18 -0400 Subject: [PATCH] GH-117457: Correct pystats uop "miss" counts (GH-117477) --- Python/ceval.c | 2 - Python/executor_cases.c.h | 670 ++++++++++++++++++----- Tools/cases_generator/tier2_generator.py | 10 +- Tools/jit/template.c | 10 +- 4 files changed, 548 insertions(+), 144 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 57ae08ee3cf85a..f718a77fb029cb 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1083,7 +1083,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } #endif OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - UOP_STAT_INC(uopcode, miss); Py_DECREF(current_executor); tstate->previous_executor = NULL; DISPATCH(); @@ -1091,7 +1090,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int exit_to_trace: assert(next_uop[-1].format == UOP_FORMAT_EXIT); OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - UOP_STAT_INC(uopcode, miss); uint32_t exit_index = next_uop[-1].exit_index; assert(exit_index < current_executor->exit_count); _PyExitData *exit = ¤t_executor->exits[exit_index]; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a6e28f69881c1c..9c6e42a1a8e54f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -14,13 +14,19 @@ case _RESUME_CHECK: { #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) JUMP_TO_JUMP_TARGET(); + if (_Py_emscripten_signal_clock == 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); uintptr_t version = _PyFrame_GetCode(frame)->_co_instrumentation_version; assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) JUMP_TO_JUMP_TARGET(); + if (eval_breaker != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -323,7 +329,10 @@ case _TO_BOOL_BOOL: { PyObject *value; value = stack_pointer[-1]; - if (!PyBool_Check(value)) JUMP_TO_JUMP_TARGET(); + if (!PyBool_Check(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); break; } @@ -332,7 +341,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyLong_CheckExact(value)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -350,7 +362,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyList_CheckExact(value)) JUMP_TO_JUMP_TARGET(); + if (!PyList_CheckExact(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; Py_DECREF(value); @@ -363,7 +378,10 @@ PyObject *res; value = stack_pointer[-1]; // This one is a bit weird, because we expect *some* failures: - if (!Py_IsNone(value)) JUMP_TO_JUMP_TARGET(); + if (!Py_IsNone(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); res = Py_False; stack_pointer[-1] = res; @@ -374,7 +392,10 @@ PyObject *value; PyObject *res; value = stack_pointer[-1]; - if (!PyUnicode_CheckExact(value)) JUMP_TO_JUMP_TARGET(); + if (!PyUnicode_CheckExact(value)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -415,8 +436,14 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyLong_CheckExact(left)) JUMP_TO_JUMP_TARGET(); - if (!PyLong_CheckExact(right)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(left)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyLong_CheckExact(right)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -473,8 +500,14 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyFloat_CheckExact(left)) JUMP_TO_JUMP_TARGET(); - if (!PyFloat_CheckExact(right)) JUMP_TO_JUMP_TARGET(); + if (!PyFloat_CheckExact(left)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyFloat_CheckExact(right)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -531,8 +564,14 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyUnicode_CheckExact(left)) JUMP_TO_JUMP_TARGET(); - if (!PyUnicode_CheckExact(right)) JUMP_TO_JUMP_TARGET(); + if (!PyUnicode_CheckExact(left)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_CheckExact(right)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -623,12 +662,24 @@ PyObject *res; sub = stack_pointer[-1]; list = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyList_CheckExact(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Deopt unless 0 <= sub < PyList_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); + if (index >= PyList_GET_SIZE(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); @@ -646,14 +697,29 @@ PyObject *res; sub = stack_pointer[-1]; str = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyUnicode_CheckExact(str)) JUMP_TO_JUMP_TARGET(); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyUnicode_CheckExact(str)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) JUMP_TO_JUMP_TARGET(); + if (PyUnicode_GET_LENGTH(str) <= index) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) JUMP_TO_JUMP_TARGET(); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); @@ -669,12 +735,24 @@ PyObject *res; sub = stack_pointer[-1]; tuple = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyTuple_CheckExact(tuple)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyTuple_CheckExact(tuple)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Deopt unless 0 <= sub < PyTuple_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) JUMP_TO_JUMP_TARGET(); + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); @@ -692,7 +770,10 @@ PyObject *res; sub = stack_pointer[-1]; dict = stack_pointer[-2]; - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_SUBSCR, hit); int rc = PyDict_GetItemRef(dict, sub, &res); if (rc == 0) { @@ -757,13 +838,25 @@ sub = stack_pointer[-1]; list = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); + if (!PyLong_CheckExact(sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyList_CheckExact(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // Ensure nonnegative, zero-or-one-digit ints. - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); + if (index >= PyList_GET_SIZE(list)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); PyList_SET_ITEM(list, index, value); @@ -782,7 +875,10 @@ sub = stack_pointer[-1]; dict = stack_pointer[-2]; value = stack_pointer[-3]; - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); @@ -1072,8 +1168,14 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; assert(oparg == 2); - if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); - if (PyTuple_GET_SIZE(seq) != 2) JUMP_TO_JUMP_TARGET(); + if (!PyTuple_CheckExact(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyTuple_GET_SIZE(seq) != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(UNPACK_SEQUENCE, hit); val0 = Py_NewRef(PyTuple_GET_ITEM(seq, 0)); val1 = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); @@ -1090,8 +1192,14 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyTuple_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); - if (PyTuple_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET(); + if (!PyTuple_CheckExact(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyTuple_GET_SIZE(seq) != oparg) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1108,8 +1216,14 @@ oparg = CURRENT_OPARG(); seq = stack_pointer[-1]; values = &stack_pointer[-1]; - if (!PyList_CheckExact(seq)) JUMP_TO_JUMP_TARGET(); - if (PyList_GET_SIZE(seq) != oparg) JUMP_TO_JUMP_TARGET(); + if (!PyList_CheckExact(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyList_GET_SIZE(seq) != oparg) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1280,8 +1394,14 @@ case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); - if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (dict->ma_keys->dk_version != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1289,8 +1409,14 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)CURRENT_OPERAND(); PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); - if (dict->ma_keys->dk_version != version) JUMP_TO_JUMP_TARGET(); + if (!PyDict_CheckExact(dict)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (dict->ma_keys->dk_version != version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1303,7 +1429,10 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - if (res == NULL) JUMP_TO_JUMP_TARGET(); + if (res == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1321,7 +1450,10 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - if (res == NULL) JUMP_TO_JUMP_TARGET(); + if (res == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1645,8 +1777,14 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); - if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); @@ -1670,8 +1808,14 @@ class = stack_pointer[-2]; global_super = stack_pointer[-3]; assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); - if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); + if (global_super != (PyObject *)&PySuper_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyType_Check(class)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1744,7 +1888,10 @@ uint32_t type_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - if (tp->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET(); + if (tp->tp_version_tag != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -1753,7 +1900,10 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET(); + if (!_PyObject_InlineValues(owner)->valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -1765,7 +1915,10 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); attr = _PyObject_InlineValues(owner)->values[index]; - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1782,7 +1935,10 @@ owner = stack_pointer[-1]; uint16_t index = (uint16_t)CURRENT_OPERAND(); attr = _PyObject_InlineValues(owner)->values[index]; - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1799,10 +1955,16 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t dict_version = (uint32_t)CURRENT_OPERAND(); - if (!PyModule_CheckExact(owner)) JUMP_TO_JUMP_TARGET(); + if (!PyModule_CheckExact(owner)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - if (dict->ma_keys->dk_version != dict_version) JUMP_TO_JUMP_TARGET(); + if (dict->ma_keys->dk_version != dict_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -1818,7 +1980,10 @@ assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1835,7 +2000,10 @@ assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); PyDictObject *dict = managed_dict->dict; - if (dict == NULL) JUMP_TO_JUMP_TARGET(); + if (dict == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(PyDict_CheckExact((PyObject *)dict)); break; } @@ -1849,19 +2017,31 @@ uint16_t hint = (uint16_t)CURRENT_OPERAND(); PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner); PyDictObject *dict = managed_dict->dict; - if (hint >= (size_t)dict->ma_keys->dk_nentries) JUMP_TO_JUMP_TARGET(); + if (hint >= (size_t)dict->ma_keys->dk_nentries) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) JUMP_TO_JUMP_TARGET(); + if (ep->me_key != name) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) JUMP_TO_JUMP_TARGET(); + if (ep->me_key != name) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } attr = ep->me_value; } - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1881,7 +2061,10 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1899,7 +2082,10 @@ uint16_t index = (uint16_t)CURRENT_OPERAND(); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - if (attr == NULL) JUMP_TO_JUMP_TARGET(); + if (attr == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1916,9 +2102,15 @@ PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND(); - if (!PyType_Check(owner)) JUMP_TO_JUMP_TARGET(); + if (!PyType_Check(owner)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(type_version != 0); - if (((PyTypeObject *)owner)->tp_version_tag != type_version) JUMP_TO_JUMP_TARGET(); + if (((PyTypeObject *)owner)->tp_version_tag != type_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -1967,8 +2159,14 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_ManagedDictPointer(owner)->dict) JUMP_TO_JUMP_TARGET(); - if (_PyObject_InlineValues(owner)->valid == 0) JUMP_TO_JUMP_TARGET(); + if (_PyObject_ManagedDictPointer(owner)->dict) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (_PyObject_InlineValues(owner)->valid == 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2063,8 +2261,14 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!_PyLong_IsCompact((PyLongObject *)left)) JUMP_TO_JUMP_TARGET(); - if (!_PyLong_IsCompact((PyLongObject *)right)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsCompact((PyLongObject *)left)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!_PyLong_IsCompact((PyLongObject *)right)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -2143,7 +2347,10 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) JUMP_TO_JUMP_TARGET(); + if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CONTAINS_OP, hit); // Note: both set and frozenset use the same seq_contains method! int res = _PySet_Contains((PySetObject *)right, left); @@ -2163,7 +2370,10 @@ oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; - if (!PyDict_CheckExact(right)) JUMP_TO_JUMP_TARGET(); + if (!PyDict_CheckExact(right)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CONTAINS_OP, hit); int res = PyDict_Contains(right, left); Py_DECREF(left); @@ -2383,7 +2593,10 @@ Py_DECREF(iter); STACK_SHRINK(1); /* The translator sets the deopt target just past END_FOR */ - if (true) JUMP_TO_JUMP_TARGET(); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } // Common case: no jump, leave it to the code generator stack_pointer[0] = next; @@ -2396,7 +2609,10 @@ case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyListIter_Type) JUMP_TO_JUMP_TARGET(); + if (Py_TYPE(iter) != &PyListIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2408,8 +2624,14 @@ _PyListIterObject *it = (_PyListIterObject *)iter; assert(Py_TYPE(iter) == &PyListIter_Type); PyListObject *seq = it->it_seq; - if (seq == NULL) JUMP_TO_JUMP_TARGET(); - if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET(); + if (seq == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if ((size_t)it->it_index >= (size_t)PyList_GET_SIZE(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2431,7 +2653,10 @@ case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - if (Py_TYPE(iter) != &PyTupleIter_Type) JUMP_TO_JUMP_TARGET(); + if (Py_TYPE(iter) != &PyTupleIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2443,8 +2668,14 @@ _PyTupleIterObject *it = (_PyTupleIterObject *)iter; assert(Py_TYPE(iter) == &PyTupleIter_Type); PyTupleObject *seq = it->it_seq; - if (seq == NULL) JUMP_TO_JUMP_TARGET(); - if (it->it_index >= PyTuple_GET_SIZE(seq)) JUMP_TO_JUMP_TARGET(); + if (seq == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (it->it_index >= PyTuple_GET_SIZE(seq)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2467,7 +2698,10 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - if (Py_TYPE(r) != &PyRangeIter_Type) JUMP_TO_JUMP_TARGET(); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2478,7 +2712,10 @@ iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; assert(Py_TYPE(r) == &PyRangeIter_Type); - if (r->len <= 0) JUMP_TO_JUMP_TARGET(); + if (r->len <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2565,7 +2802,10 @@ PyObject *owner; owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET(); + if (!_PyObject_InlineValues(owner)->valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2575,7 +2815,10 @@ uint32_t keys_version = (uint32_t)CURRENT_OPERAND(); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - if (owner_heap_type->ht_cached_keys->dk_version != keys_version) JUMP_TO_JUMP_TARGET(); + if (owner_heap_type->ht_cached_keys->dk_version != keys_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2657,7 +2900,10 @@ char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = *(PyObject **)ptr; /* This object has a __dict__, just not yet created */ - if (dict != NULL) JUMP_TO_JUMP_TARGET(); + if (dict != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2695,8 +2941,14 @@ oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - if (null != NULL) JUMP_TO_JUMP_TARGET(); - if (Py_TYPE(callable) != &PyMethod_Type) JUMP_TO_JUMP_TARGET(); + if (null != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2718,7 +2970,10 @@ } case _CHECK_PEP_523: { - if (tstate->interp->eval_frame) JUMP_TO_JUMP_TARGET(); + if (tstate->interp->eval_frame) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2729,11 +2984,20 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - if (!PyFunction_Check(callable)) JUMP_TO_JUMP_TARGET(); + if (!PyFunction_Check(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyFunctionObject *func = (PyFunctionObject *)callable; - if (func->func_version != func_version) JUMP_TO_JUMP_TARGET(); + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (self_or_null != NULL)) JUMP_TO_JUMP_TARGET(); + if (code->co_argcount != oparg + (self_or_null != NULL)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2743,8 +3007,14 @@ callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) JUMP_TO_JUMP_TARGET(); - if (tstate->py_recursion_remaining <= 1) JUMP_TO_JUMP_TARGET(); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (tstate->py_recursion_remaining <= 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2926,8 +3196,14 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) JUMP_TO_JUMP_TARGET(); - if (callable != (PyObject *)&PyType_Type) JUMP_TO_JUMP_TARGET(); + if (null != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable != (PyObject *)&PyType_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(arg)); Py_DECREF(arg); @@ -2946,8 +3222,14 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) JUMP_TO_JUMP_TARGET(); - if (callable != (PyObject *)&PyUnicode_Type) JUMP_TO_JUMP_TARGET(); + if (null != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable != (PyObject *)&PyUnicode_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); res = PyObject_Str(arg); Py_DECREF(arg); @@ -2967,8 +3249,14 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; assert(oparg == 1); - if (null != NULL) JUMP_TO_JUMP_TARGET(); - if (callable != (PyObject *)&PyTuple_Type) JUMP_TO_JUMP_TARGET(); + if (null != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (callable != (PyObject *)&PyTuple_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); res = PySequence_Tuple(arg); Py_DECREF(arg); @@ -3008,9 +3296,15 @@ args--; total_args++; } - if (!PyType_Check(callable)) JUMP_TO_JUMP_TARGET(); + if (!PyType_Check(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyTypeObject *tp = (PyTypeObject *)callable; - if (tp->tp_vectorcall == NULL) JUMP_TO_JUMP_TARGET(); + if (tp->tp_vectorcall == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); /* Free the arguments. */ @@ -3039,11 +3333,23 @@ args--; total_args++; } - if (total_args != 1) JUMP_TO_JUMP_TARGET(); - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); - if (PyCFunction_GET_FLAGS(callable) != METH_O) JUMP_TO_JUMP_TARGET(); + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!PyCFunction_CheckExact(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable) != METH_O) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); PyObject *arg = args[0]; @@ -3074,8 +3380,14 @@ args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); - if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); + if (!PyCFunction_CheckExact(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -3110,8 +3422,14 @@ args--; total_args++; } - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); - if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); + if (!PyCFunction_CheckExact(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ PyCFunctionFastWithKeywords cfunc = @@ -3145,9 +3463,15 @@ args--; total_args++; } - if (total_args != 1) JUMP_TO_JUMP_TARGET(); + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.len) JUMP_TO_JUMP_TARGET(); + if (callable != interp->callable_cache.len) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); @@ -3181,9 +3505,15 @@ args--; total_args++; } - if (total_args != 2) JUMP_TO_JUMP_TARGET(); + if (total_args != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyInterpreterState *interp = tstate->interp; - if (callable != interp->callable_cache.isinstance) JUMP_TO_JUMP_TARGET(); + if (callable != interp->callable_cache.isinstance) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; @@ -3219,15 +3549,30 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (total_args != 2) JUMP_TO_JUMP_TARGET(); - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + if (total_args != 2) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) JUMP_TO_JUMP_TARGET(); + if (meth->ml_flags != METH_O) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyObject *arg = args[1]; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3258,12 +3603,21 @@ total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, d_type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(self, d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = @@ -3296,15 +3650,30 @@ args--; total_args++; } - if (total_args != 1) JUMP_TO_JUMP_TARGET(); + if (total_args != 1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); - if (meth->ml_flags != METH_NOARGS) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (meth->ml_flags != METH_NOARGS) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); + if (tstate->c_recursion_remaining <= 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; _Py_EnterRecursiveCallTstateUnchecked(tstate); @@ -3335,11 +3704,20 @@ } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); + if (meth->ml_flags != METH_FASTCALL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; @@ -3543,7 +3921,10 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsTrue(flag)) JUMP_TO_JUMP_TARGET(); + if (!Py_IsTrue(flag)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(Py_IsTrue(flag)); break; } @@ -3552,7 +3933,10 @@ PyObject *flag; flag = stack_pointer[-1]; stack_pointer += -1; - if (!Py_IsFalse(flag)) JUMP_TO_JUMP_TARGET(); + if (!Py_IsFalse(flag)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } assert(Py_IsFalse(flag)); break; } @@ -3563,7 +3947,10 @@ stack_pointer += -1; if (!Py_IsNone(val)) { Py_DECREF(val); - if (1) JUMP_TO_JUMP_TARGET(); + if (1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } break; } @@ -3572,7 +3959,10 @@ PyObject *val; val = stack_pointer[-1]; stack_pointer += -1; - if (Py_IsNone(val)) JUMP_TO_JUMP_TARGET(); + if (Py_IsNone(val)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } Py_DECREF(val); break; } @@ -3611,12 +4001,18 @@ } case _EXIT_TRACE: { - if (1) JUMP_TO_JUMP_TARGET(); + if (1) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } case _CHECK_VALIDITY: { - if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET(); + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -3676,7 +4072,10 @@ case _CHECK_FUNCTION: { uint32_t func_version = (uint32_t)CURRENT_OPERAND(); assert(PyFunction_Check(frame->f_funcobj)); - if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) JUMP_TO_JUMP_TARGET(); + if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } @@ -3743,7 +4142,10 @@ case _CHECK_VALIDITY_AND_SET_IP: { PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND(); - if (!current_executor->vm_data.valid) JUMP_TO_JUMP_TARGET(); + if (!current_executor->vm_data.valid) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; break; } diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 114d28ee745632..944d134f12a18e 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -100,7 +100,10 @@ def tier2_replace_deopt( out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon - out.emit(") JUMP_TO_JUMP_TARGET();\n") + out.emit(") {\n") + out.emit("UOP_STAT_INC(uopcode, miss);\n") + out.emit("JUMP_TO_JUMP_TARGET();\n"); + out.emit("}\n") def tier2_replace_exit_if( @@ -115,7 +118,10 @@ def tier2_replace_exit_if( out.emit(next(tkn_iter)) emit_to(out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon - out.emit(") JUMP_TO_JUMP_TARGET();\n") + out.emit(") {\n") + out.emit("UOP_STAT_INC(uopcode, miss);\n") + out.emit("JUMP_TO_JUMP_TARGET();\n") + out.emit("}\n") def tier2_replace_oparg( diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 351bc2f3dd48de..2300bd0f1f31ec 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -85,7 +85,7 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * // Locals that the instruction implementations expect to exist: PATCH_VALUE(_PyExecutorObject *, current_executor, _JIT_EXECUTOR) int oparg; - int opcode = _JIT_OPCODE; + int uopcode = _JIT_OPCODE; // Other stuff we need handy: PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) PATCH_VALUE(uint64_t, _operand, _JIT_OPERAND) @@ -93,14 +93,14 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * PATCH_VALUE(uint16_t, _exit_index, _JIT_EXIT_INDEX) OPT_STAT_INC(uops_executed); - UOP_STAT_INC(opcode, execution_count); + UOP_STAT_INC(uopcode, execution_count); // The actual instruction definitions (only one will be used): - if (opcode == _JUMP_TO_TOP) { + if (uopcode == _JUMP_TO_TOP) { CHECK_EVAL_BREAKER(); PATCH_JUMP(_JIT_TOP); } - switch (opcode) { + switch (uopcode) { #include "executor_cases.c.h" default: Py_UNREACHABLE(); @@ -113,11 +113,9 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * GOTO_TIER_ONE(NULL); exit_to_tier1: tstate->previous_executor = (PyObject *)current_executor; - UOP_STAT_INC(opcode, miss); GOTO_TIER_ONE(_PyCode_CODE(_PyFrame_GetCode(frame)) + _target); exit_to_trace: { - UOP_STAT_INC(opcode, miss); _PyExitData *exit = ¤t_executor->exits[_exit_index]; Py_INCREF(exit->executor); tstate->previous_executor = (PyObject *)current_executor;