From e77bb5ae6c8ee233d3b48f8f3a23f03e0e5f467d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Mon, 2 Mar 2026 20:23:15 +0900 Subject: [PATCH 01/12] Add CALL_ALLOC_AND_ENTER_INIT specialization Optimizes user-defined class instantiation MyClass(args...) when tp_new == object.__new__ and __init__ is a simple PyFunction. Allocates the object directly and calls __init__ via invoke_exact_args, bypassing the generic type.__call__ dispatch path. --- crates/vm/src/frame.rs | 85 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 41ebd067c7d..219d0b321a7 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -4172,6 +4172,65 @@ impl ExecutingFrame<'_> { let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } + Instruction::CallAllocAndEnterInit => { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); + let nargs: u32 = arg.into(); + let callable = self.nth_value(nargs + 1); + let stack = &self.state.stack; + let stack_len = stack.len(); + let self_or_null_is_some = stack[stack_len - nargs as usize - 1].is_some(); + if !self_or_null_is_some + && cached_version != 0 + && let Some(cls) = callable.downcast_ref::() + && cls.tp_version_tag.load(Acquire) == cached_version + { + // Look up __init__ (guarded by type_version) + if let Some(init) = cls.get_attr(identifier!(vm, __init__)) + && let Some(init_func) = init.downcast_ref::() + { + // Allocate object directly (tp_new == object.__new__) + let dict = if cls + .slots + .flags + .has_feature(crate::types::PyTypeFlags::HAS_DICT) + { + Some(vm.ctx.new_dict()) + } else { + None + }; + let cls_ref = cls.to_owned(); + let new_obj: PyObjectRef = + PyRef::new_ref(PyBaseObject, cls_ref, dict).into(); + + // Build args: [new_obj, arg1, ..., argN] + let pos_args: Vec = + self.pop_multiple(nargs as usize).collect(); + let _null = self.pop_value_opt(); // self_or_null (None) + let _callable = self.pop_value(); // callable (type) + + let mut all_args = Vec::with_capacity(pos_args.len() + 1); + all_args.push(new_obj.clone()); + all_args.extend(pos_args); + + let init_result = init_func.invoke_exact_args(&all_args, vm)?; + + // EXIT_INIT_CHECK: __init__ must return None + if !vm.is_none(&init_result) { + return Err( + vm.new_type_error("__init__() should return None".to_owned()) + ); + } + + self.push_value(new_obj); + return Ok(None); + } + } + self.deoptimize_call(); + let args = self.collect_positional_args(nargs); + self.execute_call(args, vm) + } Instruction::CallMethodDescriptorFastWithKeywords => { // Native function interface is uniform regardless of keyword support let instr_idx = self.lasti() as usize - 1; @@ -7034,6 +7093,32 @@ impl ExecutingFrame<'_> { return; } } + // CallAllocAndEnterInit: heap type with default __new__ + if let Some(cls) = callable.downcast_ref::() + && cls.slots.flags.has_feature(PyTypeFlags::HEAPTYPE) + { + let object_new = vm.ctx.types.object_type.slots.new.load(); + let cls_new = cls.slots.new.load(); + if let (Some(cls_new_fn), Some(obj_new_fn)) = (cls_new, object_new) + && cls_new_fn as usize == obj_new_fn as usize + && let Some(init) = cls.get_attr(identifier!(vm, __init__)) + && let Some(init_func) = init.downcast_ref::() + && init_func.can_specialize_call(nargs + 1) + { + let version = cls.tp_version_tag.load(Acquire); + if version != 0 { + unsafe { + self.code + .instructions + .replace_op(instr_idx, Instruction::CallAllocAndEnterInit); + self.code + .instructions + .write_cache_u32(cache_base + 1, version); + } + return; + } + } + } // General builtin class call (any type with Callable) let callable_tag = callable as *const PyObject as u32; unsafe { From 4ab342d701f9e8a0a228909e4fb4ae7c9ea03cd0 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 3 Mar 2026 09:22:12 +0900 Subject: [PATCH 02/12] Invalidate JIT cache when __code__ is reassigned Change jitted_code from OnceCell to PyMutex> so it can be cleared on __code__ assignment. The setter now sets the cached JIT code to None to prevent executing stale machine code. --- crates/vm/src/builtins/function.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index cad82acc0e3..a41b119a6f1 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -5,8 +5,6 @@ use super::{ PyAsyncGen, PyCode, PyCoroutine, PyDictRef, PyGenerator, PyModule, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, }; -#[cfg(feature = "jit")] -use crate::common::lock::OnceCell; use crate::common::lock::PyMutex; use crate::function::ArgMapping; use crate::object::{PyAtomicRef, Traverse, TraverseFn}; @@ -75,7 +73,7 @@ pub struct PyFunction { doc: PyMutex, func_version: AtomicU32, #[cfg(feature = "jit")] - jitted_code: OnceCell, + jitted_code: PyMutex>, } static FUNC_VERSION_COUNTER: AtomicU32 = AtomicU32::new(1); @@ -214,7 +212,7 @@ impl PyFunction { doc: PyMutex::new(doc), func_version: AtomicU32::new(next_func_version()), #[cfg(feature = "jit")] - jitted_code: OnceCell::new(), + jitted_code: PyMutex::new(None), }; Ok(func) } @@ -538,7 +536,7 @@ impl Py { vm: &VirtualMachine, ) -> PyResult { #[cfg(feature = "jit")] - if let Some(jitted_code) = self.jitted_code.get() { + if let Some(jitted_code) = self.jitted_code.lock().as_ref() { use crate::convert::ToPyObject; match jit::get_jit_args(self, &func_args, jitted_code, vm) { Ok(args) => { @@ -712,6 +710,10 @@ impl PyFunction { #[pygetset(setter)] fn set___code__(&self, code: PyRef, vm: &VirtualMachine) { self.code.swap_to_temporary_refs(code, vm); + #[cfg(feature = "jit")] + { + *self.jitted_code.lock() = None; + } self.func_version.store(0, Relaxed); } @@ -948,7 +950,7 @@ impl PyFunction { #[cfg(feature = "jit")] #[pymethod] fn __jit__(zelf: PyRef, vm: &VirtualMachine) -> PyResult<()> { - if zelf.jitted_code.get().is_some() { + if zelf.jitted_code.lock().is_some() { return Ok(()); } let arg_types = jit::get_jit_arg_types(&zelf, vm)?; @@ -956,7 +958,7 @@ impl PyFunction { let code: &Py = &zelf.code; let compiled = rustpython_jit::compile(&code.code, &arg_types, ret_type) .map_err(|err| jit::new_jit_error(err.to_string(), vm))?; - let _ = zelf.jitted_code.set(compiled); + *zelf.jitted_code.lock() = Some(compiled); Ok(()) } } From 5e9ec261f2f1b18f414db435770577cc3f689052 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 3 Mar 2026 09:27:18 +0900 Subject: [PATCH 03/12] Atomic operations for specialization cache - range iterator: deduplicate fast_next/next_fast - Replace raw pointer reads/writes in CodeUnits with atomic operations (AtomicU8/AtomicU16) for thread safety - Add read_op (Acquire), read_arg (Relaxed), compare_exchange_op - Use Release ordering in replace_op to synchronize cache writes - Dispatch loop reads opcodes atomically via read_op/read_arg - Fix adaptive counter access: use read/write_adaptive_counter instead of read/write_cache_u16 (was reading wrong bytes) - Add pre-check guards to all specialize_* functions to prevent concurrent specialization races - Move modified() before attribute changes in type.__setattr__ to prevent use-after-free of cached descriptors - Use SeqCst ordering in modified() for version invalidation - Add Release fence after quicken() initialization --- crates/compiler-core/src/bytecode.rs | 124 ++++++++++++++++------- crates/vm/src/builtins/type.rs | 9 +- crates/vm/src/dict_inner.rs | 9 +- crates/vm/src/frame.rs | 144 +++++++++++++++++++++------ 4 files changed, 216 insertions(+), 70 deletions(-) diff --git a/crates/compiler-core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs index cec04b9edd9..d3dda5090ab 100644 --- a/crates/compiler-core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -8,7 +8,12 @@ use crate::{ }; use alloc::{borrow::ToOwned, boxed::Box, collections::BTreeSet, fmt, string::String, vec::Vec}; use bitflags::bitflags; -use core::{cell::UnsafeCell, hash, mem, ops::Deref}; +use core::{ + cell::UnsafeCell, + hash, mem, + ops::Deref, + sync::atomic::{AtomicU8, AtomicU16, Ordering}, +}; use itertools::Itertools; use malachite_bigint::BigInt; use num_complex::Complex64; @@ -367,8 +372,13 @@ impl TryFrom<&[u8]> for CodeUnit { pub struct CodeUnits(UnsafeCell>); -// SAFETY: All mutation of the inner buffer is serialized by `monitoring_data: PyMutex` -// in `PyCode`. The `UnsafeCell` is required because `replace_op` mutates through `&self`. +// SAFETY: All cache operations use atomic read/write instructions. +// - replace_op / compare_exchange_op: AtomicU8 store/CAS (Release) +// - cache read/write: AtomicU16 load/store (Relaxed) +// - adaptive counter: AtomicU8 load/store (Relaxed) +// Ordering is established by: +// - replace_op (Release) ↔ dispatch loop read_op (Acquire) for cache data visibility +// - tp_version_tag (Acquire) for descriptor pointer validity unsafe impl Sync for CodeUnits {} impl Clone for CodeUnits { @@ -435,45 +445,81 @@ impl Deref for CodeUnits { impl CodeUnits { /// Replace the opcode at `index` in-place without changing the arg byte. + /// Uses atomic Release store to ensure prior cache writes are visible + /// to threads that subsequently read the new opcode with Acquire. /// /// # Safety /// - `index` must be in bounds. /// - `new_op` must have the same arg semantics as the original opcode. - /// - The caller must ensure exclusive access to the instruction buffer - /// (no concurrent reads or writes to the same `CodeUnits`). pub unsafe fn replace_op(&self, index: usize, new_op: Instruction) { - unsafe { - let units = &mut *self.0.get(); - let unit_ptr = units.as_mut_ptr().add(index); - // Write only the opcode byte (first byte of CodeUnit due to #[repr(C)]) - let op_ptr = unit_ptr as *mut u8; - core::ptr::write(op_ptr, new_op.into()); - } + let units = unsafe { &*self.0.get() }; + let ptr = units.as_ptr().wrapping_add(index) as *const AtomicU8; + unsafe { &*ptr }.store(new_op.into(), Ordering::Release); + } + + /// Atomically replace opcode only if it still matches `expected`. + /// Returns true on success. Uses Release ordering on success. + /// + /// # Safety + /// - `index` must be in bounds. + pub unsafe fn compare_exchange_op( + &self, + index: usize, + expected: Instruction, + new_op: Instruction, + ) -> bool { + let units = unsafe { &*self.0.get() }; + let ptr = units.as_ptr().wrapping_add(index) as *const AtomicU8; + unsafe { &*ptr } + .compare_exchange( + expected.into(), + new_op.into(), + Ordering::Release, + Ordering::Relaxed, + ) + .is_ok() + } + + /// Atomically read the opcode at `index` with Acquire ordering. + /// Pairs with `replace_op` (Release) to ensure cache data visibility. + pub fn read_op(&self, index: usize) -> Instruction { + let units = unsafe { &*self.0.get() }; + let ptr = units.as_ptr().wrapping_add(index) as *const AtomicU8; + let byte = unsafe { &*ptr }.load(Ordering::Acquire); + // SAFETY: Only valid Instruction values are stored via replace_op/compare_exchange_op. + unsafe { mem::transmute::(byte) } + } + + /// Atomically read the arg byte at `index` with Relaxed ordering. + pub fn read_arg(&self, index: usize) -> OpArgByte { + let units = unsafe { &*self.0.get() }; + let ptr = units.as_ptr().wrapping_add(index) as *const u8; + let arg_ptr = unsafe { ptr.add(1) } as *const AtomicU8; + OpArgByte::from(unsafe { &*arg_ptr }.load(Ordering::Relaxed)) } /// Write a u16 value into a CACHE code unit at `index`. /// Each CodeUnit is 2 bytes (#[repr(C)]: op u8 + arg u8), so one u16 fits exactly. + /// Uses Relaxed atomic store; ordering is provided by replace_op (Release). /// /// # Safety /// - `index` must be in bounds and point to a CACHE entry. - /// - The caller must ensure no concurrent reads/writes to the same slot. pub unsafe fn write_cache_u16(&self, index: usize, value: u16) { - unsafe { - let units = &mut *self.0.get(); - let ptr = units.as_mut_ptr().add(index) as *mut u8; - core::ptr::write_unaligned(ptr as *mut u16, value); - } + let units = unsafe { &*self.0.get() }; + let ptr = units.as_ptr().wrapping_add(index) as *const AtomicU16; + unsafe { &*ptr }.store(value, Ordering::Relaxed); } /// Read a u16 value from a CACHE code unit at `index`. + /// Uses Relaxed atomic load; ordering is provided by read_op (Acquire). /// /// # Panics /// Panics if `index` is out of bounds. pub fn read_cache_u16(&self, index: usize) -> u16 { let units = unsafe { &*self.0.get() }; assert!(index < units.len(), "read_cache_u16: index out of bounds"); - let ptr = units.as_ptr().wrapping_add(index) as *const u8; - unsafe { core::ptr::read_unaligned(ptr as *const u16) } + let ptr = units.as_ptr().wrapping_add(index) as *const AtomicU16; + unsafe { &*ptr }.load(Ordering::Relaxed) } /// Write a u32 value across two consecutive CACHE code units starting at `index`. @@ -518,36 +564,40 @@ impl CodeUnits { lo | (hi << 32) } - /// Read the adaptive counter from the first CACHE entry's `arg` byte. - /// This preserves `op = Instruction::Cache`, unlike `read_cache_u16`. + /// Read the adaptive counter from the CACHE entry's `arg` byte at `index`. + /// Uses Relaxed atomic load. pub fn read_adaptive_counter(&self, index: usize) -> u8 { let units = unsafe { &*self.0.get() }; - u8::from(units[index].arg) + let ptr = units.as_ptr().wrapping_add(index) as *const u8; + let arg_ptr = unsafe { ptr.add(1) } as *const AtomicU8; + unsafe { &*arg_ptr }.load(Ordering::Relaxed) } - /// Write the adaptive counter to the first CACHE entry's `arg` byte. - /// This preserves `op = Instruction::Cache`, unlike `write_cache_u16`. + /// Write the adaptive counter to the CACHE entry's `arg` byte at `index`. + /// Uses Relaxed atomic store. /// /// # Safety /// - `index` must be in bounds and point to a CACHE entry. pub unsafe fn write_adaptive_counter(&self, index: usize, value: u8) { - let units = unsafe { &mut *self.0.get() }; - units[index].arg = OpArgByte::from(value); + let units = unsafe { &*self.0.get() }; + let ptr = units.as_ptr().wrapping_add(index) as *const u8; + let arg_ptr = unsafe { ptr.add(1) } as *const AtomicU8; + unsafe { &*arg_ptr }.store(value, Ordering::Relaxed); } /// Produce a clean copy of the bytecode suitable for serialization /// (marshal) and `co_code`. Specialized opcodes are mapped back to their /// base variants via `deoptimize()` and all CACHE entries are zeroed. pub fn original_bytes(&self) -> Vec { - let units = unsafe { &*self.0.get() }; - let mut out = Vec::with_capacity(units.len() * 2); - let len = units.len(); + let len = self.len(); + let mut out = Vec::with_capacity(len * 2); let mut i = 0; while i < len { - let op = units[i].op.deoptimize(); + let op = self.read_op(i).deoptimize(); + let arg = self.read_arg(i); let caches = op.cache_entries(); out.push(u8::from(op)); - out.push(u8::from(units[i].arg)); + out.push(u8::from(arg)); // Zero-fill all CACHE entries (counter + cached data) for _ in 0..caches { i += 1; @@ -562,12 +612,12 @@ impl CodeUnits { /// Initialize adaptive warmup counters for all cacheable instructions. /// Called lazily at RESUME (first execution of a code object). /// Uses the `arg` byte of the first CACHE entry, preserving `op = Instruction::Cache`. + /// All writes are atomic (Relaxed) to avoid data races with concurrent readers. pub fn quicken(&self) { - let units = unsafe { &mut *self.0.get() }; - let len = units.len(); + let len = self.len(); let mut i = 0; while i < len { - let op = units[i].op; + let op = self.read_op(i); let caches = op.cache_entries(); if caches > 0 { // Don't write adaptive counter for instrumented opcodes; @@ -575,7 +625,9 @@ impl CodeUnits { if !op.is_instrumented() { let cache_base = i + 1; if cache_base < len { - units[cache_base].arg = OpArgByte::from(ADAPTIVE_WARMUP_VALUE); + unsafe { + self.write_adaptive_counter(cache_base, ADAPTIVE_WARMUP_VALUE); + } } } i += 1 + caches; diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 212b2101cef..276a81f58b6 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -347,7 +347,7 @@ impl PyType { if old_version == 0 { return; } - self.tp_version_tag.store(0, Ordering::Release); + self.tp_version_tag.store(0, Ordering::SeqCst); // Release strong references held by cache entries for this version. // We hold owned refs that would prevent GC of class attributes after // type deletion. @@ -2168,6 +2168,11 @@ impl SetAttr for PyType { } let assign = value.is_assign(); + // Invalidate inline caches before modifying attributes. + // This ensures other threads see the version invalidation before + // any attribute changes, preventing use-after-free of cached descriptors. + zelf.modified(); + if let PySetterValue::Assign(value) = value { zelf.attributes.write().insert(attr_name, value); } else { @@ -2180,8 +2185,6 @@ impl SetAttr for PyType { ))); } } - // Invalidate inline caches that depend on this type's attributes - zelf.modified(); if attr_name.as_wtf8().starts_with("__") && attr_name.as_wtf8().ends_with("__") { if assign { diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index e4d8174abbd..2a77ea7d991 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -19,7 +19,10 @@ use crate::{ use alloc::fmt; use core::mem::size_of; use core::ops::ControlFlow; -use core::sync::atomic::{AtomicU64, Ordering::Relaxed}; +use core::sync::atomic::{ + AtomicU64, + Ordering::{Acquire, Release}, +}; use num_traits::ToPrimitive; // HashIndex is intended to be same size with hash::PyHash @@ -261,12 +264,12 @@ type PopInnerResult = ControlFlow>>; impl Dict { /// Monotonically increasing version counter for mutation tracking. pub fn version(&self) -> u64 { - self.version.load(Relaxed) + self.version.load(Acquire) } /// Bump the version counter after any mutation. fn bump_version(&self) { - self.version.fetch_add(1, Relaxed); + self.version.fetch_add(1, Release); } fn read(&self) -> PyRwLockReadGuard<'_, DictInner> { diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 219d0b321a7..389d4d464c5 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -713,7 +713,6 @@ impl ExecutingFrame<'_> { obj_name = self.code.obj_name )); // Execute until return or exception: - let instructions = &self.code.instructions; let mut arg_state = bytecode::OpArgState::default(); loop { let idx = self.lasti() as usize; @@ -730,8 +729,8 @@ impl ExecutingFrame<'_> { if vm.use_tracing.get() && !vm.is_none(&self.object.trace.lock()) && !matches!( - instructions.get(idx).map(|u| u.op), - Some(Instruction::Resume { .. } | Instruction::InstrumentedResume) + self.code.instructions.read_op(idx), + Instruction::Resume { .. } | Instruction::InstrumentedResume ) && let Some((loc, _)) = self.code.locations.get(idx) && loc.line.get() as u32 != self.state.prev_line @@ -753,8 +752,8 @@ impl ExecutingFrame<'_> { continue; } } - let bytecode::CodeUnit { op, arg } = instructions[idx]; - let arg = arg_state.extend(arg); + let op = self.code.instructions.read_op(idx); + let arg = arg_state.extend(self.code.instructions.read_arg(idx)); let mut do_extend_arg = false; let caches = op.cache_entries(); @@ -1415,12 +1414,12 @@ impl ExecutingFrame<'_> { let nargs = argc.get(arg); let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_cache_u16(cache_base); + let counter = self.code.instructions.read_adaptive_counter(cache_base); if counter > 0 { unsafe { self.code .instructions - .write_cache_u16(cache_base, counter - 1); + .write_adaptive_counter(cache_base, counter - 1); } } else { self.specialize_call_kw(vm, nargs, instr_idx, cache_base); @@ -1468,12 +1467,12 @@ impl ExecutingFrame<'_> { let op_val = opname.get(arg); let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_cache_u16(cache_base); + let counter = self.code.instructions.read_adaptive_counter(cache_base); if counter > 0 { unsafe { self.code .instructions - .write_cache_u16(cache_base, counter - 1); + .write_adaptive_counter(cache_base, counter - 1); } } else { self.specialize_compare_op(vm, op_val, instr_idx, cache_base); @@ -1483,12 +1482,12 @@ impl ExecutingFrame<'_> { Instruction::ContainsOp { invert } => { let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_cache_u16(cache_base); + let counter = self.code.instructions.read_adaptive_counter(cache_base); if counter > 0 { unsafe { self.code .instructions - .write_cache_u16(cache_base, counter - 1); + .write_adaptive_counter(cache_base, counter - 1); } } else { self.specialize_contains_op(vm, instr_idx, cache_base); @@ -1677,12 +1676,12 @@ impl ExecutingFrame<'_> { let target = bytecode::Label(self.lasti() + 1 + u32::from(arg)); let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_cache_u16(cache_base); + let counter = self.code.instructions.read_adaptive_counter(cache_base); if counter > 0 { unsafe { self.code .instructions - .write_cache_u16(cache_base, counter - 1); + .write_adaptive_counter(cache_base, counter - 1); } } else { self.specialize_for_iter(vm, instr_idx, cache_base); @@ -1912,12 +1911,12 @@ impl ExecutingFrame<'_> { Instruction::LoadSuperAttr { namei } => { let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_cache_u16(cache_base); + let counter = self.code.instructions.read_adaptive_counter(cache_base); if counter > 0 { unsafe { self.code .instructions - .write_cache_u16(cache_base, counter - 1); + .write_adaptive_counter(cache_base, counter - 1); } } else { self.specialize_load_super_attr(vm, namei.get(arg), instr_idx, cache_base); @@ -2172,12 +2171,12 @@ impl ExecutingFrame<'_> { let oparg = namei.get(arg); let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_cache_u16(cache_base); + let counter = self.code.instructions.read_adaptive_counter(cache_base); if counter > 0 { unsafe { self.code .instructions - .write_cache_u16(cache_base, counter - 1); + .write_adaptive_counter(cache_base, counter - 1); } } else { self.specialize_load_global(vm, oparg, instr_idx, cache_base); @@ -2530,6 +2529,7 @@ impl ExecutingFrame<'_> { // Lazy quickening: initialize adaptive counters on first execution if !self.code.quickened.swap(true, atomic::Ordering::Relaxed) { self.code.instructions.quicken(); + atomic::fence(atomic::Ordering::Release); } // Check if bytecode needs re-instrumentation let global_ver = vm @@ -2658,12 +2658,12 @@ impl ExecutingFrame<'_> { Instruction::StoreAttr { namei } => { let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_cache_u16(cache_base); + let counter = self.code.instructions.read_adaptive_counter(cache_base); if counter > 0 { unsafe { self.code .instructions - .write_cache_u16(cache_base, counter - 1); + .write_adaptive_counter(cache_base, counter - 1); } } else { self.specialize_store_attr(vm, namei.get(arg), instr_idx, cache_base); @@ -2736,12 +2736,12 @@ impl ExecutingFrame<'_> { Instruction::StoreSubscr => { let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_cache_u16(cache_base); + let counter = self.code.instructions.read_adaptive_counter(cache_base); if counter > 0 { unsafe { self.code .instructions - .write_cache_u16(cache_base, counter - 1); + .write_adaptive_counter(cache_base, counter - 1); } } else { self.specialize_store_subscr(vm, instr_idx, cache_base); @@ -2768,12 +2768,12 @@ impl ExecutingFrame<'_> { Instruction::ToBool => { let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_cache_u16(cache_base); + let counter = self.code.instructions.read_adaptive_counter(cache_base); if counter > 0 { unsafe { self.code .instructions - .write_cache_u16(cache_base, counter - 1); + .write_adaptive_counter(cache_base, counter - 1); } } else { self.specialize_to_bool(vm, instr_idx, cache_base); @@ -2790,12 +2790,12 @@ impl ExecutingFrame<'_> { Instruction::UnpackSequence { count } => { let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_cache_u16(cache_base); + let counter = self.code.instructions.read_adaptive_counter(cache_base); if counter > 0 { unsafe { self.code .instructions - .write_cache_u16(cache_base, counter - 1); + .write_adaptive_counter(cache_base, counter - 1); } } else { self.specialize_unpack_sequence(vm, instr_idx, cache_base); @@ -2849,12 +2849,12 @@ impl ExecutingFrame<'_> { // (receiver, v -- receiver, retval) let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_cache_u16(cache_base); + let counter = self.code.instructions.read_adaptive_counter(cache_base); if counter > 0 { unsafe { self.code .instructions - .write_cache_u16(cache_base, counter - 1); + .write_adaptive_counter(cache_base, counter - 1); } } else { self.specialize_send(instr_idx, cache_base); @@ -5226,7 +5226,10 @@ impl ExecutingFrame<'_> { // Scan backwards past CACHE entries to find the branch instruction let mut branch_idx = not_taken_idx.saturating_sub(1); while branch_idx > 0 - && matches!(self.code.instructions[branch_idx].op, Instruction::Cache) + && matches!( + self.code.instructions.read_op(branch_idx), + Instruction::Cache + ) { branch_idx -= 1; } @@ -6563,6 +6566,13 @@ impl ExecutingFrame<'_> { instr_idx: usize, cache_base: usize, ) { + // Pre-check: bail if already specialized by another thread + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::LoadAttr { .. } + ) { + return; + } let obj = self.top_value(); let cls = obj.class(); @@ -6870,6 +6880,12 @@ impl ExecutingFrame<'_> { instr_idx: usize, cache_base: usize, ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::BinaryOp { .. } + ) { + return; + } let b = self.top_value(); let a = self.nth_value(1); @@ -6985,6 +7001,12 @@ impl ExecutingFrame<'_> { instr_idx: usize, cache_base: usize, ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::Call { .. } + ) { + return; + } // Stack: [callable, self_or_null, arg1, ..., argN] // callable is at position nargs + 1 from top // self_or_null is at position nargs from top @@ -7152,6 +7174,12 @@ impl ExecutingFrame<'_> { instr_idx: usize, cache_base: usize, ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::CallKw { .. } + ) { + return; + } // Stack: [callable, self_or_null, arg1, ..., argN, kwarg_names] // callable is at position nargs + 2 from top let stack = &self.state.stack; @@ -7197,6 +7225,12 @@ impl ExecutingFrame<'_> { } fn specialize_send(&mut self, instr_idx: usize, cache_base: usize) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::Send { .. } + ) { + return; + } // Stack: [receiver, val] — receiver is at position 1 let receiver = self.nth_value(1); if self.builtin_coro(receiver).is_some() { @@ -7221,6 +7255,12 @@ impl ExecutingFrame<'_> { instr_idx: usize, cache_base: usize, ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::LoadSuperAttr { .. } + ) { + return; + } // Stack: [global_super, class, self] let global_super = self.nth_value(2); let class = self.nth_value(1); @@ -7253,6 +7293,12 @@ impl ExecutingFrame<'_> { instr_idx: usize, cache_base: usize, ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::CompareOp { .. } + ) { + return; + } let b = self.top_value(); let a = self.nth_value(1); @@ -7310,6 +7356,12 @@ impl ExecutingFrame<'_> { } fn specialize_to_bool(&mut self, vm: &VirtualMachine, instr_idx: usize, _cache_base: usize) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::ToBool + ) { + return; + } let obj = self.top_value(); let cls = obj.class(); @@ -7360,6 +7412,12 @@ impl ExecutingFrame<'_> { } fn specialize_for_iter(&mut self, vm: &VirtualMachine, instr_idx: usize, cache_base: usize) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::ForIter { .. } + ) { + return; + } let iter = self.top_value(); let new_op = if iter.downcast_ref_if_exact::(vm).is_some() { @@ -7461,6 +7519,12 @@ impl ExecutingFrame<'_> { instr_idx: usize, cache_base: usize, ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::LoadGlobal(..) + ) { + return; + } let name = self.code.names[(oparg >> 1) as usize]; // Check if name exists in globals let in_globals = self.globals.get_item_opt(name, vm).ok().flatten().is_some(); @@ -7518,6 +7582,12 @@ impl ExecutingFrame<'_> { instr_idx: usize, cache_base: usize, ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::StoreSubscr + ) { + return; + } // Stack: [value, obj, idx] — obj is TOS-1 let obj = self.nth_value(1); let idx = self.top_value(); @@ -7559,6 +7629,12 @@ impl ExecutingFrame<'_> { } fn specialize_contains_op(&mut self, vm: &VirtualMachine, instr_idx: usize, cache_base: usize) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::ContainsOp(..) + ) { + return; + } let haystack = self.top_value(); // b = TOS = haystack let new_op = if haystack.downcast_ref_if_exact::(vm).is_some() { Some(Instruction::ContainsOpDict) @@ -7603,6 +7679,12 @@ impl ExecutingFrame<'_> { instr_idx: usize, cache_base: usize, ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::UnpackSequence { .. } + ) { + return; + } let obj = self.top_value(); let new_op = if let Some(tuple) = obj.downcast_ref_if_exact::(vm) { if tuple.len() == 2 { @@ -7652,6 +7734,12 @@ impl ExecutingFrame<'_> { instr_idx: usize, cache_base: usize, ) { + if !matches!( + self.code.instructions.read_op(instr_idx), + Instruction::StoreAttr { .. } + ) { + return; + } // TOS = owner (the object being assigned to) let owner = self.top_value(); let cls = owner.class(); From c9a40ceeae23c675de004b2bd3f83692d4907f9d Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Tue, 3 Mar 2026 21:59:49 +0900 Subject: [PATCH 04/12] Fix slot wrapper override for inherited attributes For __getattribute__: only use getattro_wrapper when the type itself defines the attribute; otherwise inherit native slot from base class via MRO. For __setattr__/__delattr__: only store setattro_wrapper when the type has its own __setattr__ or __delattr__; otherwise keep the inherited base slot. --- crates/vm/src/types/slot.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 93447fb07f7..5a505013ce4 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -830,10 +830,16 @@ impl PyType { } }) { self.slots.getattro.store(Some(func)); - } else { - // __getattribute__ is a Python method somewhere in MRO; + } else if self.attributes.read().contains_key(name) { + // Self defines __getattribute__ as a non-native method; // use the wrapper to dispatch through it. self.slots.getattro.store(Some(getattro_wrapper)); + } else { + // __getattribute__ not defined in self; inherit native + // slot from base class. This handles the common case where + // object.__getattribute__ is a method_descriptor alongside + // the native getattro slot. + accessor.inherit_from_mro(self); } } else { accessor.inherit_from_mro(self); @@ -848,7 +854,15 @@ impl PyType { }) { self.slots.setattro.store(Some(func)); } else { - self.slots.setattro.store(Some(setattro_wrapper)); + let has_own = { + let guard = self.attributes.read(); + guard.contains_key(identifier!(ctx, __setattr__)) + || guard.contains_key(identifier!(ctx, __delattr__)) + }; + if has_own { + self.slots.setattro.store(Some(setattro_wrapper)); + } + // If !has_own: slot already inherited from base during type creation } } else { accessor.inherit_from_mro(self); From 2bd0d10c16c530b7354fade73a6ae5df1e56958a Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 00:01:19 +0900 Subject: [PATCH 05/12] Fix StoreAttrSlot cache overflow corrupting next instruction write_cache_u32 at cache_base+3 writes 2 code units (positions 3 and 4), but STORE_ATTR only has 4 cache entries (positions 0-3). This overwrites the next instruction with the upper 16 bits of the slot offset. Changed to write_cache_u16/read_cache_u16 since member descriptor offsets fit within u16 (max 65535 bytes). --- crates/vm/src/frame.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 389d4d464c5..ad586b17cd9 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -3408,7 +3408,7 @@ impl ExecutingFrame<'_> { if version_match { let slot_offset = - self.code.instructions.read_cache_u32(cache_base + 3) as usize; + self.code.instructions.read_cache_u16(cache_base + 3) as usize; let owner = self.pop_value(); let value = self.pop_value(); owner.set_slot(slot_offset, Some(value)); @@ -7793,7 +7793,7 @@ impl ExecutingFrame<'_> { .write_cache_u32(cache_base + 1, type_version); self.code .instructions - .write_cache_u32(cache_base + 3, offset as u32); + .write_cache_u16(cache_base + 3, offset as u16); self.code .instructions .replace_op(instr_idx, Instruction::StoreAttrSlot); From 7bd53009a87b13bfb0db78c55258b4c93802ed68 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 00:09:17 +0900 Subject: [PATCH 06/12] Exclude method_descriptor from has_python_cmp check has_python_cmp incorrectly treated method_descriptor as Python-level comparison methods, causing richcompare slot to use wrapper dispatch instead of inheriting the native slot. --- crates/vm/src/types/slot.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 5a505013ce4..62344ca9afe 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -912,8 +912,9 @@ impl PyType { let attrs = cls.attributes.read(); cmp_names.iter().any(|n| { if let Some(attr) = attrs.get(*n) { - // Check if it's a Python function (not wrapper_descriptor) + // Check if it's a Python function (not a native descriptor) !attr.class().is(ctx.types.wrapper_descriptor_type) + && !attr.class().is(ctx.types.method_descriptor_type) } else { false } From 5c81306fa4b11ef634ce63d80029a92fcd769b82 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 00:13:57 +0900 Subject: [PATCH 07/12] Fix CompareOpFloat NaN handling partial_cmp returns None for NaN comparisons. is_some_and incorrectly returned false for all NaN comparisons, but NaN != x should be true per IEEE 754 semantics. --- crates/vm/src/frame.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index ad586b17cd9..6a60ca5ab64 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -4584,10 +4584,12 @@ impl ExecutingFrame<'_> { b.downcast_ref_if_exact::(vm), ) { let op = self.compare_op_from_arg(arg); - let result = a_f - .to_f64() - .partial_cmp(&b_f.to_f64()) - .is_some_and(|ord| op.eval_ord(ord)); + let (a, b) = (a_f.to_f64(), b_f.to_f64()); + // Use Rust's IEEE 754 float comparison which handles NaN correctly + let result = match a.partial_cmp(&b) { + Some(ord) => op.eval_ord(ord), + None => op == PyComparisonOp::Ne, // NaN != anything is true + }; self.pop_value(); self.pop_value(); self.push_value(vm.ctx.new_bool(result).into()); From 5a1b29b314140eaa80b7b6c4b9312a2cd0db286f Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 00:55:34 +0900 Subject: [PATCH 08/12] Fix invoke_exact_args borrow in CallAllocAndEnterInit --- crates/vm/src/frame.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 6a60ca5ab64..10ca33f379e 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -4214,7 +4214,7 @@ impl ExecutingFrame<'_> { all_args.push(new_obj.clone()); all_args.extend(pos_args); - let init_result = init_func.invoke_exact_args(&all_args, vm)?; + let init_result = init_func.invoke_exact_args(all_args, vm)?; // EXIT_INIT_CHECK: __init__ must return None if !vm.is_none(&init_result) { From f99e95fd9d24bdfd970a38e9f54f2c16c3e854dc Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 01:09:32 +0900 Subject: [PATCH 09/12] Distinguish Python method vs not-found in slot MRO lookup Change lookup_slot_in_mro to return a 3-state SlotLookupResult enum (NativeSlot/PythonMethod/NotFound) instead of Option. Previously, both "found a Python-level method" and "found nothing" returned None, causing incorrect slot inheritance. For example, class Test(Mixin, TestCase) would inherit object.slot_init from Mixin via inherit_from_mro instead of using init_wrapper to dispatch TestCase.__init__. Apply this fix consistently to all slot update sites: update_main_slot!, update_sub_slot!, TpGetattro, TpSetattro, TpDescrSet, TpHash, TpRichcompare, SqAssItem, MpAssSubscript. --- crates/vm/src/types/slot.rs | 224 +++++++++++++++++++++++------------- 1 file changed, 141 insertions(+), 83 deletions(-) diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index 62344ca9afe..58040f7928c 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -628,6 +628,18 @@ fn del_wrapper(zelf: &PyObject, vm: &VirtualMachine) -> PyResult<()> { Ok(()) } +/// Result of looking up a slot function in MRO. +enum SlotLookupResult { + /// Found a native slot function from a wrapper_descriptor. + NativeSlot(T), + /// Found a Python-level method (not a native slot). + /// The caller should use the wrapper function. + PythonMethod, + /// No method with this name found in MRO at all. + /// The caller should inherit the slot from MRO. + NotFound, +} + impl PyType { /// Update slots based on dunder method changes /// @@ -688,16 +700,22 @@ impl PyType { macro_rules! update_main_slot { ($slot:ident, $wrapper:expr, $variant:ident) => {{ if ADD { - if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| { + match self.lookup_slot_in_mro(name, ctx, |sf| { if let SlotFunc::$variant(f) = sf { Some(*f) } else { None } }) { - self.slots.$slot.store(Some(func)); - } else { - self.slots.$slot.store(Some($wrapper)); + SlotLookupResult::NativeSlot(func) => { + self.slots.$slot.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.$slot.store(Some($wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } } } else { accessor.inherit_from_mro(self); @@ -731,16 +749,24 @@ impl PyType { }; if has_own { self.slots.$group.$slot.store(Some($wrapper)); - } else if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| { - if let SlotFunc::$variant(f) = sf { - Some(*f) - } else { - None - } - }) { - self.slots.$group.$slot.store(Some(func)); } else { - self.slots.$group.$slot.store(Some($wrapper)); + match self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::$variant(f) = sf { + Some(*f) + } else { + None + } + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.$group.$slot.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.$group.$slot.store(Some($wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } } } else { accessor.inherit_from_mro(self); @@ -764,16 +790,24 @@ impl PyType { if method.as_ref().is_some_and(|m| m.is(&ctx.none)) { self.slots.hash.store(Some(hash_not_implemented)); - } else if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| { - if let SlotFunc::Hash(f) = sf { - Some(*f) - } else { - None - } - }) { - self.slots.hash.store(Some(func)); } else { - self.slots.hash.store(Some(hash_wrapper)); + match self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::Hash(f) = sf { + Some(*f) + } else { + None + } + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.hash.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.hash.store(Some(hash_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } } } else { accessor.inherit_from_mro(self); @@ -822,24 +856,22 @@ impl PyType { // Must use wrapper to handle __getattr__ self.slots.getattro.store(Some(getattro_wrapper)); } else if ADD { - if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| { + match self.lookup_slot_in_mro(name, ctx, |sf| { if let SlotFunc::GetAttro(f) = sf { Some(*f) } else { None } }) { - self.slots.getattro.store(Some(func)); - } else if self.attributes.read().contains_key(name) { - // Self defines __getattribute__ as a non-native method; - // use the wrapper to dispatch through it. - self.slots.getattro.store(Some(getattro_wrapper)); - } else { - // __getattribute__ not defined in self; inherit native - // slot from base class. This handles the common case where - // object.__getattribute__ is a method_descriptor alongside - // the native getattro slot. - accessor.inherit_from_mro(self); + SlotLookupResult::NativeSlot(func) => { + self.slots.getattro.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.getattro.store(Some(getattro_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } } } else { accessor.inherit_from_mro(self); @@ -848,21 +880,19 @@ impl PyType { SlotAccessor::TpSetattro => { // __setattr__ and __delattr__ share the same slot if ADD { - if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| match sf { + match self.lookup_slot_in_mro(name, ctx, |sf| match sf { SlotFunc::SetAttro(f) | SlotFunc::DelAttro(f) => Some(*f), _ => None, }) { - self.slots.setattro.store(Some(func)); - } else { - let has_own = { - let guard = self.attributes.read(); - guard.contains_key(identifier!(ctx, __setattr__)) - || guard.contains_key(identifier!(ctx, __delattr__)) - }; - if has_own { + SlotLookupResult::NativeSlot(func) => { + self.slots.setattro.store(Some(func)); + } + SlotLookupResult::PythonMethod => { self.slots.setattro.store(Some(setattro_wrapper)); } - // If !has_own: slot already inherited from base during type creation + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } } } else { accessor.inherit_from_mro(self); @@ -872,13 +902,19 @@ impl PyType { SlotAccessor::TpDescrSet => { // __set__ and __delete__ share the same slot if ADD { - if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| match sf { + match self.lookup_slot_in_mro(name, ctx, |sf| match sf { SlotFunc::DescrSet(f) | SlotFunc::DescrDel(f) => Some(*f), _ => None, }) { - self.slots.descr_set.store(Some(func)); - } else { - self.slots.descr_set.store(Some(descr_set_wrapper)); + SlotLookupResult::NativeSlot(func) => { + self.slots.descr_set.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.descr_set.store(Some(descr_set_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } } } else { accessor.inherit_from_mro(self); @@ -925,16 +961,24 @@ impl PyType { if has_python_cmp { // Use wrapper to call the Python method self.slots.richcompare.store(Some(richcompare_wrapper)); - } else if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| { - if let SlotFunc::RichCompare(f, _) = sf { - Some(*f) - } else { - None - } - }) { - self.slots.richcompare.store(Some(func)); } else { - self.slots.richcompare.store(Some(richcompare_wrapper)); + match self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::RichCompare(f, _) = sf { + Some(*f) + } else { + None + } + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.richcompare.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.richcompare.store(Some(richcompare_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } } } else { accessor.inherit_from_mro(self); @@ -1377,16 +1421,24 @@ impl PyType { .as_sequence .ass_item .store(Some(sequence_setitem_wrapper)); - } else if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| match sf { - SlotFunc::SeqSetItem(f) | SlotFunc::SeqDelItem(f) => Some(*f), - _ => None, - }) { - self.slots.as_sequence.ass_item.store(Some(func)); } else { - self.slots - .as_sequence - .ass_item - .store(Some(sequence_setitem_wrapper)); + match self.lookup_slot_in_mro(name, ctx, |sf| match sf { + SlotFunc::SeqSetItem(f) | SlotFunc::SeqDelItem(f) => Some(*f), + _ => None, + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.as_sequence.ass_item.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots + .as_sequence + .ass_item + .store(Some(sequence_setitem_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } } } else { accessor.inherit_from_mro(self); @@ -1422,16 +1474,24 @@ impl PyType { .as_mapping .ass_subscript .store(Some(mapping_setitem_wrapper)); - } else if let Some(func) = self.lookup_slot_in_mro(name, ctx, |sf| match sf { - SlotFunc::MapSetSubscript(f) | SlotFunc::MapDelSubscript(f) => Some(*f), - _ => None, - }) { - self.slots.as_mapping.ass_subscript.store(Some(func)); } else { - self.slots - .as_mapping - .ass_subscript - .store(Some(mapping_setitem_wrapper)); + match self.lookup_slot_in_mro(name, ctx, |sf| match sf { + SlotFunc::MapSetSubscript(f) | SlotFunc::MapDelSubscript(f) => Some(*f), + _ => None, + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.as_mapping.ass_subscript.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots + .as_mapping + .ass_subscript + .store(Some(mapping_setitem_wrapper)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } } } else { accessor.inherit_from_mro(self); @@ -1444,14 +1504,12 @@ impl PyType { } /// Look up a method in MRO and extract the slot function if it's a slot wrapper. - /// Returns Some(slot_func) if a matching slot wrapper is found, None if a real method - /// is found or no method exists. fn lookup_slot_in_mro( &self, name: &'static PyStrInterned, ctx: &Context, extract: impl Fn(&crate::builtins::descriptor::SlotFunc) -> Option, - ) -> Option { + ) -> SlotLookupResult { use crate::builtins::descriptor::PyWrapper; // Helper to check if a class is a subclass of another by checking MRO @@ -1482,9 +1540,9 @@ impl PyType { // Look up in self's dict first if let Some(attr) = self.attributes.read().get(name).cloned() { if let Some(func) = try_extract(&attr, &mro) { - return Some(func); + return SlotLookupResult::NativeSlot(func); } - return None; + return SlotLookupResult::PythonMethod; } // Look up in MRO (mro[0] is self, so skip it) @@ -1492,13 +1550,13 @@ impl PyType { if let Some(attr) = cls.attributes.read().get(name).cloned() { // Use the slice starting from this class in MRO if let Some(func) = try_extract(&attr, &mro[i + 1..]) { - return Some(func); + return SlotLookupResult::NativeSlot(func); } - return None; + return SlotLookupResult::PythonMethod; } } // No method found in MRO - None + SlotLookupResult::NotFound } } From 8821456d8ff2d2d08a3b3403445627ca826446f5 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 01:47:49 +0900 Subject: [PATCH 10/12] Extract specialization helper functions to reduce boilerplate - deoptimize() / deoptimize_at(): replace specialized op with base op - adaptive(): decrement warmup counter or call specialize function - commit_specialization(): replace op on success, backoff on failure - execute_binary_op_int() / execute_binary_op_float(): typed binary ops Removes 10 duplicate deoptimize_* functions, consolidates 13 adaptive counter blocks, 6 binary op handlers, and 7 specialize tail patterns. Also replaces inline deopt blocks in LoadAttr/StoreAttr handlers. --- crates/vm/src/frame.rs | 1028 ++++++++++++---------------------------- 1 file changed, 307 insertions(+), 721 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 10ca33f379e..55357a2bb42 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -883,8 +883,8 @@ impl ExecutingFrame<'_> { // new traceback entries. // EndAsyncFor and CleanupThrow also re-raise non-matching exceptions. let is_reraise = match op { - Instruction::RaiseVarargs { argc } => matches!( - argc.get(arg), + Instruction::RaiseVarargs { argc: kind } => matches!( + kind.get(arg), bytecode::RaiseKind::BareRaise | bytecode::RaiseKind::ReraiseFromStack ), Instruction::Reraise { .. } @@ -897,9 +897,9 @@ impl ExecutingFrame<'_> { // need contextualization even if the exception has prior traceback let is_new_raise = matches!( op, - Instruction::RaiseVarargs { argc } + Instruction::RaiseVarargs { argc: kind } if matches!( - argc.get(arg), + kind.get(arg), bytecode::RaiseKind::Raise | bytecode::RaiseKind::RaiseCause ) ); @@ -1251,20 +1251,7 @@ impl ExecutingFrame<'_> { match instruction { Instruction::BinaryOp { op } => { let op_val = op.get(arg); - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_binary_op(vm, op_val, instr_idx, cache_base); - } - + self.adaptive(|s, ii, cb| s.specialize_binary_op(vm, op_val, ii, cb)); self.execute_bin_op(vm, op_val) } // TODO: In CPython, this does in-place unicode concatenation when @@ -1282,7 +1269,7 @@ impl ExecutingFrame<'_> { self.push_value(result.to_pyobject(vm)); Ok(None) } else { - self.deoptimize_binary_op(bytecode::BinaryOperator::InplaceAdd); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); self.execute_bin_op(vm, bytecode::BinaryOperator::InplaceAdd) } } @@ -1302,17 +1289,17 @@ impl ExecutingFrame<'_> { self.push_value(result); Ok(None) } - Instruction::BuildList { count } => { - let sz = count.get(arg) as usize; + Instruction::BuildList { count: size } => { + let sz = size.get(arg) as usize; let elements = self.pop_multiple(sz).collect(); let list_obj = vm.ctx.new_list(elements); self.push_value(list_obj.into()); Ok(None) } - Instruction::BuildMap { count } => self.execute_build_map(vm, count.get(arg)), - Instruction::BuildSet { count } => { + Instruction::BuildMap { count: size } => self.execute_build_map(vm, size.get(arg)), + Instruction::BuildSet { count: size } => { let set = PySet::default().into_ref(&vm.ctx); - for element in self.pop_multiple(count.get(arg) as usize) { + for element in self.pop_multiple(size.get(arg) as usize) { set.add(element, vm)?; } self.push_value(set.into()); @@ -1329,16 +1316,16 @@ impl ExecutingFrame<'_> { Ok(None) } */ - Instruction::BuildString { count } => { + Instruction::BuildString { count: size } => { let s: Wtf8Buf = self - .pop_multiple(count.get(arg) as usize) + .pop_multiple(size.get(arg) as usize) .map(|pyobj| pyobj.downcast::().unwrap()) .collect(); self.push_value(vm.ctx.new_str(s).into()); Ok(None) } - Instruction::BuildTuple { count } => { - let elements = self.pop_multiple(count.get(arg) as usize).collect(); + Instruction::BuildTuple { count: size } => { + let elements = self.pop_multiple(size.get(arg) as usize).collect(); let list_obj = vm.ctx.new_tuple(elements); self.push_value(list_obj.into()); Ok(None) @@ -1359,10 +1346,10 @@ impl ExecutingFrame<'_> { self.push_value(template.into_pyobject(vm)); Ok(None) } - Instruction::BuildInterpolation { format } => { + Instruction::BuildInterpolation { format: oparg } => { // oparg encoding: (conversion << 2) | has_format_spec // Stack: [value, expression_str, (format_spec)?] -> [interpolation] - let oparg_val = format.get(arg); + let oparg_val = oparg.get(arg); let has_format_spec = (oparg_val & 1) != 0; let conversion_code = oparg_val >> 2; @@ -1393,37 +1380,15 @@ impl ExecutingFrame<'_> { self.push_value(interpolation.into_pyobject(vm)); Ok(None) } - Instruction::Call { argc } => { + Instruction::Call { argc: nargs } => { // Stack: [callable, self_or_null, arg1, ..., argN] - let nargs_val = argc.get(arg); - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_call(vm, nargs_val, instr_idx, cache_base); - } + let nargs_val = nargs.get(arg); + self.adaptive(|s, ii, cb| s.specialize_call(vm, nargs_val, ii, cb)); self.execute_call_vectorcall(nargs_val, vm) } - Instruction::CallKw { argc } => { - let nargs = argc.get(arg); - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_call_kw(vm, nargs, instr_idx, cache_base); - } + Instruction::CallKw { argc: nargs } => { + let nargs = nargs.get(arg); + self.adaptive(|s, ii, cb| s.specialize_call_kw(vm, nargs, ii, cb)); // Stack: [callable, self_or_null, arg1, ..., argN, kwarg_names] self.execute_call_kw_vectorcall(nargs, vm) } @@ -1463,36 +1428,13 @@ impl ExecutingFrame<'_> { self.push_value(matched); Ok(None) } - Instruction::CompareOp { opname } => { - let op_val = opname.get(arg); - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_compare_op(vm, op_val, instr_idx, cache_base); - } + Instruction::CompareOp { opname: op } => { + let op_val = op.get(arg); + self.adaptive(|s, ii, cb| s.specialize_compare_op(vm, op_val, ii, cb)); self.execute_compare(vm, op_val) } Instruction::ContainsOp { invert } => { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_contains_op(vm, instr_idx, cache_base); - } - + self.adaptive(|s, ii, cb| s.specialize_contains_op(vm, ii, cb)); let b = self.pop_value(); let a = self.pop_value(); @@ -1503,12 +1445,14 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(value).into()); Ok(None) } - Instruction::ConvertValue { oparg } => self.convert_value(oparg.get(arg), vm), - Instruction::Copy { i } => { + Instruction::ConvertValue { oparg: conversion } => { + self.convert_value(conversion.get(arg), vm) + } + Instruction::Copy { i: index } => { // CopyItem { index: 1 } copies TOS // CopyItem { index: 2 } copies second from top // This is 1-indexed to match CPython - let idx = i.get(arg) as usize; + let idx = index.get(arg) as usize; let stack_len = self.state.stack.len(); debug_assert!(stack_len >= idx, "CopyItem: stack underflow"); let value = self.state.stack[stack_len - idx].clone(); @@ -1519,14 +1463,14 @@ impl ExecutingFrame<'_> { // Free vars are already set up at frame creation time in RustPython Ok(None) } - Instruction::DeleteAttr { namei } => self.delete_attr(vm, namei.get(arg)), + Instruction::DeleteAttr { namei: idx } => self.delete_attr(vm, idx.get(arg)), Instruction::DeleteDeref { i } => { self.state.cells_frees[i.get(arg) as usize].set(None); Ok(None) } - Instruction::DeleteFast { var_num } => { + Instruction::DeleteFast { var_num: idx } => { let fastlocals = unsafe { self.fastlocals.borrow_mut() }; - let idx = var_num.get(arg) as usize; + let idx = idx.get(arg) as usize; if fastlocals[idx].is_none() { return Err(vm.new_exception_msg( vm.ctx.exceptions.unbound_local_error.to_owned(), @@ -1540,8 +1484,8 @@ impl ExecutingFrame<'_> { fastlocals[idx] = None; Ok(None) } - Instruction::DeleteGlobal { namei } => { - let name = self.code.names[namei.get(arg) as usize]; + Instruction::DeleteGlobal { namei: idx } => { + let name = self.code.names[idx.get(arg) as usize]; match self.globals.del_item(name, vm) { Ok(()) => {} Err(e) if e.fast_isinstance(vm.ctx.exceptions.key_error) => { @@ -1551,8 +1495,8 @@ impl ExecutingFrame<'_> { } Ok(None) } - Instruction::DeleteName { namei } => { - let name = self.code.names[namei.get(arg) as usize]; + Instruction::DeleteName { namei: idx } => { + let name = self.code.names[idx.get(arg) as usize]; let res = self.locals.mapping(vm).ass_subscript(name, None, vm); match res { @@ -1565,12 +1509,12 @@ impl ExecutingFrame<'_> { Ok(None) } Instruction::DeleteSubscr => self.execute_delete_subscript(vm), - Instruction::DictUpdate { i } => { + Instruction::DictUpdate { i: index } => { // Stack before: [..., dict, ..., source] (source at TOS) // Stack after: [..., dict, ...] (source consumed) // The dict to update is at position TOS-i (before popping source) - let idx = i.get(arg); + let idx = index.get(arg); // Pop the source from TOS let source = self.pop_value(); @@ -1601,9 +1545,9 @@ impl ExecutingFrame<'_> { dict.merge_object(source, vm)?; Ok(None) } - Instruction::DictMerge { i } => { + Instruction::DictMerge { i: index } => { let source = self.pop_value(); - let idx = i.get(arg); + let idx = index.get(arg); // Get the dict to merge into (same logic as DICT_UPDATE) let dict_ref = if idx <= 1 { @@ -1674,18 +1618,7 @@ impl ExecutingFrame<'_> { Instruction::ForIter { .. } => { // Relative forward jump: target = lasti + caches + delta let target = bytecode::Label(self.lasti() + 1 + u32::from(arg)); - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_for_iter(vm, instr_idx, cache_base); - } + self.adaptive(|s, ii, cb| s.specialize_for_iter(vm, ii, cb)); self.execute_for_iter(vm, target)?; Ok(None) } @@ -1838,13 +1771,13 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_int(len).into()); Ok(None) } - Instruction::ImportFrom { namei } => { - let obj = self.import_from(vm, namei.get(arg))?; + Instruction::ImportFrom { namei: idx } => { + let obj = self.import_from(vm, idx.get(arg))?; self.push_value(obj); Ok(None) } - Instruction::ImportName { namei } => { - self.import(vm, Some(self.code.names[namei.get(arg) as usize]))?; + Instruction::ImportName { namei: idx } => { + self.import(vm, Some(self.code.names[idx.get(arg) as usize]))?; Ok(None) } Instruction::IsOp { invert } => { @@ -1907,21 +1840,11 @@ impl ExecutingFrame<'_> { })?; Ok(None) } - Instruction::LoadAttr { namei } => self.load_attr(vm, namei.get(arg)), - Instruction::LoadSuperAttr { namei } => { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_load_super_attr(vm, namei.get(arg), instr_idx, cache_base); - } - self.load_super_attr(vm, namei.get(arg)) + Instruction::LoadAttr { namei: idx } => self.load_attr(vm, idx.get(arg)), + Instruction::LoadSuperAttr { namei: idx } => { + let idx_val = idx.get(arg); + self.adaptive(|s, ii, cb| s.specialize_load_super_attr(vm, idx_val, ii, cb)); + self.load_super_attr(vm, idx_val) } Instruction::LoadBuildClass => { let build_class = if let Some(builtins_dict) = self.builtins_dict { @@ -1983,10 +1906,10 @@ impl ExecutingFrame<'_> { }); Ok(None) } - Instruction::LoadFromDictOrGlobals { i } => { + Instruction::LoadFromDictOrGlobals { i: idx } => { // PEP 649: Pop dict from stack (classdict), check there first, then globals let dict = self.pop_value(); - let name = self.code.names[i.get(arg) as usize]; + let name = self.code.names[idx.get(arg) as usize]; // Only treat KeyError as "not found", propagate other exceptions let value = if let Some(dict_obj) = dict.downcast_ref::() { @@ -2006,8 +1929,8 @@ impl ExecutingFrame<'_> { }); Ok(None) } - Instruction::LoadConst { consti } => { - self.push_value(self.code.constants[consti.get(arg) as usize].clone().into()); + Instruction::LoadConst { consti: idx } => { + self.push_value(self.code.constants[idx.get(arg) as usize].clone().into()); Ok(None) } Instruction::LoadCommonConstant { idx } => { @@ -2026,9 +1949,9 @@ impl ExecutingFrame<'_> { self.push_value(value); Ok(None) } - Instruction::LoadSmallInt { i } => { + Instruction::LoadSmallInt { i: idx } => { // Push small integer (-5..=256) directly without constant table lookup - let value = vm.ctx.new_int(i.get(arg) as i32); + let value = vm.ctx.new_int(idx.get(arg) as i32); self.push_value(value.into()); Ok(None) } @@ -2040,7 +1963,7 @@ impl ExecutingFrame<'_> { self.push_value(x); Ok(None) } - Instruction::LoadFast { var_num } => { + Instruction::LoadFast { var_num: idx } => { #[cold] fn reference_error( varname: &'static PyStrInterned, @@ -2051,27 +1974,27 @@ impl ExecutingFrame<'_> { format!("local variable '{varname}' referenced before assignment").into(), ) } - let idx = var_num.get(arg) as usize; + let idx = idx.get(arg) as usize; let x = unsafe { self.fastlocals.borrow() }[idx] .clone() .ok_or_else(|| reference_error(self.code.varnames[idx], vm))?; self.push_value(x); Ok(None) } - Instruction::LoadFastAndClear { var_num } => { + Instruction::LoadFastAndClear { var_num: idx } => { // Load value and clear the slot (for inlined comprehensions) // If slot is empty, push None (not an error - variable may not exist yet) - let idx = var_num.get(arg) as usize; + let idx = idx.get(arg) as usize; let x = unsafe { self.fastlocals.borrow_mut() }[idx] .take() .unwrap_or_else(|| vm.ctx.none()); self.push_value(x); Ok(None) } - Instruction::LoadFastCheck { var_num } => { + Instruction::LoadFastCheck { var_num: idx } => { // Same as LoadFast but explicitly checks for unbound locals // (LoadFast in RustPython already does this check) - let idx = var_num.get(arg) as usize; + let idx = idx.get(arg) as usize; let x = unsafe { self.fastlocals.borrow() }[idx] .clone() .ok_or_else(|| { @@ -2087,10 +2010,10 @@ impl ExecutingFrame<'_> { self.push_value(x); Ok(None) } - Instruction::LoadFastLoadFast { var_nums } => { + Instruction::LoadFastLoadFast { var_nums: packed } => { // Load two local variables at once // oparg encoding: (idx1 << 4) | idx2 - let oparg = var_nums.get(arg); + let oparg = packed.get(arg); let idx1 = (oparg >> 4) as usize; let idx2 = (oparg & 15) as usize; let fastlocals = unsafe { self.fastlocals.borrow() }; @@ -2121,8 +2044,8 @@ impl ExecutingFrame<'_> { // Borrow optimization not yet active; falls back to clone. // push_borrowed() is available but disabled until stack // lifetime issues at yield/exception points are resolved. - Instruction::LoadFastBorrow { var_num } => { - let idx = var_num.get(arg) as usize; + Instruction::LoadFastBorrow { var_num: idx } => { + let idx = idx.get(arg) as usize; let x = unsafe { self.fastlocals.borrow() }[idx] .clone() .ok_or_else(|| { @@ -2138,8 +2061,8 @@ impl ExecutingFrame<'_> { self.push_value(x); Ok(None) } - Instruction::LoadFastBorrowLoadFastBorrow { var_nums } => { - let oparg = var_nums.get(arg); + Instruction::LoadFastBorrowLoadFastBorrow { var_nums: packed } => { + let oparg = packed.get(arg); let idx1 = (oparg >> 4) as usize; let idx2 = (oparg & 15) as usize; let fastlocals = unsafe { self.fastlocals.borrow() }; @@ -2167,20 +2090,9 @@ impl ExecutingFrame<'_> { self.push_value(x2); Ok(None) } - Instruction::LoadGlobal { namei } => { - let oparg = namei.get(arg); - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_load_global(vm, oparg, instr_idx, cache_base); - } + Instruction::LoadGlobal { namei: idx } => { + let oparg = idx.get(arg); + self.adaptive(|s, ii, cb| s.specialize_load_global(vm, oparg, ii, cb)); let name = &self.code.names[(oparg >> 1) as usize]; let x = self.load_global_or_builtin(name, vm)?; self.push_value(x); @@ -2189,8 +2101,8 @@ impl ExecutingFrame<'_> { } Ok(None) } - Instruction::LoadName { namei } => { - let name = self.code.names[namei.get(arg) as usize]; + Instruction::LoadName { namei: idx } => { + let name = self.code.names[idx.get(arg) as usize]; let result = self.locals.mapping(vm).subscript(name, vm); match result { Ok(x) => self.push_value(x), @@ -2246,14 +2158,14 @@ impl ExecutingFrame<'_> { dict.set_item(&*key, value, vm)?; Ok(None) } - Instruction::MatchClass { count } => { + Instruction::MatchClass { count: nargs } => { // STACK[-1] is a tuple of keyword attribute names, STACK[-2] is the class being matched against, and STACK[-3] is the match subject. // nargs is the number of positional sub-patterns. let kwd_attrs = self.pop_value(); let kwd_attrs = kwd_attrs.downcast_ref::().unwrap(); let cls = self.pop_value(); let subject = self.pop_value(); - let nargs_val = count.get(arg) as usize; + let nargs_val = nargs.get(arg) as usize; // Check if subject is an instance of cls if subject.is_instance(cls.as_ref(), vm)? { @@ -2524,7 +2436,7 @@ impl ExecutingFrame<'_> { self.push_null(); Ok(None) } - Instruction::RaiseVarargs { argc } => self.execute_raise(vm, argc.get(arg)), + Instruction::RaiseVarargs { argc: kind } => self.execute_raise(vm, kind.get(arg)), Instruction::Resume { .. } => { // Lazy quickening: initialize adaptive counters on first execution if !self.code.quickened.swap(true, atomic::Ordering::Relaxed) { @@ -2651,34 +2563,24 @@ impl ExecutingFrame<'_> { Err(exc) } } - Instruction::SetFunctionAttribute { flag } => { - self.execute_set_function_attribute(vm, flag.get(arg)) + Instruction::SetFunctionAttribute { flag: attr } => { + self.execute_set_function_attribute(vm, attr.get(arg)) } Instruction::SetupAnnotations => self.setup_annotations(vm), - Instruction::StoreAttr { namei } => { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_store_attr(vm, namei.get(arg), instr_idx, cache_base); - } - self.store_attr(vm, namei.get(arg)) + Instruction::StoreAttr { namei: idx } => { + let idx_val = idx.get(arg); + self.adaptive(|s, ii, cb| s.specialize_store_attr(vm, idx_val, ii, cb)); + self.store_attr(vm, idx_val) } Instruction::StoreDeref { i } => { let value = self.pop_value(); self.state.cells_frees[i.get(arg) as usize].set(Some(value)); Ok(None) } - Instruction::StoreFast { var_num } => { + Instruction::StoreFast { var_num: idx } => { let value = self.pop_value(); let fastlocals = unsafe { self.fastlocals.borrow_mut() }; - fastlocals[var_num.get(arg) as usize] = Some(value); + fastlocals[idx.get(arg) as usize] = Some(value); Ok(None) } Instruction::StoreFastLoadFast { var_nums } => { @@ -2692,8 +2594,8 @@ impl ExecutingFrame<'_> { self.push_value(load_value); Ok(None) } - Instruction::StoreFastStoreFast { var_nums } => { - let oparg = var_nums.get(arg); + Instruction::StoreFastStoreFast { var_nums: packed } => { + let oparg = packed.get(arg); let idx1 = (oparg >> 4) as usize; let idx2 = (oparg & 15) as usize; let value1 = self.pop_value(); @@ -2703,14 +2605,14 @@ impl ExecutingFrame<'_> { fastlocals[idx2] = Some(value2); Ok(None) } - Instruction::StoreGlobal { namei } => { + Instruction::StoreGlobal { namei: idx } => { let value = self.pop_value(); self.globals - .set_item(self.code.names[namei.get(arg) as usize], value, vm)?; + .set_item(self.code.names[idx.get(arg) as usize], value, vm)?; Ok(None) } - Instruction::StoreName { namei } => { - let name = self.code.names[namei.get(arg) as usize]; + Instruction::StoreName { namei: idx } => { + let name = self.code.names[idx.get(arg) as usize]; let value = self.pop_value(); self.locals .mapping(vm) @@ -2734,18 +2636,7 @@ impl ExecutingFrame<'_> { Ok(None) } Instruction::StoreSubscr => { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_store_subscr(vm, instr_idx, cache_base); - } + self.adaptive(|s, ii, cb| s.specialize_store_subscr(vm, ii, cb)); self.execute_store_subscript(vm) } Instruction::Swap { i: index } => { @@ -2766,41 +2657,19 @@ impl ExecutingFrame<'_> { Ok(None) } Instruction::ToBool => { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_to_bool(vm, instr_idx, cache_base); - } + self.adaptive(|s, ii, cb| s.specialize_to_bool(vm, ii, cb)); let obj = self.pop_value(); let bool_val = obj.try_to_bool(vm)?; self.push_value(vm.ctx.new_bool(bool_val).into()); Ok(None) } - Instruction::UnpackEx { counts } => { - let args = counts.get(arg); + Instruction::UnpackEx { counts: args } => { + let args = args.get(arg); self.execute_unpack_ex(vm, args.before, args.after) } - Instruction::UnpackSequence { count } => { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_unpack_sequence(vm, instr_idx, cache_base); - } - self.unpack_sequence(count.get(arg), vm) + Instruction::UnpackSequence { count: size } => { + self.adaptive(|s, ii, cb| s.specialize_unpack_sequence(vm, ii, cb)); + self.unpack_sequence(size.get(arg), vm) } Instruction::WithExceptStart => { // Stack: [..., __exit__, lasti, prev_exc, exc] @@ -2847,18 +2716,7 @@ impl ExecutingFrame<'_> { } Instruction::Send { .. } => { // (receiver, v -- receiver, retval) - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_send(instr_idx, cache_base); - } + self.adaptive(|s, ii, cb| s.specialize_send(ii, cb)); let exit_label = bytecode::Label(self.lasti() + 1 + u32::from(arg)); let val = self.pop_value(); let receiver = self.top_value(); @@ -2907,19 +2765,18 @@ impl ExecutingFrame<'_> { } } } - // Deoptimize - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::Send { - delta: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); + { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + unsafe { + self.code.instructions.replace_op( + instr_idx, + Instruction::Send { delta: Arg::marker() }, + ); + self.code + .instructions + .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); + } } match self._send(receiver, val, vm)? { PyIterReturn::Return(value) => { @@ -3031,18 +2888,7 @@ impl ExecutingFrame<'_> { self.push_value(owner); Ok(None) } else { - // De-optimize - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::LoadAttr { - namei: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } + self.deoptimize_at(Instruction::LoadAttr { namei: Arg::marker() }, instr_idx, cache_base); self.load_attr_slow(vm, oparg) } } @@ -3066,9 +2912,7 @@ impl ExecutingFrame<'_> { unsafe { self.code.instructions.replace_op( instr_idx, - Instruction::LoadAttr { - namei: Arg::marker(), - }, + Instruction::LoadAttr { namei: Arg::marker() }, ); self.code .instructions @@ -3091,18 +2935,7 @@ impl ExecutingFrame<'_> { return Ok(None); } } - // De-optimize - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::LoadAttr { - namei: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } + self.deoptimize_at(Instruction::LoadAttr { namei: Arg::marker() }, instr_idx, cache_base); self.load_attr_slow(vm, oparg) } Instruction::LoadAttrInstanceValue => { @@ -3126,18 +2959,7 @@ impl ExecutingFrame<'_> { } // Not in instance dict — fall through to class lookup via slow path } - // De-optimize - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::LoadAttr { - namei: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } + self.deoptimize_at(Instruction::LoadAttr { namei: Arg::marker() }, instr_idx, cache_base); self.load_attr_slow(vm, oparg) } Instruction::LoadAttrModule => { @@ -3165,12 +2987,9 @@ impl ExecutingFrame<'_> { } // Deoptimize unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::LoadAttr { - namei: Arg::marker(), - }, - ); + self.code + .instructions + .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3199,12 +3018,9 @@ impl ExecutingFrame<'_> { return Ok(None); } unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::LoadAttr { - namei: Arg::marker(), - }, - ); + self.code + .instructions + .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3247,12 +3063,9 @@ impl ExecutingFrame<'_> { return Ok(None); } unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::LoadAttr { - namei: Arg::marker(), - }, - ); + self.code + .instructions + .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3283,12 +3096,9 @@ impl ExecutingFrame<'_> { return Ok(None); } unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::LoadAttr { - namei: Arg::marker(), - }, - ); + self.code + .instructions + .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3319,12 +3129,9 @@ impl ExecutingFrame<'_> { // Slot is None → AttributeError (fall through to slow path) } unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::LoadAttr { - namei: Arg::marker(), - }, - ); + self.code + .instructions + .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3354,12 +3161,9 @@ impl ExecutingFrame<'_> { } } unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::LoadAttr { - namei: Arg::marker(), - }, - ); + self.code + .instructions + .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3383,18 +3187,7 @@ impl ExecutingFrame<'_> { dict.set_item(attr_name, value, vm)?; return Ok(None); } - // Deoptimize - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::StoreAttr { - namei: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } + self.deoptimize_at(Instruction::StoreAttr { namei: Arg::marker() }, instr_idx, cache_base); self.store_attr(vm, attr_idx) } Instruction::StoreAttrSlot => { @@ -3417,12 +3210,9 @@ impl ExecutingFrame<'_> { // Deoptimize let attr_idx = u32::from(arg); unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::StoreAttr { - namei: Arg::marker(), - }, - ); + self.code + .instructions + .replace_op(instr_idx, Instruction::StoreAttr { namei: Arg::marker() }); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3444,10 +3234,10 @@ impl ExecutingFrame<'_> { return Ok(None); } drop(vec); - self.deoptimize_store_subscr(); + self.deoptimize(Instruction::StoreSubscr); return Err(vm.new_index_error("list assignment index out of range")); } - self.deoptimize_store_subscr(); + self.deoptimize(Instruction::StoreSubscr); obj.set_item(&*idx, value, vm)?; Ok(None) } @@ -3460,113 +3250,29 @@ impl ExecutingFrame<'_> { dict.set_item(&*idx, value, vm)?; Ok(None) } else { - self.deoptimize_store_subscr(); + self.deoptimize(Instruction::StoreSubscr); obj.set_item(&*idx, value, vm)?; Ok(None) } } // Specialized BINARY_OP opcodes Instruction::BinaryOpAddInt => { - let b = self.top_value(); - let a = self.nth_value(1); - if let (Some(a_int), Some(b_int)) = ( - a.downcast_ref_if_exact::(vm), - b.downcast_ref_if_exact::(vm), - ) { - let result = a_int.as_bigint() + b_int.as_bigint(); - self.pop_value(); - self.pop_value(); - self.push_value(vm.ctx.new_bigint(&result).into()); - Ok(None) - } else { - self.deoptimize_binary_op(bytecode::BinaryOperator::Add); - self.execute_bin_op(vm, bytecode::BinaryOperator::Add) - } + self.execute_binary_op_int(vm, |a, b| a + b, bytecode::BinaryOperator::Add) } Instruction::BinaryOpSubtractInt => { - let b = self.top_value(); - let a = self.nth_value(1); - if let (Some(a_int), Some(b_int)) = ( - a.downcast_ref_if_exact::(vm), - b.downcast_ref_if_exact::(vm), - ) { - let result = a_int.as_bigint() - b_int.as_bigint(); - self.pop_value(); - self.pop_value(); - self.push_value(vm.ctx.new_bigint(&result).into()); - Ok(None) - } else { - self.deoptimize_binary_op(bytecode::BinaryOperator::Subtract); - self.execute_bin_op(vm, bytecode::BinaryOperator::Subtract) - } + self.execute_binary_op_int(vm, |a, b| a - b, bytecode::BinaryOperator::Subtract) } Instruction::BinaryOpMultiplyInt => { - let b = self.top_value(); - let a = self.nth_value(1); - if let (Some(a_int), Some(b_int)) = ( - a.downcast_ref_if_exact::(vm), - b.downcast_ref_if_exact::(vm), - ) { - let result = a_int.as_bigint() * b_int.as_bigint(); - self.pop_value(); - self.pop_value(); - self.push_value(vm.ctx.new_bigint(&result).into()); - Ok(None) - } else { - self.deoptimize_binary_op(bytecode::BinaryOperator::Multiply); - self.execute_bin_op(vm, bytecode::BinaryOperator::Multiply) - } + self.execute_binary_op_int(vm, |a, b| a * b, bytecode::BinaryOperator::Multiply) } Instruction::BinaryOpAddFloat => { - let b = self.top_value(); - let a = self.nth_value(1); - if let (Some(a_f), Some(b_f)) = ( - a.downcast_ref_if_exact::(vm), - b.downcast_ref_if_exact::(vm), - ) { - let result = a_f.to_f64() + b_f.to_f64(); - self.pop_value(); - self.pop_value(); - self.push_value(vm.ctx.new_float(result).into()); - Ok(None) - } else { - self.deoptimize_binary_op(bytecode::BinaryOperator::Add); - self.execute_bin_op(vm, bytecode::BinaryOperator::Add) - } + self.execute_binary_op_float(vm, |a, b| a + b, bytecode::BinaryOperator::Add) } Instruction::BinaryOpSubtractFloat => { - let b = self.top_value(); - let a = self.nth_value(1); - if let (Some(a_f), Some(b_f)) = ( - a.downcast_ref_if_exact::(vm), - b.downcast_ref_if_exact::(vm), - ) { - let result = a_f.to_f64() - b_f.to_f64(); - self.pop_value(); - self.pop_value(); - self.push_value(vm.ctx.new_float(result).into()); - Ok(None) - } else { - self.deoptimize_binary_op(bytecode::BinaryOperator::Subtract); - self.execute_bin_op(vm, bytecode::BinaryOperator::Subtract) - } + self.execute_binary_op_float(vm, |a, b| a - b, bytecode::BinaryOperator::Subtract) } Instruction::BinaryOpMultiplyFloat => { - let b = self.top_value(); - let a = self.nth_value(1); - if let (Some(a_f), Some(b_f)) = ( - a.downcast_ref_if_exact::(vm), - b.downcast_ref_if_exact::(vm), - ) { - let result = a_f.to_f64() * b_f.to_f64(); - self.pop_value(); - self.pop_value(); - self.push_value(vm.ctx.new_float(result).into()); - Ok(None) - } else { - self.deoptimize_binary_op(bytecode::BinaryOperator::Multiply); - self.execute_bin_op(vm, bytecode::BinaryOperator::Multiply) - } + self.execute_binary_op_float(vm, |a, b| a * b, bytecode::BinaryOperator::Multiply) } Instruction::BinaryOpAddUnicode => { let b = self.top_value(); @@ -3581,7 +3287,7 @@ impl ExecutingFrame<'_> { self.push_value(result.to_pyobject(vm)); Ok(None) } else { - self.deoptimize_binary_op(bytecode::BinaryOperator::Add); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); self.execute_bin_op(vm, bytecode::BinaryOperator::Add) } } @@ -3603,10 +3309,10 @@ impl ExecutingFrame<'_> { return Ok(None); } drop(vec); - self.deoptimize_binary_op(bytecode::BinaryOperator::Subscr); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); return Err(vm.new_index_error("list index out of range")); } - self.deoptimize_binary_op(bytecode::BinaryOperator::Subscr); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); self.execute_bin_op(vm, bytecode::BinaryOperator::Subscr) } Instruction::BinaryOpSubscrTupleInt => { @@ -3625,10 +3331,10 @@ impl ExecutingFrame<'_> { self.push_value(value); return Ok(None); } - self.deoptimize_binary_op(bytecode::BinaryOperator::Subscr); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); return Err(vm.new_index_error("tuple index out of range")); } - self.deoptimize_binary_op(bytecode::BinaryOperator::Subscr); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); self.execute_bin_op(vm, bytecode::BinaryOperator::Subscr) } Instruction::BinaryOpSubscrDict => { @@ -3643,18 +3349,18 @@ impl ExecutingFrame<'_> { return Ok(None); } Ok(None) => { - self.deoptimize_binary_op(bytecode::BinaryOperator::Subscr); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); let key = self.pop_value(); self.pop_value(); return Err(vm.new_key_error(key)); } Err(e) => { - self.deoptimize_binary_op(bytecode::BinaryOperator::Subscr); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); return Err(e); } } } - self.deoptimize_binary_op(bytecode::BinaryOperator::Subscr); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); self.execute_bin_op(vm, bytecode::BinaryOperator::Subscr) } Instruction::BinaryOpSubscrStrInt => { @@ -3673,12 +3379,12 @@ impl ExecutingFrame<'_> { return Ok(None); } Err(e) => { - self.deoptimize_binary_op(bytecode::BinaryOperator::Subscr); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); return Err(e); } } } - self.deoptimize_binary_op(bytecode::BinaryOperator::Subscr); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); self.execute_bin_op(vm, bytecode::BinaryOperator::Subscr) } Instruction::BinaryOpSubscrListSlice => { @@ -3693,7 +3399,7 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize_binary_op(bytecode::BinaryOperator::Subscr); + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); self.execute_bin_op(vm, bytecode::BinaryOperator::Subscr) } Instruction::CallPyExactArgs => { @@ -3790,7 +3496,7 @@ impl ExecutingFrame<'_> { self.push_value_opt(_null); self.push_value(obj); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3817,7 +3523,7 @@ impl ExecutingFrame<'_> { self.push_value(obj); self.push_value(class_info); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3842,7 +3548,7 @@ impl ExecutingFrame<'_> { self.push_value_opt(_null); self.push_value(obj); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3865,7 +3571,7 @@ impl ExecutingFrame<'_> { self.push_value_opt(_null); self.push_value(obj); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3893,7 +3599,7 @@ impl ExecutingFrame<'_> { self.push_value_opt(_null); self.push_value(obj); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3922,7 +3628,7 @@ impl ExecutingFrame<'_> { self.push_value_opt(_null); self.push_value(obj); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3953,7 +3659,7 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3984,7 +3690,7 @@ impl ExecutingFrame<'_> { self.push_value(result); Ok(None) } else { - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4011,7 +3717,7 @@ impl ExecutingFrame<'_> { self.push_value(result); Ok(None) } else { - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4034,7 +3740,7 @@ impl ExecutingFrame<'_> { self.push_value_opt(self_or_null); self.push_value(item); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4069,7 +3775,7 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4105,7 +3811,7 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4142,7 +3848,7 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4168,7 +3874,7 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4227,7 +3933,7 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4265,7 +3971,7 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4297,7 +4003,7 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4312,7 +4018,7 @@ impl ExecutingFrame<'_> { let args = self.collect_positional_args(nargs); return self.execute_call(args, vm); } - self.deoptimize_call(); + self.deoptimize(Instruction::Call { argc: Arg::marker() }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4356,7 +4062,7 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize_call_kw(); + self.deoptimize(Instruction::CallKw { argc: Arg::marker() }); let args = self.collect_keyword_args(nargs); self.execute_call(args, vm) } @@ -4390,7 +4096,7 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize_call_kw(); + self.deoptimize(Instruction::CallKw { argc: Arg::marker() }); let args = self.collect_keyword_args(nargs); self.execute_call(args, vm) } @@ -4405,7 +4111,7 @@ impl ExecutingFrame<'_> { let args = self.collect_keyword_args(nargs); return self.execute_call(args, vm); } - self.deoptimize_call_kw(); + self.deoptimize(Instruction::CallKw { argc: Arg::marker() }); let args = self.collect_keyword_args(nargs); self.execute_call(args, vm) } @@ -4464,9 +4170,7 @@ impl ExecutingFrame<'_> { unsafe { self.code.instructions.replace_op( self.lasti() as usize - 1, - Instruction::LoadSuperAttr { - namei: Arg::marker(), - }, + Instruction::LoadSuperAttr { namei: Arg::marker() }, ); let cache_base = self.lasti() as usize; self.code @@ -4544,9 +4248,7 @@ impl ExecutingFrame<'_> { unsafe { self.code.instructions.replace_op( self.lasti() as usize - 1, - Instruction::LoadSuperAttr { - namei: Arg::marker(), - }, + Instruction::LoadSuperAttr { namei: Arg::marker() }, ); let cache_base = self.lasti() as usize; self.code @@ -4570,7 +4272,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(result).into()); Ok(None) } else { - self.deoptimize_compare_op(); + self.deoptimize(Instruction::CompareOp { opname: Arg::marker() }); let op = bytecode::ComparisonOperator::try_from(u32::from(arg)) .unwrap_or(bytecode::ComparisonOperator::Equal); self.execute_compare(vm, op) @@ -4595,7 +4297,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(result).into()); Ok(None) } else { - self.deoptimize_compare_op(); + self.deoptimize(Instruction::CompareOp { opname: Arg::marker() }); let op = bytecode::ComparisonOperator::try_from(u32::from(arg)) .unwrap_or(bytecode::ComparisonOperator::Equal); self.execute_compare(vm, op) @@ -4615,7 +4317,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(result).into()); Ok(None) } else { - self.deoptimize_compare_op(); + self.deoptimize(Instruction::CompareOp { opname: Arg::marker() }); let op = bytecode::ComparisonOperator::try_from(u32::from(arg)) .unwrap_or(bytecode::ComparisonOperator::Equal); self.execute_compare(vm, op) @@ -4627,7 +4329,7 @@ impl ExecutingFrame<'_> { // Already a bool, no-op Ok(None) } else { - self.deoptimize_to_bool(); + self.deoptimize(Instruction::ToBool); let obj = self.pop_value(); let result = obj.try_to_bool(vm)?; self.push_value(vm.ctx.new_bool(result).into()); @@ -4642,7 +4344,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(result).into()); Ok(None) } else { - self.deoptimize_to_bool(); + self.deoptimize(Instruction::ToBool); let obj = self.pop_value(); let result = obj.try_to_bool(vm)?; self.push_value(vm.ctx.new_bool(result).into()); @@ -4656,7 +4358,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(false).into()); Ok(None) } else { - self.deoptimize_to_bool(); + self.deoptimize(Instruction::ToBool); let obj = self.pop_value(); let result = obj.try_to_bool(vm)?; self.push_value(vm.ctx.new_bool(result).into()); @@ -4671,7 +4373,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(result).into()); Ok(None) } else { - self.deoptimize_to_bool(); + self.deoptimize(Instruction::ToBool); let obj = self.pop_value(); let result = obj.try_to_bool(vm)?; self.push_value(vm.ctx.new_bool(result).into()); @@ -4686,7 +4388,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(result).into()); Ok(None) } else { - self.deoptimize_to_bool(); + self.deoptimize(Instruction::ToBool); let obj = self.pop_value(); let result = obj.try_to_bool(vm)?; self.push_value(vm.ctx.new_bool(result).into()); @@ -4706,7 +4408,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(true).into()); Ok(None) } else { - self.deoptimize_to_bool(); + self.deoptimize(Instruction::ToBool); let obj = self.pop_value(); let result = obj.try_to_bool(vm)?; self.push_value(vm.ctx.new_bool(result).into()); @@ -4729,7 +4431,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(value).into()); Ok(None) } else { - self.deoptimize_contains_op(); + self.deoptimize(Instruction::ContainsOp { invert: Arg::marker() }); let b = self.pop_value(); let a = self.pop_value(); let invert = bytecode::Invert::try_from(u32::from(arg) as u8) @@ -4758,7 +4460,7 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(value).into()); Ok(None) } else { - self.deoptimize_contains_op(); + self.deoptimize(Instruction::ContainsOp { invert: Arg::marker() }); let b = self.pop_value(); let a = self.pop_value(); let invert = bytecode::Invert::try_from(u32::from(arg) as u8) @@ -4784,7 +4486,7 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize_unpack_sequence(); + self.deoptimize(Instruction::UnpackSequence { count: Arg::marker() }); let size = u32::from(arg); self.unpack_sequence(size, vm) } @@ -4802,7 +4504,7 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize_unpack_sequence(); + self.deoptimize(Instruction::UnpackSequence { count: Arg::marker() }); self.unpack_sequence(size as u32, vm) } Instruction::UnpackSequenceList => { @@ -4820,7 +4522,7 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize_unpack_sequence(); + self.deoptimize(Instruction::UnpackSequence { count: Arg::marker() }); self.unpack_sequence(size as u32, vm) } Instruction::ForIterRange => { @@ -4834,7 +4536,7 @@ impl ExecutingFrame<'_> { } Ok(None) } else { - self.deoptimize_for_iter(); + self.deoptimize(Instruction::ForIter { delta: Arg::marker() }); self.execute_for_iter(vm, target)?; Ok(None) } @@ -4850,7 +4552,7 @@ impl ExecutingFrame<'_> { } Ok(None) } else { - self.deoptimize_for_iter(); + self.deoptimize(Instruction::ForIter { delta: Arg::marker() }); self.execute_for_iter(vm, target)?; Ok(None) } @@ -4866,7 +4568,7 @@ impl ExecutingFrame<'_> { } Ok(None) } else { - self.deoptimize_for_iter(); + self.deoptimize(Instruction::ForIter { delta: Arg::marker() }); self.execute_for_iter(vm, target)?; Ok(None) } @@ -4886,7 +4588,7 @@ impl ExecutingFrame<'_> { } Ok(None) } else { - self.deoptimize_for_iter(); + self.deoptimize(Instruction::ForIter { delta: Arg::marker() }); self.execute_for_iter(vm, target)?; Ok(None) } @@ -4908,7 +4610,7 @@ impl ExecutingFrame<'_> { Ok(None) } else { // Name was removed from globals - self.deoptimize_load_global(); + self.deoptimize(Instruction::LoadGlobal { namei: Arg::marker() }); let x = self.load_global_or_builtin(name, vm)?; self.push_value(x); if (oparg & 1) != 0 { @@ -4917,7 +4619,7 @@ impl ExecutingFrame<'_> { Ok(None) } } else { - self.deoptimize_load_global(); + self.deoptimize(Instruction::LoadGlobal { namei: Arg::marker() }); let name = self.code.names[(oparg >> 1) as usize]; let x = self.load_global_or_builtin(name, vm)?; self.push_value(x); @@ -4946,7 +4648,7 @@ impl ExecutingFrame<'_> { return Ok(None); } // Fallback: name not found or builtins not a dict - self.deoptimize_load_global(); + self.deoptimize(Instruction::LoadGlobal { namei: Arg::marker() }); let x = self.load_global_or_builtin(name, vm)?; self.push_value(x); if (oparg & 1) != 0 { @@ -4954,7 +4656,7 @@ impl ExecutingFrame<'_> { } Ok(None) } else { - self.deoptimize_load_global(); + self.deoptimize(Instruction::LoadGlobal { namei: Arg::marker() }); let name = self.code.names[(oparg >> 1) as usize]; let x = self.load_global_or_builtin(name, vm)?; self.push_value(x); @@ -6544,20 +6246,7 @@ impl ExecutingFrame<'_> { } fn load_attr(&mut self, vm: &VirtualMachine, oparg: LoadAttr) -> FrameResult { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - - let counter = self.code.instructions.read_adaptive_counter(cache_base); - if counter > 0 { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, counter - 1); - } - } else { - self.specialize_load_attr(vm, oparg, instr_idx, cache_base); - } - + self.adaptive(|s, ii, cb| s.specialize_load_attr(vm, oparg, ii, cb)); self.load_attr_slow(vm, oparg) } @@ -6970,6 +6659,35 @@ impl ExecutingFrame<'_> { _ => None, }; + self.commit_specialization(instr_idx, cache_base, new_op); + } + + /// Adaptive counter: decrement the warmup counter, or call the specialize + /// function when it reaches zero. + #[inline] + fn adaptive(&mut self, specialize: impl FnOnce(&mut Self, usize, usize)) { + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; + let counter = self.code.instructions.read_adaptive_counter(cache_base); + if counter > 0 { + unsafe { + self.code + .instructions + .write_adaptive_counter(cache_base, counter - 1); + } + } else { + specialize(self, instr_idx, cache_base); + } + } + + /// Commit a specialization result: replace op on success, backoff on failure. + #[inline] + fn commit_specialization( + &mut self, + instr_idx: usize, + cache_base: usize, + new_op: Option, + ) { if let Some(new_op) = new_op { unsafe { self.code.instructions.replace_op(instr_idx, new_op); @@ -6983,19 +6701,79 @@ impl ExecutingFrame<'_> { } } - fn deoptimize_binary_op(&mut self, _op: bytecode::BinaryOperator) { + /// Deoptimize: replace specialized op with its base adaptive op and reset + /// the adaptive counter. Computes instr_idx/cache_base from lasti(). + #[inline] + fn deoptimize(&mut self, base_op: Instruction) { let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; + self.deoptimize_at(base_op, instr_idx, cache_base); + } + + /// Deoptimize with explicit indices (for specialized handlers that already + /// have instr_idx/cache_base in scope). + #[inline] + fn deoptimize_at(&mut self, base_op: Instruction, instr_idx: usize, cache_base: usize) { unsafe { - self.code - .instructions - .replace_op(instr_idx, Instruction::BinaryOp { op: Arg::marker() }); + self.code.instructions.replace_op(instr_idx, base_op); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); } } + /// Execute a specialized binary op on two int operands. + /// Deoptimizes if either operand is not an exact int. + #[inline] + fn execute_binary_op_int( + &mut self, + vm: &VirtualMachine, + op: impl FnOnce(&BigInt, &BigInt) -> BigInt, + deopt_op: bytecode::BinaryOperator, + ) -> FrameResult { + let b = self.top_value(); + let a = self.nth_value(1); + if let (Some(a_int), Some(b_int)) = ( + a.downcast_ref_if_exact::(vm), + b.downcast_ref_if_exact::(vm), + ) { + let result = op(a_int.as_bigint(), b_int.as_bigint()); + self.pop_value(); + self.pop_value(); + self.push_value(vm.ctx.new_bigint(&result).into()); + Ok(None) + } else { + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); + self.execute_bin_op(vm, deopt_op) + } + } + + /// Execute a specialized binary op on two float operands. + /// Deoptimizes if either operand is not an exact float. + #[inline] + fn execute_binary_op_float( + &mut self, + vm: &VirtualMachine, + op: impl FnOnce(f64, f64) -> f64, + deopt_op: bytecode::BinaryOperator, + ) -> FrameResult { + let b = self.top_value(); + let a = self.nth_value(1); + if let (Some(a_f), Some(b_f)) = ( + a.downcast_ref_if_exact::(vm), + b.downcast_ref_if_exact::(vm), + ) { + let result = op(a_f.to_f64(), b_f.to_f64()); + self.pop_value(); + self.pop_value(); + self.push_value(vm.ctx.new_float(result).into()); + Ok(None) + } else { + self.deoptimize(Instruction::BinaryOp { op: Arg::marker() }); + self.execute_bin_op(vm, deopt_op) + } + } + fn specialize_call( &mut self, vm: &VirtualMachine, @@ -7320,17 +7098,7 @@ impl ExecutingFrame<'_> { None }; - if let Some(new_op) = new_op { - unsafe { - self.code.instructions.replace_op(instr_idx, new_op); - } - } else { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } + self.commit_specialization(instr_idx, cache_base, new_op); } /// Recover the ComparisonOperator from the instruction arg byte. @@ -7341,23 +7109,8 @@ impl ExecutingFrame<'_> { .into() } - fn deoptimize_compare_op(&mut self) { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::CompareOp { - opname: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } - fn specialize_to_bool(&mut self, vm: &VirtualMachine, instr_idx: usize, _cache_base: usize) { + fn specialize_to_bool(&mut self, vm: &VirtualMachine, instr_idx: usize, cache_base: usize) { if !matches!( self.code.instructions.read_op(instr_idx), Instruction::ToBool @@ -7386,32 +7139,9 @@ impl ExecutingFrame<'_> { None }; - if let Some(new_op) = new_op { - unsafe { - self.code.instructions.replace_op(instr_idx, new_op); - } - } else { - let cache_base = instr_idx + 1; - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } + self.commit_specialization(instr_idx, cache_base, new_op); } - fn deoptimize_to_bool(&mut self) { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - unsafe { - self.code - .instructions - .replace_op(instr_idx, Instruction::ToBool); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } fn specialize_for_iter(&mut self, vm: &VirtualMachine, instr_idx: usize, cache_base: usize) { if !matches!( @@ -7434,66 +7164,9 @@ impl ExecutingFrame<'_> { None }; - if let Some(new_op) = new_op { - unsafe { - self.code.instructions.replace_op(instr_idx, new_op); - } - } else { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } - } - - fn deoptimize_call(&mut self) { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::Call { - argc: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } - - fn deoptimize_call_kw(&mut self) { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::CallKw { - argc: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } + self.commit_specialization(instr_idx, cache_base, new_op); } - fn deoptimize_for_iter(&mut self) { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::ForIter { - delta: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } /// Handle iterator exhaustion in specialized FOR_ITER handlers. /// Skips END_FOR if present at target and jumps. @@ -7523,7 +7196,7 @@ impl ExecutingFrame<'_> { ) { if !matches!( self.code.instructions.read_op(instr_idx), - Instruction::LoadGlobal(..) + Instruction::LoadGlobal { .. } ) { return; } @@ -7562,21 +7235,6 @@ impl ExecutingFrame<'_> { } } - fn deoptimize_load_global(&mut self) { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::LoadGlobal { - namei: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } fn specialize_store_subscr( &mut self, @@ -7604,36 +7262,14 @@ impl ExecutingFrame<'_> { None }; - if let Some(new_op) = new_op { - unsafe { - self.code.instructions.replace_op(instr_idx, new_op); - } - } else { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } + self.commit_specialization(instr_idx, cache_base, new_op); } - fn deoptimize_store_subscr(&mut self) { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - unsafe { - self.code - .instructions - .replace_op(instr_idx, Instruction::StoreSubscr); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } fn specialize_contains_op(&mut self, vm: &VirtualMachine, instr_idx: usize, cache_base: usize) { if !matches!( self.code.instructions.read_op(instr_idx), - Instruction::ContainsOp(..) + Instruction::ContainsOp { .. } ) { return; } @@ -7646,34 +7282,9 @@ impl ExecutingFrame<'_> { None }; - if let Some(new_op) = new_op { - unsafe { - self.code.instructions.replace_op(instr_idx, new_op); - } - } else { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } + self.commit_specialization(instr_idx, cache_base, new_op); } - fn deoptimize_contains_op(&mut self) { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::ContainsOp { - invert: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } fn specialize_unpack_sequence( &mut self, @@ -7700,34 +7311,9 @@ impl ExecutingFrame<'_> { None }; - if let Some(new_op) = new_op { - unsafe { - self.code.instructions.replace_op(instr_idx, new_op); - } - } else { - unsafe { - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } + self.commit_specialization(instr_idx, cache_base, new_op); } - fn deoptimize_unpack_sequence(&mut self) { - let instr_idx = self.lasti() as usize - 1; - let cache_base = instr_idx + 1; - unsafe { - self.code.instructions.replace_op( - instr_idx, - Instruction::UnpackSequence { - count: Arg::marker(), - }, - ); - self.code - .instructions - .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); - } - } fn specialize_store_attr( &mut self, From dd891517e545ddc27975deeca5204b878b7b1631 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" Date: Wed, 4 Mar 2026 04:31:38 +0900 Subject: [PATCH 11/12] Improve specialization guards and fix mark_stacks - CONTAINS_OP_SET: add frozenset support in handler and specialize - TO_BOOL_ALWAYS_TRUE: cache type version instead of checking slots - LOAD_GLOBAL_BUILTIN: cache builtins dict version alongside globals - mark_stacks: deoptimize specialized opcodes for correct reachability --- crates/vm/src/builtins/frame.rs | 4 +- crates/vm/src/frame.rs | 291 +++++++++++++++++++++----------- 2 files changed, 198 insertions(+), 97 deletions(-) diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs index 602810b4cdb..04abe4772d7 100644 --- a/crates/vm/src/builtins/frame.rs +++ b/crates/vm/src/builtins/frame.rs @@ -182,8 +182,8 @@ pub(crate) mod stack_analysis { } oparg = (oparg << 8) | u32::from(u8::from(instructions[i].arg)); - // De-instrument: get the underlying real instruction - let opcode = opcode.to_base().unwrap_or(opcode); + // De-instrument and de-specialize: get the underlying base instruction + let opcode = opcode.to_base().unwrap_or(opcode).deoptimize(); let caches = opcode.cache_entries(); let next_i = i + 1 + caches; diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index 55357a2bb42..b20fc6f3caa 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -6,8 +6,8 @@ use crate::{ TryFromObject, VirtualMachine, builtins::{ PyBaseException, PyBaseExceptionRef, PyBaseObject, PyCode, PyCoroutine, PyDict, PyDictRef, - PyFloat, PyGenerator, PyInt, PyInterpolation, PyList, PyModule, PyProperty, PySet, PySlice, - PyStr, PyStrInterned, PyTemplate, PyTraceback, PyType, PyUtf8Str, + PyFloat, PyFrozenSet, PyGenerator, PyInt, PyInterpolation, PyList, PyModule, PyProperty, + PySet, PySlice, PyStr, PyStrInterned, PyTemplate, PyTraceback, PyType, PyUtf8Str, asyncgenerator::PyAsyncGenWrappedValue, builtin_func::PyNativeFunction, descriptor::{MemberGetter, PyMemberDescriptor, PyMethodDescriptor}, @@ -2771,7 +2771,9 @@ impl ExecutingFrame<'_> { unsafe { self.code.instructions.replace_op( instr_idx, - Instruction::Send { delta: Arg::marker() }, + Instruction::Send { + delta: Arg::marker(), + }, ); self.code .instructions @@ -2888,7 +2890,11 @@ impl ExecutingFrame<'_> { self.push_value(owner); Ok(None) } else { - self.deoptimize_at(Instruction::LoadAttr { namei: Arg::marker() }, instr_idx, cache_base); + self.deoptimize_at( + Instruction::LoadAttr { namei: Arg::marker() }, + instr_idx, + cache_base, + ); self.load_attr_slow(vm, oparg) } } @@ -2935,7 +2941,11 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize_at(Instruction::LoadAttr { namei: Arg::marker() }, instr_idx, cache_base); + self.deoptimize_at( + Instruction::LoadAttr { namei: Arg::marker() }, + instr_idx, + cache_base, + ); self.load_attr_slow(vm, oparg) } Instruction::LoadAttrInstanceValue => { @@ -2959,7 +2969,11 @@ impl ExecutingFrame<'_> { } // Not in instance dict — fall through to class lookup via slow path } - self.deoptimize_at(Instruction::LoadAttr { namei: Arg::marker() }, instr_idx, cache_base); + self.deoptimize_at( + Instruction::LoadAttr { namei: Arg::marker() }, + instr_idx, + cache_base, + ); self.load_attr_slow(vm, oparg) } Instruction::LoadAttrModule => { @@ -3187,7 +3201,11 @@ impl ExecutingFrame<'_> { dict.set_item(attr_name, value, vm)?; return Ok(None); } - self.deoptimize_at(Instruction::StoreAttr { namei: Arg::marker() }, instr_idx, cache_base); + self.deoptimize_at( + Instruction::StoreAttr { namei: Arg::marker() }, + instr_idx, + cache_base, + ); self.store_attr(vm, attr_idx) } Instruction::StoreAttrSlot => { @@ -3496,7 +3514,9 @@ impl ExecutingFrame<'_> { self.push_value_opt(_null); self.push_value(obj); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3523,7 +3543,9 @@ impl ExecutingFrame<'_> { self.push_value(obj); self.push_value(class_info); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3548,7 +3570,9 @@ impl ExecutingFrame<'_> { self.push_value_opt(_null); self.push_value(obj); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3571,7 +3595,9 @@ impl ExecutingFrame<'_> { self.push_value_opt(_null); self.push_value(obj); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3599,7 +3625,9 @@ impl ExecutingFrame<'_> { self.push_value_opt(_null); self.push_value(obj); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3628,7 +3656,9 @@ impl ExecutingFrame<'_> { self.push_value_opt(_null); self.push_value(obj); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3659,7 +3689,9 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3690,7 +3722,9 @@ impl ExecutingFrame<'_> { self.push_value(result); Ok(None) } else { - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3717,7 +3751,9 @@ impl ExecutingFrame<'_> { self.push_value(result); Ok(None) } else { - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3740,7 +3776,9 @@ impl ExecutingFrame<'_> { self.push_value_opt(self_or_null); self.push_value(item); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3775,7 +3813,9 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3811,7 +3851,9 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3848,7 +3890,9 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3874,7 +3918,9 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3933,7 +3979,9 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -3971,7 +4019,9 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4003,7 +4053,9 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4018,7 +4070,9 @@ impl ExecutingFrame<'_> { let args = self.collect_positional_args(nargs); return self.execute_call(args, vm); } - self.deoptimize(Instruction::Call { argc: Arg::marker() }); + self.deoptimize(Instruction::Call { + argc: Arg::marker(), + }); let args = self.collect_positional_args(nargs); self.execute_call(args, vm) } @@ -4062,7 +4116,9 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize(Instruction::CallKw { argc: Arg::marker() }); + self.deoptimize(Instruction::CallKw { + argc: Arg::marker(), + }); let args = self.collect_keyword_args(nargs); self.execute_call(args, vm) } @@ -4096,7 +4152,9 @@ impl ExecutingFrame<'_> { self.push_value(result); return Ok(None); } - self.deoptimize(Instruction::CallKw { argc: Arg::marker() }); + self.deoptimize(Instruction::CallKw { + argc: Arg::marker(), + }); let args = self.collect_keyword_args(nargs); self.execute_call(args, vm) } @@ -4111,7 +4169,9 @@ impl ExecutingFrame<'_> { let args = self.collect_keyword_args(nargs); return self.execute_call(args, vm); } - self.deoptimize(Instruction::CallKw { argc: Arg::marker() }); + self.deoptimize(Instruction::CallKw { + argc: Arg::marker(), + }); let args = self.collect_keyword_args(nargs); self.execute_call(args, vm) } @@ -4397,12 +4457,12 @@ impl ExecutingFrame<'_> { } Instruction::ToBoolAlwaysTrue => { // Objects without __bool__ or __len__ are always True. - // Guard: check the type hasn't gained these slots. + // Guard: check type version hasn't changed. + let instr_idx = self.lasti() as usize - 1; + let cache_base = instr_idx + 1; let obj = self.top_value(); - let slots = &obj.class().slots; - if slots.as_number.boolean.load().is_none() - && slots.as_mapping.length.load().is_none() - && slots.as_sequence.length.load().is_none() + let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); + if cached_version != 0 && obj.class().tp_version_tag.load(Acquire) == cached_version { self.pop_value(); self.push_value(vm.ctx.new_bool(true).into()); @@ -4446,7 +4506,9 @@ impl ExecutingFrame<'_> { } Instruction::ContainsOpSet => { let b = self.top_value(); // haystack - if b.downcast_ref_if_exact::(vm).is_some() { + if b.downcast_ref_if_exact::(vm).is_some() + || b.downcast_ref_if_exact::(vm).is_some() + { let a = self.nth_value(1); // needle let found = vm._contains(b, a)?; self.pop_value(); @@ -4486,7 +4548,9 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize(Instruction::UnpackSequence { count: Arg::marker() }); + self.deoptimize(Instruction::UnpackSequence { + count: Arg::marker(), + }); let size = u32::from(arg); self.unpack_sequence(size, vm) } @@ -4504,7 +4568,9 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize(Instruction::UnpackSequence { count: Arg::marker() }); + self.deoptimize(Instruction::UnpackSequence { + count: Arg::marker(), + }); self.unpack_sequence(size as u32, vm) } Instruction::UnpackSequenceList => { @@ -4522,7 +4588,9 @@ impl ExecutingFrame<'_> { return Ok(None); } } - self.deoptimize(Instruction::UnpackSequence { count: Arg::marker() }); + self.deoptimize(Instruction::UnpackSequence { + count: Arg::marker(), + }); self.unpack_sequence(size as u32, vm) } Instruction::ForIterRange => { @@ -4536,7 +4604,9 @@ impl ExecutingFrame<'_> { } Ok(None) } else { - self.deoptimize(Instruction::ForIter { delta: Arg::marker() }); + self.deoptimize(Instruction::ForIter { + delta: Arg::marker(), + }); self.execute_for_iter(vm, target)?; Ok(None) } @@ -4552,7 +4622,9 @@ impl ExecutingFrame<'_> { } Ok(None) } else { - self.deoptimize(Instruction::ForIter { delta: Arg::marker() }); + self.deoptimize(Instruction::ForIter { + delta: Arg::marker(), + }); self.execute_for_iter(vm, target)?; Ok(None) } @@ -4568,7 +4640,9 @@ impl ExecutingFrame<'_> { } Ok(None) } else { - self.deoptimize(Instruction::ForIter { delta: Arg::marker() }); + self.deoptimize(Instruction::ForIter { + delta: Arg::marker(), + }); self.execute_for_iter(vm, target)?; Ok(None) } @@ -4588,7 +4662,9 @@ impl ExecutingFrame<'_> { } Ok(None) } else { - self.deoptimize(Instruction::ForIter { delta: Arg::marker() }); + self.deoptimize(Instruction::ForIter { + delta: Arg::marker(), + }); self.execute_for_iter(vm, target)?; Ok(None) } @@ -4633,38 +4709,39 @@ impl ExecutingFrame<'_> { let oparg = u32::from(arg); let instr_idx = self.lasti() as usize - 1; let cache_base = instr_idx + 1; - let cached_version = self.code.instructions.read_cache_u32(cache_base + 1); - let current_version = self.globals.version() as u32; - if cached_version == current_version { - // globals unchanged — name is NOT in globals, look up in builtins - let name = self.code.names[(oparg >> 1) as usize]; - if let Some(builtins_dict) = self.builtins.downcast_ref::() - && let Some(x) = builtins_dict.get_item_opt(name, vm)? - { - self.push_value(x); - if (oparg & 1) != 0 { - self.push_value_opt(None); + let cached_globals_ver = self.code.instructions.read_cache_u32(cache_base + 1); + let cached_builtins_ver = self.code.instructions.read_cache_u32(cache_base + 2); + let current_globals_ver = self.globals.version() as u32; + if cached_globals_ver == current_globals_ver { + // globals unchanged — name is NOT in globals, check builtins + if let Some(builtins_dict) = self.builtins.downcast_ref_if_exact::(vm) { + let current_builtins_ver = builtins_dict.version() as u32; + if cached_builtins_ver == current_builtins_ver { + // Both versions match — safe to look up in builtins + let name = self.code.names[(oparg >> 1) as usize]; + if let Some(x) = builtins_dict.get_item_opt(name, vm)? { + self.push_value(x); + if (oparg & 1) != 0 { + self.push_value_opt(None); + } + return Ok(None); + } } - return Ok(None); - } - // Fallback: name not found or builtins not a dict - self.deoptimize(Instruction::LoadGlobal { namei: Arg::marker() }); - let x = self.load_global_or_builtin(name, vm)?; - self.push_value(x); - if (oparg & 1) != 0 { - self.push_value_opt(None); - } - Ok(None) - } else { - self.deoptimize(Instruction::LoadGlobal { namei: Arg::marker() }); - let name = self.code.names[(oparg >> 1) as usize]; - let x = self.load_global_or_builtin(name, vm)?; - self.push_value(x); - if (oparg & 1) != 0 { - self.push_value_opt(None); } - Ok(None) } + // Version mismatch or lookup failed — deoptimize + self.deoptimize_at( + Instruction::LoadGlobal { namei: Arg::marker() }, + instr_idx, + cache_base, + ); + let name = self.code.names[(oparg >> 1) as usize]; + let x = self.load_global_or_builtin(name, vm)?; + self.push_value(x); + if (oparg & 1) != 0 { + self.push_value_opt(None); + } + Ok(None) } // All INSTRUMENTED_* opcodes delegate to a cold function to keep // the hot instruction loop free of monitoring overhead. @@ -6723,7 +6800,7 @@ impl ExecutingFrame<'_> { } /// Execute a specialized binary op on two int operands. - /// Deoptimizes if either operand is not an exact int. + /// Deoptimize if either operand is not an exact int. #[inline] fn execute_binary_op_int( &mut self, @@ -6749,7 +6826,7 @@ impl ExecutingFrame<'_> { } /// Execute a specialized binary op on two float operands. - /// Deoptimizes if either operand is not an exact float. + /// Deoptimize if either operand is not an exact float. #[inline] fn execute_binary_op_float( &mut self, @@ -7109,7 +7186,6 @@ impl ExecutingFrame<'_> { .into() } - fn specialize_to_bool(&mut self, vm: &VirtualMachine, instr_idx: usize, cache_base: usize) { if !matches!( self.code.instructions.read_op(instr_idx), @@ -7134,7 +7210,28 @@ impl ExecutingFrame<'_> { && cls.slots.as_mapping.length.load().is_none() && cls.slots.as_sequence.length.load().is_none() { - Some(Instruction::ToBoolAlwaysTrue) + // Cache type version for ToBoolAlwaysTrue guard + let mut type_version = cls.tp_version_tag.load(Acquire); + if type_version == 0 { + type_version = cls.assign_version_tag(); + } + if type_version != 0 { + unsafe { + self.code + .instructions + .write_cache_u32(cache_base + 1, type_version); + self.code + .instructions + .replace_op(instr_idx, Instruction::ToBoolAlwaysTrue); + } + } else { + unsafe { + self.code + .instructions + .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); + } + } + return; } else { None }; @@ -7142,7 +7239,6 @@ impl ExecutingFrame<'_> { self.commit_specialization(instr_idx, cache_base, new_op); } - fn specialize_for_iter(&mut self, vm: &VirtualMachine, instr_idx: usize, cache_base: usize) { if !matches!( self.code.instructions.read_op(instr_idx), @@ -7167,7 +7263,6 @@ impl ExecutingFrame<'_> { self.commit_specialization(instr_idx, cache_base, new_op); } - /// Handle iterator exhaustion in specialized FOR_ITER handlers. /// Skips END_FOR if present at target and jumps. fn for_iter_jump_on_exhausted(&mut self, target: bytecode::Label) { @@ -7206,25 +7301,33 @@ impl ExecutingFrame<'_> { let globals_version = self.globals.version() as u32; - let new_op = if in_globals { - Some(Instruction::LoadGlobalModule) - } else if self - .builtins - .downcast_ref::() - .and_then(|b| b.get_item_opt(name, vm).ok().flatten()) - .is_some() + if in_globals { + unsafe { + self.code + .instructions + .replace_op(instr_idx, Instruction::LoadGlobalModule); + self.code + .instructions + .write_cache_u32(cache_base + 1, globals_version); + } + } else if let Some(builtins_dict) = self.builtins.downcast_ref_if_exact::(vm) + && builtins_dict + .get_item_opt(name, vm) + .ok() + .flatten() + .is_some() { - Some(Instruction::LoadGlobalBuiltin) - } else { - None - }; - - if let Some(new_op) = new_op { + let builtins_version = builtins_dict.version() as u32; unsafe { - self.code.instructions.replace_op(instr_idx, new_op); + self.code + .instructions + .replace_op(instr_idx, Instruction::LoadGlobalBuiltin); self.code .instructions .write_cache_u32(cache_base + 1, globals_version); + self.code + .instructions + .write_cache_u32(cache_base + 2, builtins_version); } } else { unsafe { @@ -7235,7 +7338,6 @@ impl ExecutingFrame<'_> { } } - fn specialize_store_subscr( &mut self, vm: &VirtualMachine, @@ -7265,7 +7367,6 @@ impl ExecutingFrame<'_> { self.commit_specialization(instr_idx, cache_base, new_op); } - fn specialize_contains_op(&mut self, vm: &VirtualMachine, instr_idx: usize, cache_base: usize) { if !matches!( self.code.instructions.read_op(instr_idx), @@ -7276,7 +7377,9 @@ impl ExecutingFrame<'_> { let haystack = self.top_value(); // b = TOS = haystack let new_op = if haystack.downcast_ref_if_exact::(vm).is_some() { Some(Instruction::ContainsOpDict) - } else if haystack.downcast_ref_if_exact::(vm).is_some() { + } else if haystack.downcast_ref_if_exact::(vm).is_some() + || haystack.downcast_ref_if_exact::(vm).is_some() + { Some(Instruction::ContainsOpSet) } else { None @@ -7285,7 +7388,6 @@ impl ExecutingFrame<'_> { self.commit_specialization(instr_idx, cache_base, new_op); } - fn specialize_unpack_sequence( &mut self, vm: &VirtualMachine, @@ -7314,7 +7416,6 @@ impl ExecutingFrame<'_> { self.commit_specialization(instr_idx, cache_base, new_op); } - fn specialize_store_attr( &mut self, _vm: &VirtualMachine, From 21c29ca15bc1b8f43ff16a93cacc9ac43c81f31b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 4 Mar 2026 05:14:49 +0000 Subject: [PATCH 12/12] Auto-format: cargo fmt --all --- crates/vm/src/frame.rs | 123 +++++++++++++++++++++++++++++------------ 1 file changed, 87 insertions(+), 36 deletions(-) diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index b20fc6f3caa..28a04318379 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -2891,7 +2891,9 @@ impl ExecutingFrame<'_> { Ok(None) } else { self.deoptimize_at( - Instruction::LoadAttr { namei: Arg::marker() }, + Instruction::LoadAttr { + namei: Arg::marker(), + }, instr_idx, cache_base, ); @@ -2918,7 +2920,9 @@ impl ExecutingFrame<'_> { unsafe { self.code.instructions.replace_op( instr_idx, - Instruction::LoadAttr { namei: Arg::marker() }, + Instruction::LoadAttr { + namei: Arg::marker(), + }, ); self.code .instructions @@ -2942,7 +2946,9 @@ impl ExecutingFrame<'_> { } } self.deoptimize_at( - Instruction::LoadAttr { namei: Arg::marker() }, + Instruction::LoadAttr { + namei: Arg::marker(), + }, instr_idx, cache_base, ); @@ -2970,7 +2976,9 @@ impl ExecutingFrame<'_> { // Not in instance dict — fall through to class lookup via slow path } self.deoptimize_at( - Instruction::LoadAttr { namei: Arg::marker() }, + Instruction::LoadAttr { + namei: Arg::marker(), + }, instr_idx, cache_base, ); @@ -3001,9 +3009,12 @@ impl ExecutingFrame<'_> { } // Deoptimize unsafe { - self.code - .instructions - .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); + self.code.instructions.replace_op( + instr_idx, + Instruction::LoadAttr { + namei: Arg::marker(), + }, + ); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3032,9 +3043,12 @@ impl ExecutingFrame<'_> { return Ok(None); } unsafe { - self.code - .instructions - .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); + self.code.instructions.replace_op( + instr_idx, + Instruction::LoadAttr { + namei: Arg::marker(), + }, + ); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3077,9 +3091,12 @@ impl ExecutingFrame<'_> { return Ok(None); } unsafe { - self.code - .instructions - .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); + self.code.instructions.replace_op( + instr_idx, + Instruction::LoadAttr { + namei: Arg::marker(), + }, + ); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3110,9 +3127,12 @@ impl ExecutingFrame<'_> { return Ok(None); } unsafe { - self.code - .instructions - .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); + self.code.instructions.replace_op( + instr_idx, + Instruction::LoadAttr { + namei: Arg::marker(), + }, + ); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3143,9 +3163,12 @@ impl ExecutingFrame<'_> { // Slot is None → AttributeError (fall through to slow path) } unsafe { - self.code - .instructions - .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); + self.code.instructions.replace_op( + instr_idx, + Instruction::LoadAttr { + namei: Arg::marker(), + }, + ); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3175,9 +3198,12 @@ impl ExecutingFrame<'_> { } } unsafe { - self.code - .instructions - .replace_op(instr_idx, Instruction::LoadAttr { namei: Arg::marker() }); + self.code.instructions.replace_op( + instr_idx, + Instruction::LoadAttr { + namei: Arg::marker(), + }, + ); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -3202,7 +3228,9 @@ impl ExecutingFrame<'_> { return Ok(None); } self.deoptimize_at( - Instruction::StoreAttr { namei: Arg::marker() }, + Instruction::StoreAttr { + namei: Arg::marker(), + }, instr_idx, cache_base, ); @@ -3228,9 +3256,12 @@ impl ExecutingFrame<'_> { // Deoptimize let attr_idx = u32::from(arg); unsafe { - self.code - .instructions - .replace_op(instr_idx, Instruction::StoreAttr { namei: Arg::marker() }); + self.code.instructions.replace_op( + instr_idx, + Instruction::StoreAttr { + namei: Arg::marker(), + }, + ); self.code .instructions .write_adaptive_counter(cache_base, ADAPTIVE_BACKOFF_VALUE); @@ -4230,7 +4261,9 @@ impl ExecutingFrame<'_> { unsafe { self.code.instructions.replace_op( self.lasti() as usize - 1, - Instruction::LoadSuperAttr { namei: Arg::marker() }, + Instruction::LoadSuperAttr { + namei: Arg::marker(), + }, ); let cache_base = self.lasti() as usize; self.code @@ -4308,7 +4341,9 @@ impl ExecutingFrame<'_> { unsafe { self.code.instructions.replace_op( self.lasti() as usize - 1, - Instruction::LoadSuperAttr { namei: Arg::marker() }, + Instruction::LoadSuperAttr { + namei: Arg::marker(), + }, ); let cache_base = self.lasti() as usize; self.code @@ -4332,7 +4367,9 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(result).into()); Ok(None) } else { - self.deoptimize(Instruction::CompareOp { opname: Arg::marker() }); + self.deoptimize(Instruction::CompareOp { + opname: Arg::marker(), + }); let op = bytecode::ComparisonOperator::try_from(u32::from(arg)) .unwrap_or(bytecode::ComparisonOperator::Equal); self.execute_compare(vm, op) @@ -4357,7 +4394,9 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(result).into()); Ok(None) } else { - self.deoptimize(Instruction::CompareOp { opname: Arg::marker() }); + self.deoptimize(Instruction::CompareOp { + opname: Arg::marker(), + }); let op = bytecode::ComparisonOperator::try_from(u32::from(arg)) .unwrap_or(bytecode::ComparisonOperator::Equal); self.execute_compare(vm, op) @@ -4377,7 +4416,9 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(result).into()); Ok(None) } else { - self.deoptimize(Instruction::CompareOp { opname: Arg::marker() }); + self.deoptimize(Instruction::CompareOp { + opname: Arg::marker(), + }); let op = bytecode::ComparisonOperator::try_from(u32::from(arg)) .unwrap_or(bytecode::ComparisonOperator::Equal); self.execute_compare(vm, op) @@ -4491,7 +4532,9 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(value).into()); Ok(None) } else { - self.deoptimize(Instruction::ContainsOp { invert: Arg::marker() }); + self.deoptimize(Instruction::ContainsOp { + invert: Arg::marker(), + }); let b = self.pop_value(); let a = self.pop_value(); let invert = bytecode::Invert::try_from(u32::from(arg) as u8) @@ -4522,7 +4565,9 @@ impl ExecutingFrame<'_> { self.push_value(vm.ctx.new_bool(value).into()); Ok(None) } else { - self.deoptimize(Instruction::ContainsOp { invert: Arg::marker() }); + self.deoptimize(Instruction::ContainsOp { + invert: Arg::marker(), + }); let b = self.pop_value(); let a = self.pop_value(); let invert = bytecode::Invert::try_from(u32::from(arg) as u8) @@ -4686,7 +4731,9 @@ impl ExecutingFrame<'_> { Ok(None) } else { // Name was removed from globals - self.deoptimize(Instruction::LoadGlobal { namei: Arg::marker() }); + self.deoptimize(Instruction::LoadGlobal { + namei: Arg::marker(), + }); let x = self.load_global_or_builtin(name, vm)?; self.push_value(x); if (oparg & 1) != 0 { @@ -4695,7 +4742,9 @@ impl ExecutingFrame<'_> { Ok(None) } } else { - self.deoptimize(Instruction::LoadGlobal { namei: Arg::marker() }); + self.deoptimize(Instruction::LoadGlobal { + namei: Arg::marker(), + }); let name = self.code.names[(oparg >> 1) as usize]; let x = self.load_global_or_builtin(name, vm)?; self.push_value(x); @@ -4731,7 +4780,9 @@ impl ExecutingFrame<'_> { } // Version mismatch or lookup failed — deoptimize self.deoptimize_at( - Instruction::LoadGlobal { namei: Arg::marker() }, + Instruction::LoadGlobal { + namei: Arg::marker(), + }, instr_idx, cache_base, );