diff --git a/crates/vm/src/builtins/complex.rs b/crates/vm/src/builtins/complex.rs index 84fd0c806c4..f05e5a32faa 100644 --- a/crates/vm/src/builtins/complex.rs +++ b/crates/vm/src/builtins/complex.rs @@ -10,7 +10,9 @@ use crate::{ stdlib::warnings, types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; +use core::cell::Cell; use core::num::Wrapping; +use core::ptr::NonNull; use num_complex::Complex64; use num_traits::Zero; use rustpython_common::hash; @@ -24,11 +26,49 @@ pub struct PyComplex { value: Complex64, } +// spell-checker:ignore MAXFREELIST +thread_local! { + static COMPLEX_FREELIST: Cell> = const { Cell::new(crate::object::FreeList::new()) }; +} + impl PyPayload for PyComplex { + const MAX_FREELIST: usize = 100; + const HAS_FREELIST: bool = true; + #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.complex_type } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + COMPLEX_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop() -> Option> { + COMPLEX_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } } impl ToPyObject for Complex64 { diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 7ba173fe7e4..f2a7e6a5a29 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -77,27 +77,32 @@ impl PyPayload for PyDict { #[inline] unsafe fn freelist_push(obj: *mut PyObject) -> bool { - DICT_FREELIST.with(|fl| { - let mut list = fl.take(); - let stored = if list.len() < Self::MAX_FREELIST { - list.push(obj); - true - } else { - false - }; - fl.set(list); - stored - }) + DICT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) } #[inline] unsafe fn freelist_pop() -> Option> { - DICT_FREELIST.with(|fl| { - let mut list = fl.take(); - let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); - fl.set(list); - result - }) + DICT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() } } diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index f78b227fc38..e9267a9bf00 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -49,27 +49,32 @@ impl PyPayload for PyFloat { #[inline] unsafe fn freelist_push(obj: *mut PyObject) -> bool { - FLOAT_FREELIST.with(|fl| { - let mut list = fl.take(); - let stored = if list.len() < Self::MAX_FREELIST { - list.push(obj); - true - } else { - false - }; - fl.set(list); - stored - }) + FLOAT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) } #[inline] unsafe fn freelist_pop() -> Option> { - FLOAT_FREELIST.with(|fl| { - let mut list = fl.take(); - let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); - fl.set(list); - result - }) + FLOAT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() } } diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index d2d462b8f30..01863615ac1 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -20,7 +20,9 @@ use crate::{ types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable}, }; use alloc::fmt; +use core::cell::Cell; use core::ops::{Neg, Not}; +use core::ptr::NonNull; use malachite_bigint::{BigInt, Sign}; use num_integer::Integer; use num_traits::{One, Pow, PrimInt, Signed, ToPrimitive, Zero}; @@ -48,7 +50,15 @@ where } } +// spell-checker:ignore MAXFREELIST +thread_local! { + static INT_FREELIST: Cell> = const { Cell::new(crate::object::FreeList::new()) }; +} + impl PyPayload for PyInt { + const MAX_FREELIST: usize = 100; + const HAS_FREELIST: bool = true; + #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.int_type @@ -57,6 +67,36 @@ impl PyPayload for PyInt { fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { vm.ctx.new_int(self.value).into() } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + INT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop() -> Option> { + INT_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } } macro_rules! impl_into_pyobject_int { diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index ff3ed64f263..c13dea57169 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -89,27 +89,32 @@ impl PyPayload for PyList { #[inline] unsafe fn freelist_push(obj: *mut PyObject) -> bool { - LIST_FREELIST.with(|fl| { - let mut list = fl.take(); - let stored = if list.len() < Self::MAX_FREELIST { - list.push(obj); - true - } else { - false - }; - fl.set(list); - stored - }) + LIST_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) } #[inline] unsafe fn freelist_pop() -> Option> { - LIST_FREELIST.with(|fl| { - let mut list = fl.take(); - let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); - fl.set(list); - result - }) + LIST_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() } } diff --git a/crates/vm/src/builtins/range.rs b/crates/vm/src/builtins/range.rs index ec1a662ddad..795ec230ba9 100644 --- a/crates/vm/src/builtins/range.rs +++ b/crates/vm/src/builtins/range.rs @@ -15,7 +15,9 @@ use crate::{ Representable, SelfIter, }, }; +use core::cell::Cell; use core::cmp::max; +use core::ptr::NonNull; use crossbeam_utils::atomic::AtomicCell; use malachite_bigint::{BigInt, Sign}; use num_integer::Integer; @@ -67,11 +69,49 @@ pub struct PyRange { pub step: PyIntRef, } +// spell-checker:ignore MAXFREELIST +thread_local! { + static RANGE_FREELIST: Cell> = const { Cell::new(crate::object::FreeList::new()) }; +} + impl PyPayload for PyRange { + const MAX_FREELIST: usize = 6; + const HAS_FREELIST: bool = true; + #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.range_type } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + RANGE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop() -> Option> { + RANGE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() + } } impl PyRange { diff --git a/crates/vm/src/builtins/slice.rs b/crates/vm/src/builtins/slice.rs index c0b68ed9e8a..aeb3337c7d8 100644 --- a/crates/vm/src/builtins/slice.rs +++ b/crates/vm/src/builtins/slice.rs @@ -60,27 +60,32 @@ impl PyPayload for PySlice { #[inline] unsafe fn freelist_push(obj: *mut PyObject) -> bool { - SLICE_FREELIST.with(|fl| { - let mut list = fl.take(); - let stored = if list.len() < Self::MAX_FREELIST { - list.push(obj); - true - } else { - false - }; - fl.set(list); - stored - }) + SLICE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let stored = if list.len() < Self::MAX_FREELIST { + list.push(obj); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) } #[inline] unsafe fn freelist_pop() -> Option> { - SLICE_FREELIST.with(|fl| { - let mut list = fl.take(); - let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); - fl.set(list); - result - }) + SLICE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) }); + fl.set(list); + result + }) + .ok() + .flatten() } } diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index fce1eaf7e35..77330b2665e 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -858,11 +858,11 @@ impl PyInner { } } -/// Thread-local freelist storage that properly deallocates cached objects -/// on thread teardown. +/// Thread-local freelist storage for reusing object allocations. /// -/// Wraps a `Vec<*mut PyObject>` and implements `Drop` to convert each -/// raw pointer back into `Box>` for proper deallocation. +/// Wraps a `Vec<*mut PyObject>`. On thread teardown, `Drop` frees raw +/// `PyInner` allocations without running payload destructors to avoid +/// accessing already-destroyed thread-local storage (GC state, other freelists). pub(crate) struct FreeList { items: Vec<*mut PyObject>, _marker: core::marker::PhantomData, @@ -885,8 +885,16 @@ impl Default for FreeList { impl Drop for FreeList { fn drop(&mut self) { + // During thread teardown, we cannot safely run destructors on cached + // objects because their Drop impls may access thread-local storage + // (GC state, other freelists) that is already destroyed. + // Instead, free just the raw allocation. The payload's heap fields + // (BigInt, PyObjectRef, etc.) are leaked, but this is bounded by + // MAX_FREELIST per type per thread. for ptr in self.items.drain(..) { - drop(unsafe { Box::from_raw(ptr as *mut PyInner) }); + unsafe { + alloc::alloc::dealloc(ptr as *mut u8, core::alloc::Layout::new::>()); + } } } }