From 8ea2e8a4fd82902ea73e00b907ed4562822af2cf Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 18 Apr 2026 15:50:25 +0200 Subject: [PATCH 01/10] Implement support for DLR get/set --- src/runtime/InteropConfiguration.cs | 1 + .../Mixins/DynamicObjectMixinsProvider.cs | 47 ++++++++ src/runtime/Mixins/dlr.py | 16 +++ src/runtime/PythonEngine.cs | 2 +- src/runtime/Runtime.cs | 2 + src/runtime/Types/ClassBase.cs | 110 ++++++++++++++++++ src/runtime/Types/ClassDerived.cs | 2 +- .../Types/DynamicObjectMemberAccessor.cs | 84 +++++++++++++ src/runtime/Util/ConcurrentLruCache.cs | 103 ++++++++++++++++ src/testing/dlrtest.cs | 31 +++++ tests/test_dynamic.py | 67 +++++++++++ 11 files changed, 463 insertions(+), 2 deletions(-) create mode 100644 src/runtime/Mixins/DynamicObjectMixinsProvider.cs create mode 100644 src/runtime/Mixins/dlr.py create mode 100644 src/runtime/Types/DynamicObjectMemberAccessor.cs create mode 100644 src/runtime/Util/ConcurrentLruCache.cs create mode 100644 src/testing/dlrtest.cs create mode 100644 tests/test_dynamic.py diff --git a/src/runtime/InteropConfiguration.cs b/src/runtime/InteropConfiguration.cs index 781d0d01f..0cd441ebc 100644 --- a/src/runtime/InteropConfiguration.cs +++ b/src/runtime/InteropConfiguration.cs @@ -22,6 +22,7 @@ public static InteropConfiguration MakeDefault() { DefaultBaseTypeProvider.Instance, new CollectionMixinsProvider(new Lazy(() => Py.Import("clr._extras.collections"))), + new DynamicObjectMixinsProvider(new Lazy(() => Py.Import("clr._extras.dlr"))), }, }; } diff --git a/src/runtime/Mixins/DynamicObjectMixinsProvider.cs b/src/runtime/Mixins/DynamicObjectMixinsProvider.cs new file mode 100644 index 000000000..6afa31f7e --- /dev/null +++ b/src/runtime/Mixins/DynamicObjectMixinsProvider.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Dynamic; + +namespace Python.Runtime.Mixins; + +class DynamicObjectMixinsProvider : IPythonBaseTypeProvider, IDisposable +{ + readonly Lazy mixinsModule; + + public DynamicObjectMixinsProvider(Lazy mixinsModule) => + this.mixinsModule = mixinsModule ?? throw new ArgumentNullException(nameof(mixinsModule)); + + public PyObject Mixins => mixinsModule.Value; + + public IEnumerable GetBaseTypes(Type type, IList existingBases) + { + if (type is null) + throw new ArgumentNullException(nameof(type)); + + if (existingBases is null) + throw new ArgumentNullException(nameof(existingBases)); + + if (!typeof(IDynamicMetaObjectProvider).IsAssignableFrom(type)) + return existingBases; + + var newBases = new List(existingBases) + { + new(Mixins.GetAttr("DynamicMetaObjectProviderMixin")) + }; + + if (type.IsInterface && type.BaseType is null) + { + newBases.RemoveAll(@base => PythonReferenceComparer.Instance.Equals(@base, Runtime.PyBaseObjectType)); + } + + return newBases; + } + + public void Dispose() + { + if (this.mixinsModule.IsValueCreated) + { + this.mixinsModule.Value.Dispose(); + } + } +} diff --git a/src/runtime/Mixins/dlr.py b/src/runtime/Mixins/dlr.py new file mode 100644 index 000000000..745e78a69 --- /dev/null +++ b/src/runtime/Mixins/dlr.py @@ -0,0 +1,16 @@ +""" +Implements helpers for Dynamic Language Runtime (DLR) types. +""" + +class DynamicMetaObjectProviderMixin: + def __dir__(self): + names = set(super().__dir__()) + + get_names = getattr(self, "GetDynamicMemberNames", None) + if callable(get_names): + try: + names.update(get_names()) + except Exception: + pass + + return list(sorted(names)) diff --git a/src/runtime/PythonEngine.cs b/src/runtime/PythonEngine.cs index fd04d4a3e..264835fff 100644 --- a/src/runtime/PythonEngine.cs +++ b/src/runtime/PythonEngine.cs @@ -299,7 +299,7 @@ static void LoadSubmodule(BorrowedReference targetModuleDict, string fullName, s static void LoadMixins(BorrowedReference targetModuleDict) { - foreach (string nested in new[] { "collections" }) + foreach (string nested in new[] { "collections", "dlr" }) { LoadSubmodule(targetModuleDict, fullName: "clr._extras." + nested, diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 399608733..865b070e2 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -151,6 +151,7 @@ internal static void Initialize(bool initSigs = false) GenericUtil.Reset(); ClassManager.Reset(); + ClassBase.Reset(); ClassDerivedObject.Reset(); TypeManager.Initialize(); CLRObject.creationBlocked = false; @@ -280,6 +281,7 @@ internal static void Shutdown() NullGCHandles(ExtensionType.loadedExtensions); ClassManager.RemoveClasses(); + ClassBase.Reset(); TypeManager.RemoveTypes(); _typesInitialized = false; diff --git a/src/runtime/Types/ClassBase.cs b/src/runtime/Types/ClassBase.cs index 3fcb7ca4f..4b5b60d07 100644 --- a/src/runtime/Types/ClassBase.cs +++ b/src/runtime/Types/ClassBase.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Collections.Generic; using System.Diagnostics; +using System.Dynamic; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; @@ -22,6 +23,10 @@ namespace Python.Runtime [Serializable] internal class ClassBase : ManagedType, IDeserializationCallback { + static readonly DynamicObjectMemberAccessor dynamicMemberAccessor = new(); + + internal static void Reset() => dynamicMemberAccessor.Clear(); + [NonSerialized] internal List dotNetMembers = new(); internal Indexer? indexer; @@ -603,6 +608,105 @@ static IEnumerable GetCallImplementations(Type type) => type.GetMethods(BindingFlags.Public | BindingFlags.Instance) .Where(m => m.Name == "__call__"); + static NewReference tp_getattro_dlr(BorrowedReference ob, BorrowedReference key) + { + var attr = Runtime.PyObject_GenericGetAttr(ob, key); + if (!attr.IsNull()) + { + return attr; + } + + // Only run the DLR binder if the error was AttributeError, otherwise preserve the original error + if (Runtime.PyErr_ExceptionMatches(Exceptions.AttributeError) == 0) + { + return default; + } + + if (!Runtime.PyString_Check(key)) + { + return default; + } + + if (GetManagedObject(ob) is not CLRObject co) + { + return default; + } + + // Slot registration already guarantees this type supports DLR + var dynamicObject = (IDynamicMetaObjectProvider)co.inst; + + string? memberName = Runtime.GetManagedString(key); + if (memberName is null) + { + return default; + } + + if (!dynamicMemberAccessor.TryGetMember(dynamicObject, memberName, out object? value)) + { + return default; + } + + // Clear the lingering AttributeError + Runtime.PyErr_Clear(); + + using var pyValue = value.ToPython(); + return pyValue.NewReferenceOrNull(); + } + + static int tp_setattro_dlr(BorrowedReference ob, BorrowedReference key, BorrowedReference val) + { + int result = Runtime.PyObject_GenericSetAttr(ob, key, val); + if (result == 0) + { + return 0; + } + + // Preserve non-attribute errors exactly as they are. + if (Runtime.PyErr_ExceptionMatches(Exceptions.AttributeError) == 0) + { + return -1; + } + + // Deletion fallback is intentionally not handled by DLR binder yet. + if (val == null) + { + return -1; + } + + if (!Runtime.PyString_Check(key)) + { + return -1; + } + + if (GetManagedObject(ob) is not CLRObject co) + { + return -1; + } + + // Slot registration already guarantees this type supports DLR. + var dynamicObject = (IDynamicMetaObjectProvider)co.inst; + + string? memberName = Runtime.GetManagedString(key); + if (memberName is null) + { + return -1; + } + + if (!Converter.ToManaged(val, typeof(object), out object? managedValue, true)) + { + return -1; + } + + if (!dynamicMemberAccessor.TrySetMember(dynamicObject, memberName, managedValue)) + { + return -1; + } + + // Clear the lingering AttributeError + Runtime.PyErr_Clear(); + return 0; + } + public virtual void InitializeSlots(BorrowedReference pyType, SlotsHolder slotsHolder) { if (!this.type.Valid) return; @@ -612,6 +716,12 @@ public virtual void InitializeSlots(BorrowedReference pyType, SlotsHolder slotsH TypeManager.InitializeSlotIfEmpty(pyType, TypeOffset.tp_call, new Interop.BBB_N(tp_call_impl), slotsHolder); } + if (typeof(IDynamicMetaObjectProvider).IsAssignableFrom(this.type.Value)) + { + TypeManager.InitializeSlotIfEmpty(pyType, TypeOffset.tp_getattro, new Interop.BB_N(tp_getattro_dlr), slotsHolder); + TypeManager.InitializeSlotIfEmpty(pyType, TypeOffset.tp_setattro, new Interop.BBB_I32(tp_setattro_dlr), slotsHolder); + } + if (indexer is not null) { if (indexer.CanGet) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 592eefd55..8b6739421 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -40,7 +40,7 @@ static ClassDerivedObject() moduleBuilders = new Dictionary, ModuleBuilder>(); } - public static void Reset() + public static new void Reset() { assemblyBuilders = new Dictionary(); moduleBuilders = new Dictionary, ModuleBuilder>(); diff --git a/src/runtime/Types/DynamicObjectMemberAccessor.cs b/src/runtime/Types/DynamicObjectMemberAccessor.cs new file mode 100644 index 000000000..e06b64c32 --- /dev/null +++ b/src/runtime/Types/DynamicObjectMemberAccessor.cs @@ -0,0 +1,84 @@ +using System; +using System.Dynamic; +using System.Runtime.CompilerServices; +using Microsoft.CSharp.RuntimeBinder; + +namespace Python.Runtime; + +class DynamicObjectMemberAccessor +{ + const int MaxCacheEntries = 1000; + + readonly ConcurrentLruCache> getters = new(MaxCacheEntries); + readonly ConcurrentLruCache> setters = new(MaxCacheEntries); + + static readonly CSharpArgumentInfo[] getArgumentInfo = + { + CSharpArgumentInfo.Create(CSharpArgumentInfoFlags.None, null), + }; + + static readonly CSharpArgumentInfo[] setArgumentInfo = + { + CSharpArgumentInfo.Create(CSharpArgumentInfoFlags.None, null), + CSharpArgumentInfo.Create(CSharpArgumentInfoFlags.None, null), + }; + + public bool TryGetMember(IDynamicMetaObjectProvider obj, string memberName, out object? value) + { + if (obj is null) + throw new ArgumentNullException(nameof(obj)); + if (memberName is null) + throw new ArgumentNullException(nameof(memberName)); + + var getter = getters.GetOrAdd(new MemberKey(obj.GetType(), memberName), static key => + { + var binder = Binder.GetMember(CSharpBinderFlags.None, key.MemberName, key.Type, getArgumentInfo); + var callSite = CallSite>.Create(binder); + return obj => callSite.Target(callSite, obj); + }); + + try + { + value = getter(obj); + return true; + } + catch (RuntimeBinderException) + { + value = null; + return false; + } + } + + public bool TrySetMember(IDynamicMetaObjectProvider obj, string memberName, object? value) + { + if (obj is null) + throw new ArgumentNullException(nameof(obj)); + if (memberName is null) + throw new ArgumentNullException(nameof(memberName)); + + var setter = setters.GetOrAdd(new MemberKey(obj.GetType(), memberName), static key => + { + var binder = Binder.SetMember(CSharpBinderFlags.None, key.MemberName, key.Type, setArgumentInfo); + var callSite = CallSite>.Create(binder); + return (obj, value) => callSite.Target(callSite, obj, value); + }); + + try + { + setter(obj, value); + return true; + } + catch (RuntimeBinderException) + { + return false; + } + } + + readonly record struct MemberKey(Type Type, string MemberName); + + public void Clear() + { + getters.Clear(); + setters.Clear(); + } +} diff --git a/src/runtime/Util/ConcurrentLruCache.cs b/src/runtime/Util/ConcurrentLruCache.cs new file mode 100644 index 000000000..42acc15d1 --- /dev/null +++ b/src/runtime/Util/ConcurrentLruCache.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; + +namespace Python.Runtime; + +internal sealed class ConcurrentLruCache where TKey : notnull +{ + readonly ConcurrentDictionary> map = new(); + readonly LinkedList lru = new(); + readonly object gate = new(); + + sealed record CacheItem(TKey Key, TValue Value); + + public ConcurrentLruCache(int capacity) + { + if (capacity <= 0) + throw new ArgumentOutOfRangeException(nameof(capacity), "Capacity must be greater than zero."); + + Capacity = capacity; + } + + public int Capacity { get; private set; } + + public int Count => map.Count; + + public TValue GetOrAdd(TKey key, Func valueFactory) + { + if (valueFactory is null) + throw new ArgumentNullException(nameof(valueFactory)); + + if (TryGetValue(key, out var existing)) + return existing; + + var created = valueFactory(key); + + lock (gate) + { + if (map.TryGetValue(key, out var alreadyAdded)) + { + MoveToFront(alreadyAdded); + return alreadyAdded.Value.Value; + } + + var item = new CacheItem(key, created); + var node = new LinkedListNode(item); + lru.AddFirst(node); + map[key] = node; + EvictOverflow(); + return created; + } + } + + public bool TryGetValue(TKey key, out TValue value) + { + if (map.TryGetValue(key, out var node)) + { + lock (gate) + { + if (map.TryGetValue(key, out node)) + { + MoveToFront(node); + value = node.Value.Value; + return true; + } + } + } + + value = default!; + return false; + } + + public void Clear() + { + lock (gate) + { + lru.Clear(); + map.Clear(); + } + } + + void MoveToFront(LinkedListNode node) + { + if (ReferenceEquals(lru.First, node)) + return; + + lru.Remove(node); + lru.AddFirst(node); + } + + void EvictOverflow() + { + while (map.Count > Capacity) + { + var last = lru.Last; + if (last is null) + return; + + lru.RemoveLast(); + map.TryRemove(last.Value.Key, out _); + } + } +} diff --git a/src/testing/dlrtest.cs b/src/testing/dlrtest.cs new file mode 100644 index 000000000..28814789c --- /dev/null +++ b/src/testing/dlrtest.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using System.Dynamic; + +namespace Python.Test; + +public class DynamicMappingObject : DynamicObject +{ + readonly Dictionary storage = []; + + // Native members for testing that regular CLR access is unaffected. + public string Label = "default"; + public int Multiplier { get; set; } = 1; + public int Multiply(int value) => value * Multiplier; + + // Test helper: bypass normal member binding and write directly to dynamic storage. + public void SetDynamicValue(string name, object value) => storage[name] = value; + + public override bool TryGetMember(GetMemberBinder binder, out object result) + => storage.TryGetValue(binder.Name, out result); + + public override bool TrySetMember(SetMemberBinder binder, object value) + { + storage[binder.Name] = value; + return true; + } + + public override bool TryDeleteMember(DeleteMemberBinder binder) + => storage.Remove(binder.Name); + + public override IEnumerable GetDynamicMemberNames() => storage.Keys; +} diff --git a/tests/test_dynamic.py b/tests/test_dynamic.py new file mode 100644 index 000000000..2ec26da3b --- /dev/null +++ b/tests/test_dynamic.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +import pytest +from System.Collections.Generic import Dictionary +from System.Dynamic import ExpandoObject + +from Python.Test import DynamicMappingObject + + +def _mro_names(obj): + return [f"{t.__module__}.{t.__name__}" for t in type(obj).__mro__] + + +@pytest.mark.parametrize( + "obj, expected", + [ + (DynamicMappingObject(), True), + (ExpandoObject(), True), + (Dictionary[str, int](), False), + ], +) +def test_dlr_mixin_presence(obj, expected): + has_mixin = "clr._extras.dlr.DynamicMetaObjectProviderMixin" in _mro_names(obj) + assert has_mixin is expected + + +@pytest.mark.parametrize("obj", [DynamicMappingObject(), ExpandoObject()]) +def test_dynamic_binder(obj): + assert "answer" not in dir(obj) + assert "wrong_answer" not in dir(obj) + + setattr(obj, "answer", 42) + obj.wrong_answer = 54 + + assert obj.answer == 42 + assert obj.wrong_answer == 54 + + assert "answer" in dir(obj) + assert "wrong_answer" in dir(obj) + + +def test_native_members_are_accessible_and_keep_priority(): + obj = DynamicMappingObject() + setattr(obj, "answer", 42) + obj.SetDynamicValue("Multiplier", 999) + + # Native field + assert obj.Label == "default" + obj.Label = "changed" + assert obj.Label == "changed" + + # Native property takes precedence over dynamic fallback + assert obj.Multiplier == 1 + obj.Multiplier = 7 + assert obj.Multiplier == 7 + + # Native method + obj.Multiplier = 3 + assert obj.Multiply(5) == 15 + +def test_dynamic_and_native_members_coexist(): + obj = DynamicMappingObject() + setattr(obj, "answer", 42) + obj.Multiplier = 2 + assert obj.answer == 42 + assert obj.Multiplier == 2 + assert obj.Multiply(10) == 20 From e60bfbc9d3b3d5aaa1f65f6f6aae6d896cd60ea3 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 3 May 2026 11:38:01 +0200 Subject: [PATCH 02/10] Extend the tests --- src/testing/dlrtest.cs | 26 +++++++++++++---- tests/test_dynamic.py | 63 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/src/testing/dlrtest.cs b/src/testing/dlrtest.cs index 28814789c..a685adb25 100644 --- a/src/testing/dlrtest.cs +++ b/src/testing/dlrtest.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Dynamic; @@ -5,7 +6,9 @@ namespace Python.Test; public class DynamicMappingObject : DynamicObject { - readonly Dictionary storage = []; + Dictionary storage; + + Dictionary Storage => storage ??= []; // Native members for testing that regular CLR access is unaffected. public string Label = "default"; @@ -13,19 +16,30 @@ public class DynamicMappingObject : DynamicObject public int Multiply(int value) => value * Multiplier; // Test helper: bypass normal member binding and write directly to dynamic storage. - public void SetDynamicValue(string name, object value) => storage[name] = value; + public void SetDynamicValue(string name, object value) => Storage[name] = value; + + // Test helper: retrieve the actual value stored in C# (for verification that None was stored as null) + public object GetDynamicValue(string name) => Storage.TryGetValue(name, out var value) ? value : null; + + public override bool TryGetMember(GetMemberBinder binder, out object result) - => storage.TryGetValue(binder.Name, out result); + => Storage.TryGetValue(binder.Name, out result); + + public object TestProp + { + get => "TEST_PROP"; + set => throw new InvalidOperationException("Can't write to TestProp"); + } public override bool TrySetMember(SetMemberBinder binder, object value) { - storage[binder.Name] = value; + Storage[binder.Name] = value; return true; } public override bool TryDeleteMember(DeleteMemberBinder binder) - => storage.Remove(binder.Name); + => binder is not null && Storage.Remove(binder.Name); - public override IEnumerable GetDynamicMemberNames() => storage.Keys; + public override IEnumerable GetDynamicMemberNames() => Storage.Keys; } diff --git a/tests/test_dynamic.py b/tests/test_dynamic.py index 2ec26da3b..e6fafccfe 100644 --- a/tests/test_dynamic.py +++ b/tests/test_dynamic.py @@ -65,3 +65,66 @@ def test_dynamic_and_native_members_coexist(): assert obj.answer == 42 assert obj.Multiplier == 2 assert obj.Multiply(10) == 20 + + +@pytest.mark.parametrize("obj", [DynamicMappingObject(), ExpandoObject()]) +def test_set_and_get_dynamic_property(obj): + """Test that setting and getting dynamic properties goes through DLR binder.""" + # Get initial value (should be None for non-existent property) + assert not hasattr(obj, "MyProp") + + # Set a dynamic property to a value + obj.MyProp = 42 + assert obj.MyProp == 42 + + # Set to None and verify it stays None through DLR + obj.MyProp = None + assert obj.MyProp is None + + # Set to another value and verify + obj.MyProp = "hello" + assert obj.MyProp == "hello" + + +def test_update_dynamic_value(): + """Setting from Python must update the backing dynamic store in C#.""" + obj = DynamicMappingObject() + obj.SetDynamicValue("TestProp", "initial") + assert obj.TestProp == "initial" + + obj.TestProp = None + + assert obj.TestProp is None + assert obj.GetDynamicValue("TestProp") is None + + +def test_derive_from_dynamic_class(): + class MyMappingObject(DynamicMappingObject): + __namespace__ = "PythonNetTest" + + def __init__(self): + self._custom = 0 + + @property + def custom_property(self): + return self._custom + + @custom_property.setter + def custom_property(self, i): + self._custom += i + + + obj = MyMappingObject() + with pytest.raises(AttributeError): + x = obj.unknown_property + + assert obj.custom_property == 0 + + obj.custom_property = 5 + assert obj.custom_property == 5 + + obj.custom_property = 5 + assert obj.custom_property == 10 + + obj.other_property = None + assert obj.other_property is None \ No newline at end of file From 194341c1aabe077279da24d48081bb9225c0e8e3 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 4 May 2026 12:52:33 +0200 Subject: [PATCH 03/10] Fix ClassDerived issues, add support for deletions, add more tests --- src/runtime/Mixins/dlr.py | 8 +- src/runtime/TypeManager.cs | 199 ++++++++++++++++++ src/runtime/Types/ClassDerived.cs | 37 ++++ .../Types/DynamicObjectMemberAccessor.cs | 127 ++++++++++- src/testing/dlrtest.cs | 6 - tests/test_dynamic.py | 55 ++++- 6 files changed, 413 insertions(+), 19 deletions(-) diff --git a/src/runtime/Mixins/dlr.py b/src/runtime/Mixins/dlr.py index 745e78a69..cd44035e4 100644 --- a/src/runtime/Mixins/dlr.py +++ b/src/runtime/Mixins/dlr.py @@ -6,10 +6,12 @@ class DynamicMetaObjectProviderMixin: def __dir__(self): names = set(super().__dir__()) - get_names = getattr(self, "GetDynamicMemberNames", None) - if callable(get_names): + get_dynamic_member_names = getattr(self, "GetDynamicMemberNames", None) + if callable(get_dynamic_member_names): try: - names.update(get_names()) + for name in get_dynamic_member_names(): + if isinstance(name, str): + names.add(name) except Exception: pass diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index dbff1fbd4..7776ab9ab 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Dynamic; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; @@ -37,6 +38,182 @@ internal class TypeManager "tp_clear", }; + static readonly DynamicObjectMemberAccessor dynamicMemberAccessor = new(); + + static bool HasClrMember(object instance, string memberName) => + instance.GetType().GetMember(memberName, BindingFlags.Public | BindingFlags.Instance).Length > 0; + + static bool IsPythonSpecialAttributeName(string memberName) => + memberName.Length > 4 && memberName.StartsWith("__") && memberName.EndsWith("__"); + + static bool TryGetDynamicInstance(BorrowedReference ob, out object instance, out IDynamicMetaObjectProvider dynamicObject) + { + if (ManagedType.GetManagedObject(ob) is CLRObject co && co.inst is IDynamicMetaObjectProvider coDynamic) + { + instance = co.inst; + dynamicObject = coDynamic; + return true; + } + + if (Converter.ToManaged(ob, typeof(IDynamicMetaObjectProvider), out object? managedDynamic, false) + && managedDynamic is IDynamicMetaObjectProvider convertedDynamic) + { + instance = managedDynamic; + dynamicObject = convertedDynamic; + return true; + } + + if (Converter.ToManaged(ob, typeof(object), out object? managedInstance, false) + && managedInstance is IDynamicMetaObjectProvider boxedDynamic) + { + instance = managedInstance; + dynamicObject = boxedDynamic; + return true; + } + + instance = null!; + dynamicObject = null!; + return false; + } + + static bool TryGetManagedDynamicInstance(BorrowedReference ob, out object instance, out IDynamicMetaObjectProvider dynamicObject) + { + if (ManagedType.GetManagedObject(ob) is CLRObject co && co.inst is IDynamicMetaObjectProvider coDynamic) + { + instance = co.inst; + dynamicObject = coDynamic; + return true; + } + + instance = null!; + dynamicObject = null!; + return false; + } + + public static NewReference tp_getattro_dlr_proxy(BorrowedReference ob, BorrowedReference key) + { + bool hasStringKey = Runtime.PyString_Check(key); + string? memberName = hasStringKey ? Runtime.GetManagedString(key) : null; + + if (memberName is not null && TryGetManagedDynamicInstance(ob, out object preInstance, out var preDynamicObject)) + { + if (memberName == nameof(DynamicObjectMemberAccessor.GetDynamicMemberNames) + && !HasClrMember(preInstance, memberName)) + { + using var pyMemberNames = new Func>( + () => dynamicMemberAccessor.GetDynamicMemberNames(preDynamicObject)).ToPython(); + return pyMemberNames.NewReferenceOrNull(); + } + } + + var attr = Runtime.PyObject_GenericGetAttr(ob, key); + if (!attr.IsNull()) + { + return attr; + } + + if (memberName == null) + { + return default; + } + + if (Runtime.PyErr_ExceptionMatches(Exceptions.AttributeError) == 0) + { + return default; + } + + if (!TryGetManagedDynamicInstance(ob, out object instance, out var dynamicObject)) + { + return default; + } + + if (memberName is null) + { + return default; + } + + if (HasClrMember(instance, memberName)) + { + return default; + } + + if (IsPythonSpecialAttributeName(memberName)) + { + return default; + } + + bool resolved; + object? value; + try + { + resolved = dynamicMemberAccessor.TryGetMember(dynamicObject, memberName, out value); + } + catch + { + return default; + } + + if (!resolved) + { + return default; + } + + Runtime.PyErr_Clear(); + + using var pyValue = value.ToPython(); + return pyValue.NewReferenceOrNull(); + } + + public static int tp_setattro_dlr_proxy(BorrowedReference ob, BorrowedReference key, BorrowedReference val) + { + string? memberName = Runtime.PyString_Check(key) ? Runtime.GetManagedString(key) : null; + bool hasDynamicInstance = TryGetDynamicInstance(ob, out object instance, out var dynamicObject); + bool canUseDynamic = hasDynamicInstance + && memberName is not null + && !HasClrMember(instance, memberName); + + if (canUseDynamic) + { + if (val == null) + { + if (dynamicMemberAccessor.TryDeleteMember(dynamicObject, memberName!)) + { + Runtime.PyErr_Clear(); + return 0; + } + } + else + { + object? managedValue = null; + if (val != Runtime.PyNone && !Converter.ToManaged(val, typeof(object), out managedValue, true)) + { + return -1; + } + + if (dynamicMemberAccessor.TrySetMember(dynamicObject, memberName!, managedValue)) + { + Runtime.PyErr_Clear(); + return 0; + } + } + } + + int result = Runtime.PyObject_GenericSetAttr(ob, key, val); + if (result == 0 && canUseDynamic) + { + object? managedValue = null; + if (val != null && val != Runtime.PyNone && !Converter.ToManaged(val, typeof(object), out managedValue, true)) + { + return 0; + } + + dynamicMemberAccessor.TrySetMember(dynamicObject, memberName!, managedValue); + Runtime.PyErr_Clear(); + } + + return result; + } + internal static void Initialize() { Debug.Assert(cache.Count == 0, "Cache should be empty", @@ -303,6 +480,12 @@ internal static void InitializeClass(PyType type, ClassBase impl, Type clrType) impl.InitializeSlots(type, slotsHolder); + if (typeof(IDynamicMetaObjectProvider).IsAssignableFrom(clrType) + && !typeof(IPythonDerivedType).IsAssignableFrom(clrType)) + { + InitializeSlot(type, TypeOffset.tp_getattro, new Interop.BB_N(tp_getattro_dlr_proxy), slotsHolder); + } + OperatorMethod.FixupSlots(type, clrType); // Leverage followup initialization from the Python runtime. Note // that the type of the new type must PyType_Type at the time we @@ -313,6 +496,22 @@ internal static void InitializeClass(PyType type, ClassBase impl, Type clrType) throw PythonException.ThrowLastAsClrException(); } + if (typeof(IDynamicMetaObjectProvider).IsAssignableFrom(clrType) + && !typeof(IPythonDerivedType).IsAssignableFrom(clrType)) + { + MethodInfo? setMethod = typeof(TypeManager).GetMethod( + nameof(tp_setattro_dlr_proxy), + BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public); + + if (setMethod is null) + { + throw new MissingMethodException("DLR attribute slot handlers were not found"); + } + + InitializeSlot(type, TypeOffset.tp_setattro, setMethod, slotsHolder); + Runtime.PyType_Modified(type.Reference); + } + var dict = Util.ReadRef(type, TypeOffset.tp_dict); string mn = clrType.Namespace ?? ""; using (var mod = Runtime.PyString_FromString(mn)) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 8b6739421..3129e0a89 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Dynamic; using System.Diagnostics; using System.Linq; using System.Reflection; @@ -232,6 +233,13 @@ internal static Type CreateDerivedType(string name, continue; } + // Avoid re-entrant DLR binder recursion when Python derives from + // DynamicObject-based types (including overrides in intermediate bases). + if (IsDynamicObjectHookMethod(method)) + { + continue; + } + // skip if this property has already been overridden if ((method.Name.StartsWith("get_") || method.Name.StartsWith("set_")) && pyProperties.Contains(method.Name.Substring(4))) @@ -300,6 +308,35 @@ internal static Type CreateDerivedType(string name, return type; } + static bool IsDynamicObjectHookMethod(MethodInfo method) + { + MethodInfo origin = method.GetBaseDefinition(); + Type? originType = origin.DeclaringType; + if (originType == typeof(DynamicObject)) + { + return origin.Name switch + { + nameof(DynamicObject.TryGetMember) + or nameof(DynamicObject.TrySetMember) + or nameof(DynamicObject.TryDeleteMember) + or nameof(DynamicObject.TryInvokeMember) + or nameof(DynamicObject.TryConvert) + or nameof(DynamicObject.TryGetIndex) + or nameof(DynamicObject.TrySetIndex) + or nameof(DynamicObject.GetDynamicMemberNames) + or nameof(IDynamicMetaObjectProvider.GetMetaObject) => true, + _ => false, + }; + } + + if (originType == typeof(IDynamicMetaObjectProvider)) + { + return origin.Name == nameof(IDynamicMetaObjectProvider.GetMetaObject); + } + + return false; + } + /// /// Add a constructor override that calls the python ctor after calling the base type constructor. /// diff --git a/src/runtime/Types/DynamicObjectMemberAccessor.cs b/src/runtime/Types/DynamicObjectMemberAccessor.cs index e06b64c32..9e1018a15 100644 --- a/src/runtime/Types/DynamicObjectMemberAccessor.cs +++ b/src/runtime/Types/DynamicObjectMemberAccessor.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; using System.Dynamic; +using System.Linq; +using System.Linq.Expressions; using System.Runtime.CompilerServices; using Microsoft.CSharp.RuntimeBinder; @@ -11,6 +14,7 @@ class DynamicObjectMemberAccessor readonly ConcurrentLruCache> getters = new(MaxCacheEntries); readonly ConcurrentLruCache> setters = new(MaxCacheEntries); + readonly ConcurrentLruCache> deleters = new(MaxCacheEntries); static readonly CSharpArgumentInfo[] getArgumentInfo = { @@ -32,9 +36,23 @@ public bool TryGetMember(IDynamicMetaObjectProvider obj, string memberName, out var getter = getters.GetOrAdd(new MemberKey(obj.GetType(), memberName), static key => { + if (typeof(DynamicObject).IsAssignableFrom(key.Type)) + { + var getBinder = new GetMemberNameBinder(key.MemberName); + return obj => + { + if (((DynamicObject)obj).TryGetMember(getBinder, out object? result)) + { + return result; + } + + throw new RuntimeBinderException($"Could not get member '{key.MemberName}'"); + }; + } + var binder = Binder.GetMember(CSharpBinderFlags.None, key.MemberName, key.Type, getArgumentInfo); - var callSite = CallSite>.Create(binder); - return obj => callSite.Target(callSite, obj); + var callSite = CallSite>.Create(binder); + return obj => callSite.Target(callSite, (IDynamicMetaObjectProvider)obj); }); try @@ -58,9 +76,21 @@ public bool TrySetMember(IDynamicMetaObjectProvider obj, string memberName, obje var setter = setters.GetOrAdd(new MemberKey(obj.GetType(), memberName), static key => { + if (typeof(DynamicObject).IsAssignableFrom(key.Type)) + { + var setBinder = new SetMemberNameBinder(key.MemberName); + return (obj, value) => + { + if (!((DynamicObject)obj).TrySetMember(setBinder, value)) + { + throw new RuntimeBinderException($"Could not set member '{key.MemberName}'"); + } + }; + } + var binder = Binder.SetMember(CSharpBinderFlags.None, key.MemberName, key.Type, setArgumentInfo); - var callSite = CallSite>.Create(binder); - return (obj, value) => callSite.Target(callSite, obj, value); + var callSite = CallSite>.Create(binder); + return (obj, value) => callSite.Target(callSite, (IDynamicMetaObjectProvider)obj, value); }); try @@ -74,11 +104,100 @@ public bool TrySetMember(IDynamicMetaObjectProvider obj, string memberName, obje } } + public bool TryDeleteMember(IDynamicMetaObjectProvider obj, string memberName) + { + if (obj is null) + throw new ArgumentNullException(nameof(obj)); + if (memberName is null) + throw new ArgumentNullException(nameof(memberName)); + + var deleter = deleters.GetOrAdd(new MemberKey(obj.GetType(), memberName), static key => + { + if (typeof(DynamicObject).IsAssignableFrom(key.Type)) + { + var binder = new DeleteMemberNameBinder(key.MemberName); + return obj => ((DynamicObject)obj).TryDeleteMember(binder); + } + + if (typeof(ExpandoObject).IsAssignableFrom(key.Type)) + { + return obj => ((IDictionary)(ExpandoObject)obj).Remove(key.MemberName); + } + + return _ => false; + }); + + try + { + return deleter(obj); + } + catch (RuntimeBinderException) + { + return false; + } + } + + public IReadOnlyCollection GetDynamicMemberNames(IDynamicMetaObjectProvider obj) + { + if (obj is null) + throw new ArgumentNullException(nameof(obj)); + + if (obj is ExpandoObject expandoObject) + { + return ((IDictionary)expandoObject).Keys.ToArray(); + } + + if (obj is DynamicObject dynamicObject) + { + return dynamicObject.GetDynamicMemberNames().ToArray(); + } + + var metaObject = obj.GetMetaObject(Expression.Constant(obj)); + return metaObject.GetDynamicMemberNames().ToArray(); + } + readonly record struct MemberKey(Type Type, string MemberName); + sealed class DeleteMemberNameBinder : DeleteMemberBinder + { + public DeleteMemberNameBinder(string name) + : base(name, ignoreCase: false) + { + } + + public override DynamicMetaObject FallbackDeleteMember(DynamicMetaObject target, DynamicMetaObject? errorSuggestion) + => errorSuggestion ?? throw new RuntimeBinderException($"Could not delete member '{Name}'"); + } + + sealed class GetMemberNameBinder : GetMemberBinder + { + public GetMemberNameBinder(string name) + : base(name, ignoreCase: false) + { + } + + public override DynamicMetaObject FallbackGetMember(DynamicMetaObject target, DynamicMetaObject? errorSuggestion) + => errorSuggestion ?? throw new RuntimeBinderException($"Could not get member '{Name}'"); + } + + sealed class SetMemberNameBinder : SetMemberBinder + { + public SetMemberNameBinder(string name) + : base(name, ignoreCase: false) + { + } + + public override DynamicMetaObject FallbackSetMember( + DynamicMetaObject target, + DynamicMetaObject value, + DynamicMetaObject? errorSuggestion) + => errorSuggestion ?? throw new RuntimeBinderException($"Could not set member '{Name}'"); + } + public void Clear() { getters.Clear(); setters.Clear(); + deleters.Clear(); } } diff --git a/src/testing/dlrtest.cs b/src/testing/dlrtest.cs index a685adb25..5805e8871 100644 --- a/src/testing/dlrtest.cs +++ b/src/testing/dlrtest.cs @@ -26,12 +26,6 @@ public class DynamicMappingObject : DynamicObject public override bool TryGetMember(GetMemberBinder binder, out object result) => Storage.TryGetValue(binder.Name, out result); - public object TestProp - { - get => "TEST_PROP"; - set => throw new InvalidOperationException("Can't write to TestProp"); - } - public override bool TrySetMember(SetMemberBinder binder, object value) { Storage[binder.Name] = value; diff --git a/tests/test_dynamic.py b/tests/test_dynamic.py index e6fafccfe..b18ac1fdd 100644 --- a/tests/test_dynamic.py +++ b/tests/test_dynamic.py @@ -87,15 +87,58 @@ def test_set_and_get_dynamic_property(obj): def test_update_dynamic_value(): - """Setting from Python must update the backing dynamic store in C#.""" + """Dynamic-only members should use DLR get/set/modify/delete end-to-end.""" obj = DynamicMappingObject() - obj.SetDynamicValue("TestProp", "initial") - assert obj.TestProp == "initial" + assert not hasattr(obj, "DynamicOnly") - obj.TestProp = None + # Initial set should create a dynamic member + obj.DynamicOnly = "initial" + assert obj.DynamicOnly == "initial" - assert obj.TestProp is None - assert obj.GetDynamicValue("TestProp") is None + # Modify flows through TrySetMember + obj.DynamicOnly = "updated" + assert obj.DynamicOnly == "updated" + + # Setting None keeps a present member with None value + obj.DynamicOnly = None + assert obj.DynamicOnly is None + + # Delete flows through TryDeleteMember + del obj.DynamicOnly + assert "DynamicOnly" not in dir(obj) + assert not hasattr(obj, "DynamicOnly") + + +def test_dynamic_set_none_updates_managed_store_after_get(): + """Regression: get->set(None)->get must route through DLR and update managed storage.""" + obj = DynamicMappingObject() + obj.SetDynamicValue("MyProp", "initial") + + x = obj.MyProp + assert x == "initial" + + obj.MyProp = None + + y = obj.MyProp + assert y is None + assert obj.GetDynamicValue("MyProp") is None + + +@pytest.mark.parametrize("obj", [DynamicMappingObject(), ExpandoObject()]) +def test_dynamic_member_lifecycle(obj): + """Dynamic members should support set/modify/get/delete via the DLR binder.""" + name = "LifecycleMember" + + assert not hasattr(obj, name) + + setattr(obj, name, 1) + assert getattr(obj, name) == 1 + + setattr(obj, name, 2) + assert getattr(obj, name) == 2 + + delattr(obj, name) + assert not hasattr(obj, name) def test_derive_from_dynamic_class(): From c33012f3c460d700a200595f22b5f270724b29ba Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Tue, 5 May 2026 13:13:32 +0200 Subject: [PATCH 04/10] Remove superfluous code from ClassBase, moved to TypeManager --- src/runtime/Runtime.cs | 2 - src/runtime/TypeManager.cs | 24 ++------ src/runtime/Types/ClassBase.cs | 109 --------------------------------- 3 files changed, 6 insertions(+), 129 deletions(-) diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 865b070e2..399608733 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -151,7 +151,6 @@ internal static void Initialize(bool initSigs = false) GenericUtil.Reset(); ClassManager.Reset(); - ClassBase.Reset(); ClassDerivedObject.Reset(); TypeManager.Initialize(); CLRObject.creationBlocked = false; @@ -281,7 +280,6 @@ internal static void Shutdown() NullGCHandles(ExtensionType.loadedExtensions); ClassManager.RemoveClasses(); - ClassBase.Reset(); TypeManager.RemoveTypes(); _typesInitialized = false; diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 7776ab9ab..e18970ce2 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -218,6 +218,7 @@ internal static void Initialize() { Debug.Assert(cache.Count == 0, "Cache should be empty", "Some errors may occurred on last shutdown"); + dynamicMemberAccessor.Clear(); using (var plainType = SlotHelper.CreateObjectType()) { subtype_traverse = Util.ReadIntPtr(plainType.Borrow(), TypeOffset.tp_traverse); @@ -241,6 +242,8 @@ internal static void RemoveTypes() } } + dynamicMemberAccessor.Clear(); + foreach (var type in cache.Values) { type.Dispose(); @@ -480,12 +483,6 @@ internal static void InitializeClass(PyType type, ClassBase impl, Type clrType) impl.InitializeSlots(type, slotsHolder); - if (typeof(IDynamicMetaObjectProvider).IsAssignableFrom(clrType) - && !typeof(IPythonDerivedType).IsAssignableFrom(clrType)) - { - InitializeSlot(type, TypeOffset.tp_getattro, new Interop.BB_N(tp_getattro_dlr_proxy), slotsHolder); - } - OperatorMethod.FixupSlots(type, clrType); // Leverage followup initialization from the Python runtime. Note // that the type of the new type must PyType_Type at the time we @@ -496,19 +493,10 @@ internal static void InitializeClass(PyType type, ClassBase impl, Type clrType) throw PythonException.ThrowLastAsClrException(); } - if (typeof(IDynamicMetaObjectProvider).IsAssignableFrom(clrType) - && !typeof(IPythonDerivedType).IsAssignableFrom(clrType)) + if (typeof(IDynamicMetaObjectProvider).IsAssignableFrom(clrType)) { - MethodInfo? setMethod = typeof(TypeManager).GetMethod( - nameof(tp_setattro_dlr_proxy), - BindingFlags.Static | BindingFlags.NonPublic | BindingFlags.Public); - - if (setMethod is null) - { - throw new MissingMethodException("DLR attribute slot handlers were not found"); - } - - InitializeSlot(type, TypeOffset.tp_setattro, setMethod, slotsHolder); + InitializeSlot(type, TypeOffset.tp_getattro, new Interop.BB_N(tp_getattro_dlr_proxy), slotsHolder); + InitializeSlot(type, TypeOffset.tp_setattro, new Interop.BBB_I32(tp_setattro_dlr_proxy), slotsHolder); Runtime.PyType_Modified(type.Reference); } diff --git a/src/runtime/Types/ClassBase.cs b/src/runtime/Types/ClassBase.cs index 4b5b60d07..3b6744a62 100644 --- a/src/runtime/Types/ClassBase.cs +++ b/src/runtime/Types/ClassBase.cs @@ -23,10 +23,6 @@ namespace Python.Runtime [Serializable] internal class ClassBase : ManagedType, IDeserializationCallback { - static readonly DynamicObjectMemberAccessor dynamicMemberAccessor = new(); - - internal static void Reset() => dynamicMemberAccessor.Clear(); - [NonSerialized] internal List dotNetMembers = new(); internal Indexer? indexer; @@ -608,105 +604,6 @@ static IEnumerable GetCallImplementations(Type type) => type.GetMethods(BindingFlags.Public | BindingFlags.Instance) .Where(m => m.Name == "__call__"); - static NewReference tp_getattro_dlr(BorrowedReference ob, BorrowedReference key) - { - var attr = Runtime.PyObject_GenericGetAttr(ob, key); - if (!attr.IsNull()) - { - return attr; - } - - // Only run the DLR binder if the error was AttributeError, otherwise preserve the original error - if (Runtime.PyErr_ExceptionMatches(Exceptions.AttributeError) == 0) - { - return default; - } - - if (!Runtime.PyString_Check(key)) - { - return default; - } - - if (GetManagedObject(ob) is not CLRObject co) - { - return default; - } - - // Slot registration already guarantees this type supports DLR - var dynamicObject = (IDynamicMetaObjectProvider)co.inst; - - string? memberName = Runtime.GetManagedString(key); - if (memberName is null) - { - return default; - } - - if (!dynamicMemberAccessor.TryGetMember(dynamicObject, memberName, out object? value)) - { - return default; - } - - // Clear the lingering AttributeError - Runtime.PyErr_Clear(); - - using var pyValue = value.ToPython(); - return pyValue.NewReferenceOrNull(); - } - - static int tp_setattro_dlr(BorrowedReference ob, BorrowedReference key, BorrowedReference val) - { - int result = Runtime.PyObject_GenericSetAttr(ob, key, val); - if (result == 0) - { - return 0; - } - - // Preserve non-attribute errors exactly as they are. - if (Runtime.PyErr_ExceptionMatches(Exceptions.AttributeError) == 0) - { - return -1; - } - - // Deletion fallback is intentionally not handled by DLR binder yet. - if (val == null) - { - return -1; - } - - if (!Runtime.PyString_Check(key)) - { - return -1; - } - - if (GetManagedObject(ob) is not CLRObject co) - { - return -1; - } - - // Slot registration already guarantees this type supports DLR. - var dynamicObject = (IDynamicMetaObjectProvider)co.inst; - - string? memberName = Runtime.GetManagedString(key); - if (memberName is null) - { - return -1; - } - - if (!Converter.ToManaged(val, typeof(object), out object? managedValue, true)) - { - return -1; - } - - if (!dynamicMemberAccessor.TrySetMember(dynamicObject, memberName, managedValue)) - { - return -1; - } - - // Clear the lingering AttributeError - Runtime.PyErr_Clear(); - return 0; - } - public virtual void InitializeSlots(BorrowedReference pyType, SlotsHolder slotsHolder) { if (!this.type.Valid) return; @@ -716,12 +613,6 @@ public virtual void InitializeSlots(BorrowedReference pyType, SlotsHolder slotsH TypeManager.InitializeSlotIfEmpty(pyType, TypeOffset.tp_call, new Interop.BBB_N(tp_call_impl), slotsHolder); } - if (typeof(IDynamicMetaObjectProvider).IsAssignableFrom(this.type.Value)) - { - TypeManager.InitializeSlotIfEmpty(pyType, TypeOffset.tp_getattro, new Interop.BB_N(tp_getattro_dlr), slotsHolder); - TypeManager.InitializeSlotIfEmpty(pyType, TypeOffset.tp_setattro, new Interop.BBB_I32(tp_setattro_dlr), slotsHolder); - } - if (indexer is not null) { if (indexer.CanGet) From 33fccdbe3657cf8fbfa6bcce3584b5729852017a Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Tue, 5 May 2026 13:40:40 +0200 Subject: [PATCH 05/10] Recover behaviour for derived types --- src/runtime/TypeManager.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index e18970ce2..ca551e763 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -172,6 +172,18 @@ public static int tp_setattro_dlr_proxy(BorrowedReference ob, BorrowedReference && memberName is not null && !HasClrMember(instance, memberName); + // For Python-derived types (IPythonDerivedType), the Python descriptor protocol + // (e.g. @property setters) takes priority over DLR member storage. Try + // PyObject_GenericSetAttr first; only fall back to DLR on AttributeError. + if (canUseDynamic && instance is IPythonDerivedType) + { + int pyResult = Runtime.PyObject_GenericSetAttr(ob, key, val); + if (pyResult == 0) return 0; + if (Runtime.PyErr_ExceptionMatches(Exceptions.AttributeError) == 0) return pyResult; + Runtime.PyErr_Clear(); + // fall through to DLR path below + } + if (canUseDynamic) { if (val == null) From 0c95341c2756096acbe1f305deac6ac8c56200a0 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Tue, 5 May 2026 16:59:21 +0200 Subject: [PATCH 06/10] Simplify the setattr and getattr functions a bit --- src/runtime/TypeManager.cs | 132 +++++++++++------------------- src/runtime/Types/ClassDerived.cs | 2 +- 2 files changed, 50 insertions(+), 84 deletions(-) diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index ca551e763..30a1a9563 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -5,6 +5,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Diagnostics; + using Python.Runtime.Native; using Python.Runtime.StateSerialization; @@ -76,74 +77,48 @@ static bool TryGetDynamicInstance(BorrowedReference ob, out object instance, out return false; } - static bool TryGetManagedDynamicInstance(BorrowedReference ob, out object instance, out IDynamicMetaObjectProvider dynamicObject) + public static NewReference tp_getattro_dlr_proxy(BorrowedReference ob, BorrowedReference key) { - if (ManagedType.GetManagedObject(ob) is CLRObject co && co.inst is IDynamicMetaObjectProvider coDynamic) + var isDynamic = TryGetDynamicInstance(ob, out object instance, out IDynamicMetaObjectProvider dynamicObject); + + // The whole DLR machinery only makes sense with string keys and dynamic objects + if (!isDynamic || !Runtime.PyString_Check(key)) { - instance = co.inst; - dynamicObject = coDynamic; - return true; + return Runtime.PyObject_GenericGetAttr(ob, key); } - instance = null!; - dynamicObject = null!; - return false; - } - - public static NewReference tp_getattro_dlr_proxy(BorrowedReference ob, BorrowedReference key) - { - bool hasStringKey = Runtime.PyString_Check(key); - string? memberName = hasStringKey ? Runtime.GetManagedString(key) : null; + string memberName = Runtime.GetManagedString(key)!; - if (memberName is not null && TryGetManagedDynamicInstance(ob, out object preInstance, out var preDynamicObject)) + // Forward requests to GetDynamicMemberNames to the mixin implementation + if (memberName == nameof(DynamicObjectMemberAccessor.GetDynamicMemberNames) + && !HasClrMember(instance, memberName)) { - if (memberName == nameof(DynamicObjectMemberAccessor.GetDynamicMemberNames) - && !HasClrMember(preInstance, memberName)) - { - using var pyMemberNames = new Func>( - () => dynamicMemberAccessor.GetDynamicMemberNames(preDynamicObject)).ToPython(); - return pyMemberNames.NewReferenceOrNull(); - } + using var pyMemberNames = new Func>( + () => dynamicMemberAccessor.GetDynamicMemberNames(dynamicObject) + ).ToPython(); + return pyMemberNames.NewReferenceOrNull(); } + // Now, first try to access the Python attribute var attr = Runtime.PyObject_GenericGetAttr(ob, key); if (!attr.IsNull()) - { return attr; - } - - if (memberName == null) - { - return default; - } + // attr is null, so an exception must be set. If that exception is not an AttributeError, + // we return from this function immediately without clearing. All later returns until the + // very end will lead to the AttributeError getting raised. if (Runtime.PyErr_ExceptionMatches(Exceptions.AttributeError) == 0) { return default; } - if (!TryGetManagedDynamicInstance(ob, out object instance, out var dynamicObject)) + if (HasClrMember(instance, memberName) || IsPythonSpecialAttributeName(memberName)) { return default; } - if (memberName is null) - { - return default; - } - - if (HasClrMember(instance, memberName)) - { - return default; - } - - if (IsPythonSpecialAttributeName(memberName)) - { - return default; - } - - bool resolved; - object? value; + bool resolved = false; + object? value = null; try { resolved = dynamicMemberAccessor.TryGetMember(dynamicObject, memberName, out value); @@ -166,64 +141,55 @@ public static NewReference tp_getattro_dlr_proxy(BorrowedReference ob, BorrowedR public static int tp_setattro_dlr_proxy(BorrowedReference ob, BorrowedReference key, BorrowedReference val) { - string? memberName = Runtime.PyString_Check(key) ? Runtime.GetManagedString(key) : null; - bool hasDynamicInstance = TryGetDynamicInstance(ob, out object instance, out var dynamicObject); - bool canUseDynamic = hasDynamicInstance - && memberName is not null - && !HasClrMember(instance, memberName); + var isDynamic = TryGetDynamicInstance(ob, out object instance, out IDynamicMetaObjectProvider dynamicObject); + + // The whole DLR machinery only makes sense with string keys and dynamic objects + if (!isDynamic || !Runtime.PyString_Check(key)) + { + return Runtime.PyObject_GenericSetAttr(ob, key, val); + } + + string memberName = Runtime.GetManagedString(key)!; // For Python-derived types (IPythonDerivedType), the Python descriptor protocol - // (e.g. @property setters) takes priority over DLR member storage. Try - // PyObject_GenericSetAttr first; only fall back to DLR on AttributeError. - if (canUseDynamic && instance is IPythonDerivedType) + // (e.g. @property setters) takes priority over DLR member storage. + if (instance is IPythonDerivedType) { int pyResult = Runtime.PyObject_GenericSetAttr(ob, key, val); - if (pyResult == 0) return 0; - if (Runtime.PyErr_ExceptionMatches(Exceptions.AttributeError) == 0) return pyResult; + if (pyResult == 0) + return 0; + + if (Runtime.PyErr_ExceptionMatches(Exceptions.AttributeError) == 0) + return pyResult; + Runtime.PyErr_Clear(); - // fall through to DLR path below + // Fall through to DLR fallback below } - if (canUseDynamic) + if (!HasClrMember(instance, memberName) && !IsPythonSpecialAttributeName(memberName)) { + // Try DLR member storage first + bool handled = false; + if (val == null) { - if (dynamicMemberAccessor.TryDeleteMember(dynamicObject, memberName!)) - { - Runtime.PyErr_Clear(); - return 0; - } + handled = dynamicMemberAccessor.TryDeleteMember(dynamicObject, memberName); } else { object? managedValue = null; if (val != Runtime.PyNone && !Converter.ToManaged(val, typeof(object), out managedValue, true)) - { return -1; - } - if (dynamicMemberAccessor.TrySetMember(dynamicObject, memberName!, managedValue)) - { - Runtime.PyErr_Clear(); - return 0; - } + handled = dynamicMemberAccessor.TrySetMember(dynamicObject, memberName, managedValue); } - } - int result = Runtime.PyObject_GenericSetAttr(ob, key, val); - if (result == 0 && canUseDynamic) - { - object? managedValue = null; - if (val != null && val != Runtime.PyNone && !Converter.ToManaged(val, typeof(object), out managedValue, true)) - { + if (handled) return 0; - } - - dynamicMemberAccessor.TrySetMember(dynamicObject, memberName!, managedValue); - Runtime.PyErr_Clear(); } - return result; + // Fall back to Python attribute setting + return Runtime.PyObject_GenericSetAttr(ob, key, val); } internal static void Initialize() diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 3129e0a89..69eba2cc2 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -41,7 +41,7 @@ static ClassDerivedObject() moduleBuilders = new Dictionary, ModuleBuilder>(); } - public static new void Reset() + public static void Reset() { assemblyBuilders = new Dictionary(); moduleBuilders = new Dictionary, ModuleBuilder>(); From b527385535fcdd059e67c455453abbb31586e290 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Tue, 5 May 2026 17:03:21 +0200 Subject: [PATCH 07/10] Use explicit IsNull on BorrowedReference --- src/runtime/TypeManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 30a1a9563..496b5d357 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -171,7 +171,7 @@ public static int tp_setattro_dlr_proxy(BorrowedReference ob, BorrowedReference // Try DLR member storage first bool handled = false; - if (val == null) + if (val.IsNull) { handled = dynamicMemberAccessor.TryDeleteMember(dynamicObject, memberName); } From 18f4c0f39a0c9ca55a07282e99e00b0847c9301a Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 7 May 2026 18:59:18 +0200 Subject: [PATCH 08/10] Catch errors in setting/deleting properties - Catch exceptions in TrySet/DeleteMember - Convert the exceptions into Python exceptions - Add tests for the remaining cases - Add a note on why the field has to be lazily initialized (general issue with derived classes) --- src/runtime/TypeManager.cs | 31 ++++++++++++------ src/testing/dlrtest.cs | 65 ++++++++++++++++++++++++++++++++------ tests/test_dynamic.py | 38 +++++++++++++++++++++- 3 files changed, 115 insertions(+), 19 deletions(-) diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 496b5d357..30a99690a 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -169,19 +169,32 @@ public static int tp_setattro_dlr_proxy(BorrowedReference ob, BorrowedReference if (!HasClrMember(instance, memberName) && !IsPythonSpecialAttributeName(memberName)) { // Try DLR member storage first - bool handled = false; + bool handled; - if (val.IsNull) + try { - handled = dynamicMemberAccessor.TryDeleteMember(dynamicObject, memberName); + if (val.IsNull) + { + handled = dynamicMemberAccessor.TryDeleteMember(dynamicObject, memberName); + } + else + { + object? managedValue = null; + if (val != Runtime.PyNone && !Converter.ToManaged(val, typeof(object), out managedValue, true)) + return -1; + + handled = dynamicMemberAccessor.TrySetMember(dynamicObject, memberName, managedValue); + if (!handled) + { + Exceptions.SetError(Exceptions.AttributeError, $"'{instance.GetType().Name}' object has no attribute '{memberName}'"); + return -1; + } + } } - else + catch (Exception e) { - object? managedValue = null; - if (val != Runtime.PyNone && !Converter.ToManaged(val, typeof(object), out managedValue, true)) - return -1; - - handled = dynamicMemberAccessor.TrySetMember(dynamicObject, memberName, managedValue); + Exceptions.SetError(e); + return -1; } if (handled) diff --git a/src/testing/dlrtest.cs b/src/testing/dlrtest.cs index 5805e8871..b58f5fc82 100644 --- a/src/testing/dlrtest.cs +++ b/src/testing/dlrtest.cs @@ -4,12 +4,40 @@ namespace Python.Test; -public class DynamicMappingObject : DynamicObject +/// +/// Base class for dynamic test helpers. Uses lazy storage initialization so that +/// Python-derived subclasses can safely call DynamicObject member hooks before +/// managed field initializers have run. +/// +public class DynamicStorageObject : DynamicObject { Dictionary storage; - Dictionary Storage => storage ??= []; + // Python-defined subclasses may reach this type without running managed field + // initializers (see ClassDerivedObject.NewObjectToPython). Via the lazy init + // we can ensure that the access is still safe, even when the constructor has + // not run. + protected Dictionary Storage => storage ??= []; + public void AddDynamicMember(string name, object value) => Storage[name] = value; + + public override bool TryGetMember(GetMemberBinder binder, out object result) + => Storage.TryGetValue(binder.Name, out result); + + public override bool TrySetMember(SetMemberBinder binder, object value) + { + Storage[binder.Name] = value; + return true; + } + + public override bool TryDeleteMember(DeleteMemberBinder binder) + => Storage.Remove(binder.Name); + + public override IEnumerable GetDynamicMemberNames() => Storage.Keys; +} + +public class DynamicMappingObject : DynamicStorageObject +{ // Native members for testing that regular CLR access is unaffected. public string Label = "default"; public int Multiplier { get; set; } = 1; @@ -20,20 +48,39 @@ public class DynamicMappingObject : DynamicObject // Test helper: retrieve the actual value stored in C# (for verification that None was stored as null) public object GetDynamicValue(string name) => Storage.TryGetValue(name, out var value) ? value : null; +} - - - public override bool TryGetMember(GetMemberBinder binder, out object result) - => Storage.TryGetValue(binder.Name, out result); - +public class RejectingSetDynamicObject : DynamicStorageObject +{ public override bool TrySetMember(SetMemberBinder binder, object value) { + if (!Storage.ContainsKey(binder.Name)) + return false; + Storage[binder.Name] = value; return true; } +} + +public class ThrowingSetDynamicObject : DynamicStorageObject +{ + public override bool TrySetMember(SetMemberBinder binder, object value) + => throw new InvalidOperationException($"TrySetMember failed for '{binder.Name}'"); +} +public class RejectingDeleteDynamicObject : DynamicStorageObject +{ public override bool TryDeleteMember(DeleteMemberBinder binder) - => binder is not null && Storage.Remove(binder.Name); + { + if (!Storage.ContainsKey(binder.Name)) + return false; - public override IEnumerable GetDynamicMemberNames() => Storage.Keys; + return Storage.Remove(binder.Name); + } +} + +public class ThrowingDeleteDynamicObject : DynamicStorageObject +{ + public override bool TryDeleteMember(DeleteMemberBinder binder) + => throw new InvalidOperationException($"TryDeleteMember failed for '{binder.Name}'"); } diff --git a/tests/test_dynamic.py b/tests/test_dynamic.py index b18ac1fdd..b8caa24b3 100644 --- a/tests/test_dynamic.py +++ b/tests/test_dynamic.py @@ -5,6 +5,10 @@ from System.Dynamic import ExpandoObject from Python.Test import DynamicMappingObject +from Python.Test import RejectingDeleteDynamicObject +from Python.Test import RejectingSetDynamicObject +from Python.Test import ThrowingDeleteDynamicObject +from Python.Test import ThrowingSetDynamicObject def _mro_names(obj): @@ -170,4 +174,36 @@ def custom_property(self, i): assert obj.custom_property == 10 obj.other_property = None - assert obj.other_property is None \ No newline at end of file + assert obj.other_property is None + + +def test_trysetmember_false_raises_attributeerror_instead_of_silent_python_setattr(): + obj = RejectingSetDynamicObject() + + with pytest.raises(AttributeError): + obj.typoed_name = 42 + + assert not hasattr(obj, "typoed_name") + + +def test_trysetmember_exception_is_raised_in_python(): + obj = ThrowingSetDynamicObject() + + with pytest.raises(Exception, match="TrySetMember failed for 'bad_name'"): + obj.bad_name = 42 + + +def test_trydeletemember_false_raises_attributeerror(): + obj = RejectingDeleteDynamicObject() + obj.AddDynamicMember("existing_name", 42) + + with pytest.raises(AttributeError): + del obj.missing_name + + +def test_trydeletemember_exception_is_raised_in_python(): + obj = ThrowingDeleteDynamicObject() + obj.bad_name = 42 + + with pytest.raises(Exception, match="TryDeleteMember failed for 'bad_name'"): + del obj.bad_name \ No newline at end of file From a2ed667b5b83701963e91db93c5b4239fe7e9edb Mon Sep 17 00:00:00 2001 From: greateggsgreg <36009512+greateggsgreg@users.noreply.github.com> Date: Mon, 11 May 2026 07:24:32 -0400 Subject: [PATCH 09/10] Propagate exceptions from TryGetMember in tp_getattro_dlr_proxy (#2718) The dynamic getter swallowed any exception from TryGetMember and returned default to Python with the prior AttributeError still set, so user code observed a misleading AttributeError instead of the real failure. Set a Python exception in the catch arm. We use RuntimeError with the message string rather than Converter.ToPython(e) because wrapping the CLR exception object can trigger type initialisation that re-enters this same slot on the live dynamic object, producing infinite recursion. Mirrors the symmetry already present in the setter (#2706 review, @lostmsu) and adds a regression test alongside the existing ThrowingSetDynamicObject coverage. --- src/runtime/TypeManager.cs | 7 ++++++- src/testing/dlrtest.cs | 6 ++++++ tests/test_dynamic.py | 9 +++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 30a99690a..3b3c6db1a 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -123,8 +123,13 @@ public static NewReference tp_getattro_dlr_proxy(BorrowedReference ob, BorrowedR { resolved = dynamicMemberAccessor.TryGetMember(dynamicObject, memberName, out value); } - catch + catch (Exception e) { + // Avoid wrapping the CLR exception via Converter.ToPython here: that would trigger + // CLR type initialisation which can re-enter this slot on the same live object, + // causing infinite recursion. A plain RuntimeError with the message is safe. + Runtime.PyErr_Clear(); + Exceptions.SetError(Exceptions.RuntimeError, e.Message); return default; } diff --git a/src/testing/dlrtest.cs b/src/testing/dlrtest.cs index b58f5fc82..783a3a133 100644 --- a/src/testing/dlrtest.cs +++ b/src/testing/dlrtest.cs @@ -62,6 +62,12 @@ public override bool TrySetMember(SetMemberBinder binder, object value) } } +public class ThrowingGetDynamicObject : DynamicStorageObject +{ + public override bool TryGetMember(GetMemberBinder binder, out object result) + => throw new InvalidOperationException($"TryGetMember failed for '{binder.Name}'"); +} + public class ThrowingSetDynamicObject : DynamicStorageObject { public override bool TrySetMember(SetMemberBinder binder, object value) diff --git a/tests/test_dynamic.py b/tests/test_dynamic.py index b8caa24b3..f093ee19a 100644 --- a/tests/test_dynamic.py +++ b/tests/test_dynamic.py @@ -8,6 +8,7 @@ from Python.Test import RejectingDeleteDynamicObject from Python.Test import RejectingSetDynamicObject from Python.Test import ThrowingDeleteDynamicObject +from Python.Test import ThrowingGetDynamicObject from Python.Test import ThrowingSetDynamicObject @@ -186,6 +187,14 @@ def test_trysetmember_false_raises_attributeerror_instead_of_silent_python_setat assert not hasattr(obj, "typoed_name") +def test_trygetmember_exception_is_raised_in_python(): + obj = ThrowingGetDynamicObject() + obj.AddDynamicMember("any_key", 1) + + with pytest.raises(Exception, match="TryGetMember failed for 'any_key'"): + _ = obj.any_key + + def test_trysetmember_exception_is_raised_in_python(): obj = ThrowingSetDynamicObject() From a3c20693d2aae63dfe355303cd74d1435f8926be Mon Sep 17 00:00:00 2001 From: greateggsgreg Date: Mon, 11 May 2026 22:50:05 -0400 Subject: [PATCH 10/10] Cache HasClrMember and align DLR setter exception path - Cache HasClrMember reflection per (Type, name) so tp_getattro_dlr_proxy / tp_setattro_dlr_proxy avoid repeated GetMember() calls on every attribute access of DLR-aware objects. - Mirror tp_setattro_dlr_proxy's catch arm to the getter's safer SetError(RuntimeError, e.Message) shape instead of SetError(Exception), keeping both slots re-entry-safe on live dynamic objects. Related to #2706. --- src/runtime/TypeManager.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 3b3c6db1a..c02d94a1f 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Dynamic; using System.Linq; @@ -41,8 +42,14 @@ internal class TypeManager static readonly DynamicObjectMemberAccessor dynamicMemberAccessor = new(); + // tp_getattro_dlr_proxy / tp_setattro_dlr_proxy hit HasClrMember on every + // attribute access; cache the reflection result per (Type, name). + static readonly ConcurrentDictionary<(Type, string), bool> _hasClrMemberCache = new(); + static bool HasClrMember(object instance, string memberName) => - instance.GetType().GetMember(memberName, BindingFlags.Public | BindingFlags.Instance).Length > 0; + _hasClrMemberCache.GetOrAdd( + (instance.GetType(), memberName), + k => k.Item1.GetMember(k.Item2, BindingFlags.Public | BindingFlags.Instance).Length > 0); static bool IsPythonSpecialAttributeName(string memberName) => memberName.Length > 4 && memberName.StartsWith("__") && memberName.EndsWith("__"); @@ -198,7 +205,9 @@ public static int tp_setattro_dlr_proxy(BorrowedReference ob, BorrowedReference } catch (Exception e) { - Exceptions.SetError(e); + // Same reasoning as the getter: avoid Converter.ToPython(e) to keep this + // slot re-entry-safe on live dynamic objects. + Exceptions.SetError(Exceptions.RuntimeError, e.Message); return -1; }