diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3396b83cc..98cd36538 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,101 @@ name: Main +on: + push: + branches: + - master + pull_request: + +jobs: + build-test: + name: Build and Test + runs-on: ${{ matrix.os }}-latest + timeout-minutes: 15 + + strategy: + fail-fast: false + matrix: + os: [windows, ubuntu, macos] + python: ["3.7", "3.8", "3.9", "3.10", "3.11"] + platform: [x64, x86] + exclude: + - os: ubuntu + platform: x86 + - os: macos + platform: x86 + + steps: + - name: Set Environment on macOS + uses: maxim-lobanov/setup-xamarin@v1 + if: ${{ matrix.os == 'macos' }} + with: + mono-version: latest + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup .NET + uses: actions/setup-dotnet@v1 + with: + dotnet-version: '6.0.x' + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + architecture: ${{ matrix.platform }} + + - name: Install dependencies + run: | + pip install --upgrade -r requirements.txt + pip install numpy # for tests + + - name: Build and Install + run: | + pip install -v . + + - name: Set Python DLL path and PYTHONHOME (non Windows) + if: ${{ matrix.os != 'windows' }} + run: | + echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV + echo PYTHONHOME=$(python -c 'import sys; print(sys.prefix)') >> $GITHUB_ENV + + - name: Set Python DLL path and PYTHONHOME (Windows) + if: ${{ matrix.os == 'windows' }} + run: | + Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONNET_PYDLL=$(python -m find_libpython)" + Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONHOME=$(python -c 'import sys; print(sys.prefix)')" + + - name: Embedding tests + run: dotnet test --runtime any-${{ matrix.platform }} --logger "console;verbosity=detailed" src/embed_tests/ + env: + MONO_THREADS_SUSPEND: preemptive # https://github.com/mono/mono/issues/21466 + + - name: Python Tests (Mono) + if: ${{ matrix.os != 'windows' }} + run: pytest --runtime mono + + # TODO: Run these tests on Windows x86 + - name: Python Tests (.NET Core) + if: ${{ matrix.platform == 'x64' }} + run: pytest --runtime coreclr + + - name: Python Tests (.NET Framework) + if: ${{ matrix.os == 'windows' }} + run: pytest --runtime netfx + + - name: Python tests run from .NET + run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/ + + - name: Perf tests + if: ${{ (matrix.python == '3.8') && (matrix.platform == 'x64') }} + run: | + pip install --force --no-deps --target src/perf_tests/baseline/ pythonnet==2.5.2 + dotnet test --configuration Release --runtime any-${{ matrix.platform }} --logger "console;verbosity=detailed" src/perf_tests/ + + # TODO: Run mono tests on Windows? +name: Main + on: push: branches: diff --git a/CHANGELOG.md b/CHANGELOG.md index 29b4c4a68..d20877bab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,946 @@ project adheres to [Semantic Versioning][]. This document follows the conventions laid out in [Keep a CHANGELOG][]. +## [Unreleased][] + +### Added + +### Changed + +### Fixed + +## [3.0.1](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.1) - 2022-11-03 + +### Added + +- Support for Python 3.11 + +### Changed + +- Allow decoders to override conversion of types derived from primitive types + +### Fixed + +- Fixed objects leaking when Python attached event handlers to them even if they were later removed +- Fixed `PyInt` conversion to `BigInteger` and `System.String` produced incorrect result for values between 128 and 255. +- Fixed implementing a generic interface with a Python class + + +## [3.0.0](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.0) - 2022-09-29 + +### Added + +- Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax +- Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). +- Add GetPythonThreadID and Interrupt methods in PythonEngine +- Ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. ([#1355][i1355]) +- Ability to override .NET methods that have `out` or `ref` in Python by returning the modified parameter values in a tuple. ([#1481][i1481]) +- `PyType` - a wrapper for Python type objects, that also permits creating new heap types from `TypeSpec` +- Improved exception handling: + * exceptions can now be converted with codecs + * `InnerException` and `__cause__` are propagated properly +- `__name__` and `__signature__` to reflected .NET methods +- .NET collection types now implement standard Python collection interfaces from `collections.abc`. +See [Mixins/collections.py](src/runtime/Mixins/collections.py). +- you can cast objects to generic .NET interfaces without specifying generic arguments as long as there is no ambiguity. +- .NET arrays implement Python buffer protocol +- Python integer interoperability with `System.Numerics.BigInteger` +- Python.NET will correctly resolve .NET methods, that accept `PyList`, `PyInt`, +and other `PyObject` derived types when called from Python. +- .NET classes, that have `__call__` method are callable from Python +- `PyIterable` type, that wraps any iterable object in Python +- `PythonEngine` properties for supported Python versions: `MinSupportedVersion`, `MaxSupportedVersion`, and `IsSupportedVersion` +- The runtime that is loaded on `import clr` can now be configured via environment variables + + +### Changed +- Drop support for Python 2, 3.4, 3.5, and 3.6 +- `wchar_t` size aka `Runtime.UCS` is now determined at runtime +- `clr.AddReference` may now throw errors besides `FileNotFoundException`, that provide more +details about the cause of the failure +- `clr.AddReference` no longer adds ".dll" implicitly +- `PyIter(PyObject)` constructor replaced with static `PyIter.GetIter(PyObject)` method +- Python runtime can no longer be shut down if the Python error indicator is set, as it would have unpredictable behavior +- BREAKING: Return values from .NET methods that return an interface are now automatically + wrapped in that interface. This is a breaking change for users that rely on being + able to access members that are part of the implementation class, but not the + interface. Use the new `__implementation__` or `__raw_implementation__` properties to + if you need to "downcast" to the implementation class. +- BREAKING: `==` and `!=` operators on `PyObject` instances now use Python comparison + (previously was equivalent to `object.ReferenceEquals(,)`) +- BREAKING: Parameters marked with `ParameterAttributes.Out` are no longer returned in addition + to the regular method return value (unless they are passed with `ref` or `out` keyword). +- BREAKING: Drop support for the long-deprecated CLR.* prefix. +- `PyObject` now implements `IEnumerable` in addition to `IEnumerable` +- floating point values passed from Python are no longer silently truncated +when .NET expects an integer [#1342][i1342] +- More specific error messages for method argument mismatch +- members of `PyObject` inherited from `System.Object and `DynamicObject` now autoacquire GIL +- BREAKING: when inheriting from .NET types in Python if you override `__init__` you +must explicitly call base constructor using `super().__init__(.....)`. Not doing so will lead +to undefined behavior. +- BREAKING: most `PyScope` methods will never return `null`. Instead, `PyObject` `None` will be returned. +- BREAKING: `PyScope` was renamed to `PyModule` +- BREAKING: Methods with `ref` or `out` parameters and void return type return a tuple of only the `ref` and `out` parameters. +- BREAKING: to call Python from .NET `Runtime.PythonDLL` property must be set to Python DLL name +or the DLL must be loaded in advance. This must be done before calling any other Python.NET functions. +- BREAKING: `PyObject.Length()` now raises a `PythonException` when object does not support a concept of length. +- BREAKING: disabled implicit conversion from C# enums to Python `int` and back. +One must now either use enum members (e.g. `MyEnum.Option`), or use enum constructor +(e.g. `MyEnum(42)` or `MyEnum(42, True)` when `MyEnum` does not have a member with value 42). +- BREAKING: disabled implicit conversion from Python objects implementing sequence protocol to +.NET arrays when the target .NET type is `System.Object`. The conversion is still attempted when the +target type is a `System.Array`. +- Sign Runtime DLL with a strong name +- Implement loading through `clr_loader` instead of the included `ClrModule`, enables + support for .NET Core +- BREAKING: .NET and Python exceptions are preserved when crossing Python/.NET boundary +- BREAKING: custom encoders are no longer called for instances of `System.Type` +- `PythonException.Restore` no longer clears `PythonException` instance. +- Replaced the old `__import__` hook hack with a PEP302-style Meta Path Loader +- BREAKING: Names of .NET types (e.g. `str(__class__)`) changed to better support generic types +- BREAKING: overload resolution will no longer prefer basic types. Instead, first matching overload will +be chosen. +- BREAKING: acquiring GIL using `Py.GIL` no longer forces `PythonEngine` to initialize +- BREAKING: `Exec` and `Eval` from `PythonEngine` no longer accept raw pointers. +- BREAKING: .NET collections and arrays are no longer automatically converted to +Python collections. Instead, they implement standard Python +collection interfaces from `collections.abc`. +See [Mixins/collections.py](src/runtime/Mixins/collections.py). +- BREAKING: When trying to convert Python `int` to `System.Object`, result will +be of type `PyInt` instead of `System.Int32` due to possible loss of information. +Python `float` will continue to be converted to `System.Double`. +- BREAKING: Python.NET will no longer implicitly convert types like `numpy.float64`, that implement `__float__` to +`System.Single` and `System.Double`. An explicit conversion is required on Python or .NET side. +- BREAKING: `PyObject.GetHashCode` can fail. +- BREAKING: Python.NET will no longer implicitly convert any Python object to `System.Boolean`. +- BREAKING: `PyObject.GetAttr(name, default)` now only ignores `AttributeError` (previously ignored all exceptions). +- BREAKING: `PyObject` no longer implements `IEnumerable`. +Instead, `PyIterable` does that. +- BREAKING: `IPyObjectDecoder.CanDecode` `objectType` parameter type changed from `PyObject` to `PyType` + +### Fixed + +- Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash +- Fixed parameterless .NET constructor being silently called when a matching constructor overload is not found ([#238][i238]) +- Fix incorrect dereference in params array handling +- Fixes issue with function resolution when calling overloaded function with keyword arguments from python ([#1097][i1097]) +- Fix `object[]` parameters taking precedence when should not in overload resolution +- Fixed a bug where all .NET class instances were considered Iterable +- Fix incorrect choice of method to invoke when using keyword arguments. +- Fix non-delegate types incorrectly appearing as callable. +- Indexers can now be used with interface objects +- Fixed a bug where indexers could not be used if they were inherited +- Made it possible to use `__len__` also on `ICollection<>` interface objects +- Fixed issue when calling PythonException.Format where another exception would be raise for unnormalized exceptions +- Made it possible to call `ToString`, `GetHashCode`, and `GetType` on inteface objects +- Fixed objects returned by enumerating `PyObject` being disposed too soon +- Incorrectly using a non-generic type with type parameters now produces a helpful Python error instead of throwing NullReferenceException +- `import` may now raise errors with more detail than "No module named X" +- Exception stacktraces on `PythonException.StackTrace` are now properly formatted +- Providing an invalid type parameter to a generic type or method produces a helpful Python error +- Empty parameter names (as can be generated from F#) do not cause crashes +- Unicode strings with surrogates were truncated when converting from Python +- `Reload` mode now supports generic methods (previously Python would stop seeing them after reload) +- Temporarily fixed issue resolving method overload when method signature has `out` parameters ([#1672](i1672)) +- Decimal default parameters are now correctly taken into account + +### Removed + +- `ShutdownMode` has been removed. The only shutdown mode supported now is an equivalent of `ShutdownMode.Reload`. +There is no need to specify it. +- implicit assembly loading (you have to explicitly `clr.AddReference` before doing import) +- messages in `PythonException` no longer start with exception type +- `PyScopeManager`, `PyScopeException`, `PyScope` (use `PyModule` instead) +- support for .NET Framework 4.0-4.6; Mono before 5.4. Python.NET now requires .NET Standard 2.0 +(see [the matrix](https://docs.microsoft.com/en-us/dotnet/standard/net-standard#net-implementation-support)) + +## [2.5.2](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.2) - 2021-02-05 + +Bugfix release. + +### Fixed +- Fix `object[]` parameters taking precedence when should not in overload resolution +- Empty parameter names (as can be generated from F#) do not cause crashes + +## [2.5.1](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.1) - 2020-06-18 + +Bugfix release. + +### Fixed + +- Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash +- Fix incorrect dereference in params array handling + +## [2.5.0](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.0) - 2020-06-14 + +This version improves performance on benchmarks significantly compared to 2.3. + +### Added + +- Automatic NuGet package generation in appveyor and local builds +- Function that sets `Py_NoSiteFlag` to 1. +- Support for Jetson Nano. +- Support for `__len__` for .NET classes that implement ICollection +- `PyExport` attribute to hide .NET types from Python +- `PythonException.Format` method to format exceptions the same as + `traceback.format_exception` +- `Runtime.None` to be able to pass `None` as parameter into Python from .NET +- `PyObject.IsNone()` to check if a Python object is None in .NET. +- Support for Python 3.8 +- Codecs as the designated way to handle automatic conversions between + .NET and Python types +- Added Python 3 buffer api support and PyBuffer interface for fast byte and numpy array read/write ([#980][p980]) + +### Changed + +- Added argument types information to "No method matches given arguments" message +- Moved wheel import in setup.py inside of a try/except to prevent pip collection failures +- Removes `PyLong_GetMax` and `PyClass_New` when targetting Python3 +- Improved performance of calls from Python to C# +- Added support for converting python iterators to C# arrays +- Changed usage of the obsolete function + `GetDelegateForFunctionPointer(IntPtr, Type)` to + `GetDelegateForFunctionPointer(IntPtr)` +- When calling C# from Python, enable passing argument of any type to a + parameter of C# type `object` by wrapping it into `PyObject` instance. + ([#881][i881]) +- Added support for kwarg parameters when calling .NET methods from Python +- Changed method for finding MSBuild using vswhere +- Reworked `Finalizer`. Now objects drop into its queue upon finalization, + which is periodically drained when new objects are created. +- Marked `Runtime.OperatingSystemName` and `Runtime.MachineName` as + `Obsolete`, should never have been `public` in the first place. They also + don't necessarily return a result that matches the `platform` module's. +- Unconditionally depend on `pycparser` for the interop module generation + +### Fixed + +- Fixed runtime that fails loading when using pythonnet in an environment + together with Nuitka +- Fixes bug where delegates get casts (dotnetcore) +- Determine size of interpreter longs at runtime +- Handling exceptions ocurred in ModuleObject's getattribute +- Fill `__classcell__` correctly for Python subclasses of .NET types +- Fixed issue with params methods that are not passed an array. +- Use UTF8 to encode strings passed to `PyRun_String` on Python 3 + +## [2.4.0][] - 2019-05-15 + +### Added + +- Added support for embedding python into dotnet core 2.0 (NetStandard 2.0) +- Added new build system (pythonnet.15.sln) based on dotnetcore-sdk/xplat(crossplatform msbuild). + Currently there two side-by-side build systems that produces the same output (net40) from the same sources. + After a some transition time, current (mono/ msbuild 14.0) build system will be removed. +- NUnit upgraded to 3.7 (eliminates travis-ci random bug) +- Added C# `PythonEngine.AddShutdownHandler` to help client code clean up on shutdown. +- Added `clr.GetClrType` ([#432][i432])([#433][p433]) +- Allowed passing `None` for nullable args ([#460][p460]) +- Added keyword arguments based on C# syntax for calling CPython methods ([#461][p461]) +- Catches exceptions thrown in C# iterators (yield returns) and rethrows them in python ([#475][i475])([#693][p693]) +- Implemented GetDynamicMemberNames() for PyObject to allow dynamic object members to be visible in the debugger ([#443][i443])([#690][p690]) +- Incorporated reference-style links to issues and pull requests in the CHANGELOG ([#608][i608]) +- Added PyObject finalizer support, Python objects referred by C# can be auto collect now ([#692][p692]). +- Added detailed comments about aproaches and dangers to handle multi-app-domains ([#625][p625]) +- Python 3.7 support, builds and testing added. Defaults changed from Python 3.6 to 3.7 ([#698][p698]) +- Added support for C# types to provide `__repr__` ([#680][p680]) + +### Changed + +- PythonException included C# call stack +- Reattach python exception traceback information (#545) +- PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) +- Refactored MethodBinder.Bind in preparation to make it extensible (#829) +- Look for installed Windows 10 sdk's during installation instead of relying on specific versions. +- Remove `LoadLibrary` call. ([#880][p880]) + +### Fixed + +- Fixed secondary PythonEngine.Initialize call, all sensitive static variables now reseted. + This is a hidden bug. Once python cleaning up enough memory, objects from previous engine run becomes corrupted. ([#534][p534]) +- Fixed Visual Studio 2017 compat ([#434][i434]) for setup.py +- Fixed crashes when integrating pythonnet in Unity3d ([#714][i714]), + related to unloading the Application Domain +- Fixed interop methods with Py_ssize_t. NetCoreApp 2.0 is more sensitive than net40 and requires this fix. ([#531][p531]) +- Fixed crash on exit of the Python interpreter if a python class + derived from a .NET class has a `__namespace__` or `__assembly__` + attribute ([#481][i481]) +- Fixed conversion of 'float' and 'double' values ([#486][i486]) +- Fixed 'clrmethod' for python 2 ([#492][i492]) +- Fixed double calling of constructor when deriving from .NET class ([#495][i495]) +- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) +- Fixed `LockRecursionException` when loading assemblies ([#627][i627]) +- Fixed errors breaking .NET Remoting on method invoke ([#276][i276]) +- Fixed PyObject.GetHashCode ([#676][i676]) +- Fix memory leaks due to spurious handle incrementation ([#691][i691]) +- Fix spurious assembly loading exceptions from private types ([#703][i703]) +- Fix inheritance of non-abstract base methods ([#755][i755]) + + +## [2.3.0][] - 2017-03-11 + +### Added + +- Added Code Coverage ([#345][p345]) +- Added `PySys_SetArgvEx` ([#347][p347]) +- Added XML Documentation ([#349][p349]) +- Added `Embedded_Tests` on AppVeyor ([#224][i224])([#353][p353]) +- Added `Embedded_Tests` on Travis ([#224][i224])([#391][p391]) +- Added PY3 settings to solution configuration-manager ([#346][p346]) +- Added `Slack` ([#384][p384])([#383][i383])([#386][p386]) +- Added function of passing an arbitrary .NET object as the value + of an attribute of `PyObject` ([#370][i370])([#373][p373]) +- Added `Coverity scan` ([#390][i390]) +- Added `bumpversion` for version control ([#319][i319])([#398][p398]) +- Added `tox` for local testing ([#345][p345]) +- Added `requirements.txt` +- Added to `PythonEngine` methods `Eval` and `Exec` ([#389][p389]) +- Added implementations of `ICustomMarshal` ([#407][p407]) +- Added docker images ([#322][i322]) +- Added hooks in `pyinstaller` and `cx_freeze` for `pythonnet` ([#66][i66]) + +### Changed + +- Refactored python `unittests` ([#329][p329]) +- Refactored python `setup.py` ([#337][p337]) +- Refactored remaining of Build Directives on `runtime.cs` ([#339][p339]) +- Refactored `Embedded_Tests` to make easier to write tests ([#369][p369]) +- Changed `unittests` to `pytest` ([#368][p368]) +- Upgraded NUnit framework from `2.6.3` to `3.5.0` ([#341][p341]) +- Downgraded NUnit framework from `3.5.0` to `2.6.4` ([#353][p353]) +- Upgraded NUnit framework from `2.6.4` to `3.6.0` ([#371][p371]) +- Unfroze Mono version on Travis ([#345][p345]) +- Changed `conda.recipe` build to only pull-requests ([#345][p345]) +- Combine `Py_DEBUG` and `PYTHON_WITH_PYDEBUG` flags ([#362][i362]) + +### Deprecated + +- Deprecated `RunString` ([#401][i401]) + +### Fixed + +- Fixed crash during Initialization ([#262][i262])([#343][p343]) +- Fixed crash during Shutdown ([#365][p365]) +- Fixed multiple build warnings +- Fixed method signature match for Object Type ([#203][i203])([#377][p377]) +- Fixed outdated version number in AssemblyInfo ([#398][p398]) +- Fixed wrong version number in `conda.recipe` ([#398][p398]) +- Fixed fixture location for Python tests and `Embedded_Tests` +- Fixed `PythonException` crash during Shutdown ([#400][p400]) +- Fixed `AppDomain` unload during GC ([#397][i397])([#400][p400]) +- Fixed `Py_Main` & `PySys_SetArgvEx` `no mem error` on `UCS4/PY3` ([#399][p399]) +- Fixed `Python.Runtime.dll.config` on macOS ([#120][i120]) +- Fixed crash on `PythonEngine.Version` ([#413][i413]) +- Fixed `PythonEngine.PythonPath` issues ([#179][i179])([#414][i414])([#415][p415]) +- Fixed missing information on 'No method matches given arguments' by adding the method name + +### Removed + +- Removed `six` dependency for `unittests` ([#329][p329]) +- Removed `Mono.Unix` dependency for `UCS4` ([#360][p360]) +- Removed need for `Python.Runtime.dll.config` +- Removed PY32 build option `PYTHON_WITH_WIDE_UNICODE` ([#417][i417]) + +## [2.2.2][] - 2017-01-29 + +### Fixed + +- Missing files from packaging ([#336][i336]) + +## [2.2.1][] - 2017-01-26 + +- `v2.2.0` had a release issue on PyPi. Bumped to `v2.2.1` + +### Added + +- Python 3.6 support ([#310][p310]) +- Added `__version__` to module ([#312][p312]) +- Added `conda` recipe ([#281][p281]) +- Nuget update on build ([#268][p268]) +- Added `__cause__` attribute on exception ([#287][p287]) + +### Changed + +- License to MIT ([#314][p314]) +- Project clean-up ([#320][p320]) +- Refactor `#if` directives +- Rename Decref/Incref to XDecref/XIncre ([#275][p275]) +- Remove printing if Decref is called with NULL ([#275][p275]) + +### Removed + +- Python 2.6 support ([#270][i270]) +- Python 3.2 support ([#270][i270]) + +### Fixed + +- Fixed `isinstance` refcount_leak ([#273][p273]) +- Comparison Operators ([#294][p294]) +- Improved Linux support ([#300][p300]) +- Exception pickling ([#286][p286]) + +## [2.2.0-dev1][] - 2016-09-19 + +### Changed + +- Switch to C# 6.0 ([#219][p219]) +- `setup.py` improvements for locating build tools ([#208][p208]) +- unmanaged exports updated ([#206][p206]) +- Mono update pinned to 4.2.4.4 ([#233][p233]) + +### Fixed + +- Fixed relative imports ([#219][p219]) +- Fixed recursive types ([#250][p250]) +- Demo fix - stream reading ([#225][p225]) + +## [2.1.0][] - 2016-04-12 + +### Added + +- Added Python 3.2 support. ([#78][p78]) +- Added Python 3.3 support. ([#78][p78]) +- Added Python 3.4 support. ([#78][p78]) +- Added Python 3.5 support. ([#163][p163]) +- Managed types can be sub-classed in Python ([#78][p78]) +- Uses dynamic objects for cleaner code when embedding Python ([#78][p78]) + +### Changed + +- Better Linux support (with or without --enable-shared option) ([#78][p78]) + +### Removed + +- Implicit Type Casting ([#131][i131]) + +## [2.0.0][] - 2015-06-26 + +- Release + +## 2.0.0-alpha.2 + +### Changed + +- First work on Python 2.5 compatibility. The destination version can be + set by defining PYTHON24 or PYTHON25. Python 2.6 compatibility is in + work. + +- Added VS 2005 solution and project files including a UnitTest + configuration which runs the unit test suite. + +- Enhanced unit test suite. All test cases are combined in a single + test suite now. + +- Fixed bugs in generics support for all Python versions. + +- Fixed exception bugs for Python 2.5+. When compiled for Python 2.5+ all + managed exceptions are based on Python's `exceptions.Exception` class. + +- Added deprecation warnings for importing from `CLR.*` and the CLR module. + +- Implemented support for methods with variable arguments + `spam(params object[] egg)` + +- Fixed Mono support by adding a custom marshaler for UCS-4 unicode, + fixing a some ref counter bugs and creating a new makefile.mono. + +- Added a standard python extension to load the clr environment. + The `src/monoclr/` directory contains additional sample code like a + Python binary linked against `libpython2.x.so` and some example code + how to embed Mono and PythonNet in a C application. + +- Added yet another python prompt. This time it's a C application that + embedds both Python and Mono. It may be useful as an example app for + others and I need it to debug a nasty bug. + +- Implemented `ModuleFunctionAttribute` and added + `ForbidPythonThreadsAttribute`. The latter is required for module + functions which invoke Python methods. + +- Added `clr.setPreload()`, `clr.getPreload()`, + `clr.AddReference("assembly name")`, `clr.FindAssembly("name")` + and `clr.ListAssemblies(verbose)`. Automatic preloading can be enabled + with clr.setPreload/True). Preloading is automatically enabled for + interactive Python shells and disabled in all other cases. + +- New Makefile that works for Windows and Mono and autodetects the Python + version and UCS 2/4 setting. + +- Added code for Python 2.3. PythonNet can be build for Python 2.3 again + but it is not fully supported. + +- Changed the PythonException.Message value so it displays the name of + the exception class `Exception` instead of its representation + ``. + +- Added `Python.Runtime.dll.config`. + +## 2.0.0-alpha.1 + +### Changed + +- Moved the Python for .NET project to Sourceforge and moved version + control to Subversion. + +- Removed `CallConvCdecl` attributes and the IL hack that they supported. + .NET 2.x now supports `UnmanagedFunctionPointer`, which does the right + thing without the hackery required in 1.x. This removes a dependency + on ILASM to build the package and better supports Mono (in theory). + +- Refactored import and assembly management machinery. The old `CLR.` + syntax for import is deprecated, but still supported until 3.x. The + recommended style now is to use `from System import xxx`, etc. We + also now support `from X import *` correctly. + +- Implemented a (lowercase) `clr` module to match IronPython for code + compatibility. Methods of this module should be used to explicitly + load assemblies. Implicit (name-based) assembly loading will still + work until 3.x, but it is deprecated. + +- Implemented support for generic types and generic methods using the + same patterns and syntax as IronPython. See the documentation for + usage details. + +- Many small and large performance improvements, switched to generic + collections for some internals, better algorithms for assembly + scanning, etc. + +- Fixed an unboxing issue in generated delegate implementation code + that affected delegates that return value types. + +## [1.0.0][] - 2006-04-08 + +### Changed + +- Backported the refactored import and assembly management from the 2.x + line, mainly to improve the possibility of code-compatibility with + IronPython. + +## 1.0.0-rc.2 + +### Changed + +- Changed some uses of Finalize as a static method name that confused the + Mono compiler and people reading the code. Note that this may be a + breaking change if anyone was calling `PythonEngine.Finalize()`. If so, + you should now use `PythonEngine.Shutdown()`. + +- Tweaked assembly lookup to ensure that assemblies can be found in the + current working directory, even after changing directories using things + like `os.chdir()` from Python. + +- Fixed some incorrect finalizers (thanks to Greg Chapman for the report) + that may have caused some threading oddities. + +- Tweaked support for out and ref parameters. If a method has a return + type of void and a single ref or out parameter, that parameter will be + returned as the result of the method. This matches the current behavior + of IronPython and makes it more likely that code can be moved between + Python for .NET and IP in the future. + +- Refactored part of the assembly manager to remove a potential case of + thread-deadlock in multi-threaded applications. + +- Added a `__str__` method to managed exceptions that returns the Message + attribute of the exception and the StackTrace (if available). + +## 1.0.0-rc.1 + +### Changed + +- Implemented a workaround for the fact that exceptions cannot be new-style + classes in the CPython interpreter. Managed exceptions can now be raised + and caught naturally from Python (hooray!) + +- Implemented support for invoking methods with out and ref parameters. + Because there is no real equivalent to these in Python, methods that + have out or ref parameters will return a tuple. The tuple will contain + the result of the method as its first item, followed by out parameter + values in the order of their declaration in the method signature. + +- Fixed a refcount problem that caused a crash when CLR was imported in + an existing installed Python interpreter. + +- Added an automatic conversion from Python strings to `byte[]`. This makes + it easier to pass `byte[]` data to managed methods (or set properties, + etc.) as a Python string without having to write explicit conversion + code. Also works for sbyte arrays. Note that `byte` and `sbyte` arrays + returned from managed methods or obtained from properties or fields + do _not_ get converted to Python strings - they remain instances of + `Byte[]` or `SByte[]`. + +- Added conversion of generic Python sequences to object arrays when + appropriate (thanks to Mackenzie Straight for the patch). + +- Added a bit of cautionary documentation for embedders, focused on + correct handling of the Python global interpreter lock from managed + code for code that calls into Python. + +- `PyObject.FromManagedObject` now correctly returns the Python None object + if the input is a null reference. Also added a new `AsManagedObject` + method to `PyObject`, making it easier to convert a Python-wrapped managed + object to the real managed object. + +- Created a simple installer for windows platforms. + +## 1.0.0-beta.5 + +### Changed + +- Refactored and fixed threading and global interpreter lock handling, + which was badly broken before. Also added a number of threading and + GIL-handling tests. + +- Related to the GIL fixes, added a note to embedders in the README + about using the AcquireLock and ReleaseLock methods of the PythonEngine + class to manage the GIL. + +- Fixed a problem in `Single <--> float` conversion for cultures that use + different decimal symbols than Python. + +- Added a new `ReloadModule` method to the `PythonEngine` class that hooks + Python module reloading (`PyImport_ReloadModule`). + +- Added a new `StringAsModule` method to the PythonEngine class that can + create a module from a managed string of code. + +- Added a default `__str__` implementation for Python wrappers of managed + objects that calls the `ToString` method of the managed object. + +## 1.0.0-beta.4 + +### Changed + +- Fixed a problem that made it impossible to override "special" methods + like `__getitem__` in subclasses of managed classes. Now the tests all + pass, and there is much rejoicing. + +- Managed classes reflected to Python now have an `__doc__` attribute that + contains a listing of the class constructor signatures. + +- Fixed a problem that prevented passing null (None) for array arguments. + +- Added a number of new argument conversion tests. Thanks to Laurent + Caumont for giving Python for .NET a good workout with managed DirectX. + +- Updated the bundled C Python runtime and libraries to Python 2.4. The + current release is known to also run with Python 2.3. It is known + _not_ to work with older versions due to changes in CPython type + object structure. + +- Mostly fixed the differences in the way that import works depending + on whether you are using the bundled interpreter or an existing Python + interpreter. The hack I used makes import work uniformly for imports + done in Python modules. Unfortunately, there is still a limitation + when using the interpreter interactively: you need to do `import CLR` + first before importing any sub-names when running with an existing + Python interpreter. + + The reason is that the first import of `CLR` installs the CLR import + hook, but for an existing interpreter the standard importer is still + in control for the duration of that first import, so sub-names won't + be found until the next import, which will use the now-installed hook. + +- Added support to directly iterate over objects that support IEnumerator + (as well as IEnumerable). Thanks to Greg Chapman for prodding me ;) + +- Added a section to the README dealing with rebuilding Python for .NET + against other CPython versions. + +- Fixed a problem with accessing properties when only the interface for + an object is known. For example, `ICollection(ob).Count` failed because + Python for .NET mistakenly decided that Count was abstract. + +- Fixed some problems with how COM-based objects are exposed and how + members of inherited interfaces are exposed. Thanks to Bruce Dodson + for patches on this. + +- Changed the Runtime class to use a const string to target the + appropriate CPython dll in DllImport attributes. Now you only + have to change one line to target a new Python version. + +## 1.0.0-beta.3 + +### Changed + +- A dumb bug that could cause a crash on startup on some platforms was + fixed. Decided to update the beta for this, as a number of people + were running into the problem. + +## 1.0.0-beta.2 + +### Changed + +- Exceptions raised as a result of getting or setting properties were + not very helpful (target invokation exception). This has been changed + to pass through the inner exception the way that methods do, which is + much more likely to be the real exception that caused the problem. + +- Events were refactored as the implementation was based on some bad + assumptions. As a result, subscription and unsubscription now works + correctly. A change from beta 1 is that event objects are no longer + directly callable - this was not appropriate, since the internal + implementation of an event is private and cant work reliably. Instead, + you should the appropriate `OnSomeEvent` method published by a class + to fire an event. + +- The distribution did not include the key file, making it a pain for + people to build from source. Added the key file to the distribution + buildout for beta 2. + +- Assemblies can now be found and loaded if they are on the PYTHONPATH. + Previously only the appbase and the GAC were checked. The system now + checks PYTHONPATH first, then the appbase, then the GAC. + +- Fixed a bug in constructor invokation during object instantiation. + +## 1.0.0-beta.1 + +### Changed + +- Added the baseline of the managed embedding API. Some of the details + are still subject to change based on some real-world use and feedback. + + The embedding API is based on the `PyObject` class, along with a number + of specific `PyDict`, `PyList`, (etc.) classes that expose the respective + interfaces of the built-in Python types. The basic structure and usage + is intended be familar to anyone who has used Python / C++ wrapper + libraries like CXX or Boost. + +- Started integrating NUnit2 to support unit tests for the embedding + layer - still need to add the embedding tests (many already exist, + but were written for an older version of NUnit). + +- Added Python iteration protocol support for arrays and managed objects + that implement IEnumerable. This means that you can now use the Python + idiom `for item in object:` on any array or IEnumerable object. + +- Added automatic conversion from Python sequence types to managed array + types. This means, for example, that you can now call a managed method + like AddRange that expects an array with any Python object that supports + the Python sequence protocol, provided the items of the sequence are + convertible to the item type of the managed array. + +- Added new demo scripts, mostly more substantial winforms examples. + +- Finished the unit tests for event support, and fixed lots of problems + with events and delegates as a result. This is one of the trickier + parts of the integration layer, and there is good coverage of these + in the unit tests now. + +- Did a fair amount of profiling with an eval version of ANTS (which is + quite nice, BTW) and made a few changes as a result. + +- Type management was refactored, fixing the issue that caused segfaults + when GC was enabled. Unit tests, stress tests and demo apps now all run + nicely with Python GC enabled. There are one or two things left to fix, + but the fixes should not have any user impact. + +- Changed to base PythonNet on Python 2.3.2. This is considered the most + stable release, and a good 25 - 30% faster as well. + +- Added a new `CLR.dll` that acts as an extension module that allows an + existing unmodified Python 2.3 installation to simply `import CLR` to + bootstrap the managed integration layer. + +- A bug was causing managed methods to only expose overloads declared in + a particular class, hiding inherited overloads of the same name. Fixed + the bug and added some unit tests. + +- Added a virtual `__doc__` attribute to managed methods that contains + the signature of the method. This also means that the Python `help` + function now provides signature info when used on a managed class. + +- Calling managed methods and events `unbound` (passing the instance as + the first argument) now works. There is a caveat for methods - if a + class declares both static and instance methods with the same name, + it is not possible to call that instance method unbound (the static + method will always be called). + +- Overload selection for overloaded methods is now much better and uses + a method resolution algorithm similar to that used by Jython. + +- Changed the managed python.exe wrapper to run as an STA thread, which + seems to be more compatible with winforms apps. This needs a better + solution long-term. One possibility would be a command line switch + so that -sta or -mta could control the python.exe apartment state. + +- Added support for the Python boolean type (True, False). Bool values + now appear as True or False to Python. + +## 1.0.0-alpha.2 + +### Changed + +- Added a Mono makefile. Thanks to Camilo Uribe for help in testing and + working out problems on Mono. Note that it not currently possible to + build PythonNet using mono, due to the use of some IL attributes that + the mono assembler / disassembler doesn't support yet. + +- Preliminary tests show that PythonNet _does_ actually run under mono, + though the test suite bombs out before the end with an "out of memory" + error from the mono runtime. It's just a guess at this point, but I + suspect there may be a limited pool for allocating certain reflection + structures, and Python uses the reflection infrastructure quite heavily. + +- Removed decoys like the non-working embedding APIs; lots of internal + refactoring. + +- Implemented indexer support. Managed instances that implement indexers + can now be used naturally from Python (e.g. `someobject[0]`). + +- Implemented sequence protocol support for managed arrays. + +- Implemented basic thread state management; calls to managed methods + no longer block Python. I won't go so far as to say the thread + choreography is "finished", as I don't have a comprehensive set of + tests to back that up yet (and it will take some work to write a + sufficiently large and evil set of tests). + +- Fixed a bug that caused conversions of managed strings to PyUnicode to + produce mangled values in certain situations. + +- Fixed a number of problems related to subclassing a managed class, + including the fact that it didn't work :) + +- Fixed all of the bugs that were causing tests to fail. This release + contains all new bugs and new failing tests. Progress! :) + +## 1.0.0-alpha.1 + +### Added + +- Initial (mostly) working experimental release. + +[keep a changelog]: http://keepachangelog.com/ + +[semantic versioning]: http://semver.org/ + +[unreleased]: ../../compare/v2.3.0...HEAD + +[2.3.0]: ../../compare/v2.2.2...v2.3.0 + +[2.2.2]: ../../compare/v2.2.1...v2.2.2 + +[2.2.1]: ../../compare/v2.2.0-dev1...v2.2.1 + +[2.2.0-dev1]: ../../compare/v2.1.0...v2.2.0-dev1 + +[2.1.0]: ../../compare/v2.0.0...v2.1.0 + +[2.0.0]: ../../compare/1.0...v2.0.0 + +[1.0.0]: https://github.com/pythonnet/pythonnet/releases/tag/1.0 + +[i714]: https://github.com/pythonnet/pythonnet/issues/714 +[i608]: https://github.com/pythonnet/pythonnet/issues/608 +[i443]: https://github.com/pythonnet/pythonnet/issues/443 +[p690]: https://github.com/pythonnet/pythonnet/pull/690 +[i475]: https://github.com/pythonnet/pythonnet/issues/475 +[p693]: https://github.com/pythonnet/pythonnet/pull/693 +[i432]: https://github.com/pythonnet/pythonnet/issues/432 +[p433]: https://github.com/pythonnet/pythonnet/pull/433 +[p460]: https://github.com/pythonnet/pythonnet/pull/460 +[p461]: https://github.com/pythonnet/pythonnet/pull/461 +[p433]: https://github.com/pythonnet/pythonnet/pull/433 +[i434]: https://github.com/pythonnet/pythonnet/issues/434 +[i481]: https://github.com/pythonnet/pythonnet/issues/481 +[i486]: https://github.com/pythonnet/pythonnet/issues/486 +[i492]: https://github.com/pythonnet/pythonnet/issues/492 +[i495]: https://github.com/pythonnet/pythonnet/issues/495 +[p607]: https://github.com/pythonnet/pythonnet/pull/607 +[i627]: https://github.com/pythonnet/pythonnet/issues/627 +[i276]: https://github.com/pythonnet/pythonnet/issues/276 +[i676]: https://github.com/pythonnet/pythonnet/issues/676 +[p345]: https://github.com/pythonnet/pythonnet/pull/345 +[p347]: https://github.com/pythonnet/pythonnet/pull/347 +[p349]: https://github.com/pythonnet/pythonnet/pull/349 +[i224]: https://github.com/pythonnet/pythonnet/issues/224 +[p353]: https://github.com/pythonnet/pythonnet/pull/353 +[p391]: https://github.com/pythonnet/pythonnet/pull/391 +[p346]: https://github.com/pythonnet/pythonnet/pull/346 +[p384]: https://github.com/pythonnet/pythonnet/pull/384 +[i383]: https://github.com/pythonnet/pythonnet/issues/383 +[p386]: https://github.com/pythonnet/pythonnet/pull/386 +[i370]: https://github.com/pythonnet/pythonnet/issues/370 +[p373]: https://github.com/pythonnet/pythonnet/pull/373 +[i390]: https://github.com/pythonnet/pythonnet/issues/390 +[i319]: https://github.com/pythonnet/pythonnet/issues/319 +[p398]: https://github.com/pythonnet/pythonnet/pull/398 +[p345]: https://github.com/pythonnet/pythonnet/pull/345 +[p389]: https://github.com/pythonnet/pythonnet/pull/389 +[p407]: https://github.com/pythonnet/pythonnet/pull/407 +[i322]: https://github.com/pythonnet/pythonnet/issues/322 +[i66]: https://github.com/pythonnet/pythonnet/issues/66 +[p329]: https://github.com/pythonnet/pythonnet/pull/329 +[p337]: https://github.com/pythonnet/pythonnet/pull/337 +[p339]: https://github.com/pythonnet/pythonnet/pull/339 +[p369]: https://github.com/pythonnet/pythonnet/pull/369 +[p368]: https://github.com/pythonnet/pythonnet/pull/368 +[p341]: https://github.com/pythonnet/pythonnet/pull/341 +[p353]: https://github.com/pythonnet/pythonnet/pull/353 +[p371]: https://github.com/pythonnet/pythonnet/pull/371 +[p345]: https://github.com/pythonnet/pythonnet/pull/345 +[i362]: https://github.com/pythonnet/pythonnet/issues/362 +[i401]: https://github.com/pythonnet/pythonnet/issues/401 +[i262]: https://github.com/pythonnet/pythonnet/issues/262 +[p343]: https://github.com/pythonnet/pythonnet/pull/343 +[p365]: https://github.com/pythonnet/pythonnet/pull/365 +[i203]: https://github.com/pythonnet/pythonnet/issues/203 +[p377]: https://github.com/pythonnet/pythonnet/pull/377 +[p398]: https://github.com/pythonnet/pythonnet/pull/398 +[p400]: https://github.com/pythonnet/pythonnet/pull/400 +[i397]: https://github.com/pythonnet/pythonnet/issues/397 +[p399]: https://github.com/pythonnet/pythonnet/pull/399 +[i120]: https://github.com/pythonnet/pythonnet/issues/120 +[i413]: https://github.com/pythonnet/pythonnet/issues/413 +[i179]: https://github.com/pythonnet/pythonnet/issues/179 +[i414]: https://github.com/pythonnet/pythonnet/issues/414 +[p415]: https://github.com/pythonnet/pythonnet/pull/415 +[p329]: https://github.com/pythonnet/pythonnet/pull/329 +[p360]: https://github.com/pythonnet/pythonnet/pull/360 +[i417]: https://github.com/pythonnet/pythonnet/issues/417 +[i336]: https://github.com/pythonnet/pythonnet/issues/336 +[p310]: https://github.com/pythonnet/pythonnet/pull/310 +[p312]: https://github.com/pythonnet/pythonnet/pull/312 +[p281]: https://github.com/pythonnet/pythonnet/pull/281 +[p268]: https://github.com/pythonnet/pythonnet/pull/268 +[p287]: https://github.com/pythonnet/pythonnet/pull/287 +[p314]: https://github.com/pythonnet/pythonnet/pull/314 +[p320]: https://github.com/pythonnet/pythonnet/pull/320 +[p275]: https://github.com/pythonnet/pythonnet/pull/275 +[i270]: https://github.com/pythonnet/pythonnet/issues/270 +[p273]: https://github.com/pythonnet/pythonnet/pull/273 +[p294]: https://github.com/pythonnet/pythonnet/pull/294 +[p300]: https://github.com/pythonnet/pythonnet/pull/300 +[p286]: https://github.com/pythonnet/pythonnet/pull/286 +[p219]: https://github.com/pythonnet/pythonnet/pull/219 +[p208]: https://github.com/pythonnet/pythonnet/pull/208 +[p206]: https://github.com/pythonnet/pythonnet/pull/206 +[p233]: https://github.com/pythonnet/pythonnet/pull/233 +[p219]: https://github.com/pythonnet/pythonnet/pull/219 +[p250]: https://github.com/pythonnet/pythonnet/pull/250 +[p225]: https://github.com/pythonnet/pythonnet/pull/225 +[p78]: https://github.com/pythonnet/pythonnet/pull/78 +[p163]: https://github.com/pythonnet/pythonnet/pull/163 +[p625]: https://github.com/pythonnet/pythonnet/pull/625 +[i131]: https://github.com/pythonnet/pythonnet/issues/131 +[p531]: https://github.com/pythonnet/pythonnet/pull/531 +[i755]: https://github.com/pythonnet/pythonnet/pull/755 +[p534]: https://github.com/pythonnet/pythonnet/pull/534 +[i449]: https://github.com/pythonnet/pythonnet/issues/449 +[i1342]: https://github.com/pythonnet/pythonnet/issues/1342 +[i238]: https://github.com/pythonnet/pythonnet/issues/238 +[i1481]: https://github.com/pythonnet/pythonnet/issues/1481 +[i1672]: https://github.com/pythonnet/pythonnet/pull/1672 +# Changelog + +All notable changes to Python.NET will be documented in this file. This +project adheres to [Semantic Versioning][]. + +This document follows the conventions laid out in [Keep a CHANGELOG][]. + ## Unreleased ### Added diff --git a/pyproject.toml b/pyproject.toml index 4ece5f3a4..bff6aac05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,64 @@ license = {text = "MIT"} readme = "README.rst" +dependencies = [ + "clr_loader>=0.2.2,<0.3.0" +] + +requires-python = ">=3.7, <3.12" + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: C#", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", +] + +dynamic = ["version"] + +[[project.authors]] +name = "The Contributors of the Python.NET Project" +email = "pythonnet@python.org" + +[project.urls] +Homepage = "https://pythonnet.github.io/" +Sources = "https://github.com/pythonnet/pythonnet" + +[tool.setuptools] +zip-safe = false +py-modules = ["clr"] + +[tool.setuptools.dynamic.version] +file = "version.txt" + +[tool.setuptools.packages.find] +include = ["pythonnet*"] + +[tool.pytest.ini_options] +xfail_strict = true +testpaths = [ + "tests" +] +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "pythonnet" +description = ".NET and Mono integration for Python" +license = {text = "MIT"} + +readme = "README.rst" + dependencies = [ "clr_loader>=0.2.6,<0.3.0" ] diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index c8b8ecb6e..9e90e89c9 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -348,6 +348,530 @@ import clr from datetime import datetime +from Python.EmbeddingTest import Codecs, DateTimeDecoder + +DateTimeDecoder.Setup() +"); + scope.Exec("Codecs.AcceptsDateTime(datetime(2021, 1, 22))"); + } + + [Test] + public void FloatDerivedDecoded() + { + using var scope = Py.CreateScope(); + scope.Exec(@"class FloatDerived(float): pass"); + using var floatDerived = scope.Eval("FloatDerived"); + var decoder = new DecoderReturningPredefinedValue(floatDerived, 42); + PyObjectConversions.RegisterDecoder(decoder); + using var result = scope.Eval("FloatDerived()"); + object decoded = result.As(); + Assert.AreEqual(42, decoded); + } + + [Test] + public void ExceptionDecodedNoInstance() + { + PyObjectConversions.RegisterDecoder(new InstancelessExceptionDecoder()); + using var scope = Py.CreateScope(); + var error = Assert.Throws(() => PythonEngine.Exec( + $"[].__iter__().__next__()")); + Assert.AreEqual(TestExceptionMessage, error.Message); + } + + public static void AcceptsDateTime(DateTime v) {} + + [Test] + public void As_Object_AffectedByDecoders() + { + var everythingElseToSelf = new EverythingElseToSelfDecoder(); + PyObjectConversions.RegisterDecoder(everythingElseToSelf); + + var pyObj = PythonEngine.Eval("iter"); + var decoded = pyObj.As(); + Assert.AreSame(everythingElseToSelf, decoded); + } + + public class EverythingElseToSelfDecoder : IPyObjectDecoder + { + public bool CanDecode(PyType objectType, Type targetType) + { + return targetType.IsAssignableFrom(typeof(EverythingElseToSelfDecoder)); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + value = (T)(object)this; + return true; + } + } + + class ValueErrorWrapper : Exception + { + public ValueErrorWrapper(string message) : base(message) { } + } + + class ValueErrorCodec : IPyObjectEncoder, IPyObjectDecoder + { + public bool CanDecode(PyType objectType, Type targetType) + => this.CanEncode(targetType) + && PythonReferenceComparer.Instance.Equals(objectType, PythonEngine.Eval("ValueError")); + + public bool CanEncode(Type type) => type == typeof(ValueErrorWrapper) + || typeof(ValueErrorWrapper).IsSubclassOf(type); + + public bool TryDecode(PyObject pyObj, out T value) + { + var message = pyObj.GetAttr("args")[0].As(); + value = (T)(object)new ValueErrorWrapper(message); + return true; + } + + public PyObject TryEncode(object value) + { + var error = (ValueErrorWrapper)value; + return PythonEngine.Eval("ValueError").Invoke(error.Message.ToPython()); + } + } + + class InstancelessExceptionDecoder : IPyObjectDecoder, IDisposable + { + readonly PyObject PyErr = Py.Import("clr.interop").GetAttr("PyErr"); + + public bool CanDecode(PyType objectType, Type targetType) + => PythonReferenceComparer.Instance.Equals(PyErr, objectType); + + public void Dispose() + { + PyErr.Dispose(); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj.HasAttr("value")) + { + value = default; + return false; + } + + value = (T)(object)new ValueErrorWrapper(TestExceptionMessage); + return true; + } + } + } + + /// + /// "Decodes" only objects of exact type . + /// Result is just the raw proxy to the encoder instance itself. + /// + class ObjectToEncoderInstanceEncoder : IPyObjectEncoder + { + public bool CanEncode(Type type) => type == typeof(T); + public PyObject TryEncode(object value) => PyObject.FromManagedObject(this); + } + + /// + /// Decodes object of specified Python type to the predefined value + /// + /// Type of the + class DecoderReturningPredefinedValue : IPyObjectDecoder + { + public PyObject TheOnlySupportedSourceType { get; } + public TTarget DecodeResult { get; } + + public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult) + { + this.TheOnlySupportedSourceType = objectType; + this.DecodeResult = decodeResult; + } + + public bool CanDecode(PyType objectType, Type targetType) + => PythonReferenceComparer.Instance.Equals(objectType, TheOnlySupportedSourceType) + && targetType == typeof(TTarget); + public bool TryDecode(PyObject pyObj, out T value) + { + if (typeof(T) != typeof(TTarget)) + throw new ArgumentException(nameof(T)); + value = (T)(object)DecodeResult; + return true; + } + } + + public class DateTimeDecoder : IPyObjectDecoder + { + public static void Setup() + { + PyObjectConversions.RegisterDecoder(new DateTimeDecoder()); + } + + public bool CanDecode(PyType objectType, Type targetType) + { + return targetType == typeof(DateTime); + } + + public bool TryDecode(PyObject pyObj, out T value) + { + var dt = new DateTime( + pyObj.GetAttr("year").As(), + pyObj.GetAttr("month").As(), + pyObj.GetAttr("day").As(), + pyObj.GetAttr("hour").As(), + pyObj.GetAttr("minute").As(), + pyObj.GetAttr("second").As()); + value = (T)(object)dt; + return true; + } + } +} +namespace Python.EmbeddingTest { + using System; + using System.Collections.Generic; + using System.Linq; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class Codecs + { + [SetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void TupleConversionsGeneric() + { + TupleConversionsGeneric, ValueTuple>(); + } + + static void TupleConversionsGeneric() + { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (var scope = Py.CreateScope()) + { + void Accept(T value) => restored = value; + using var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleConversionsObject() + { + TupleConversionsObject, ValueTuple>(); + } + static void TupleConversionsObject() + { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42.0, "42", new object()); + T restored = default; + using (var scope = Py.CreateScope()) + { + void Accept(object value) => restored = (T)value; + using var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripObject() + { + TupleRoundtripObject, ValueTuple>(); + } + static void TupleRoundtripObject() + { + var tuple = Activator.CreateInstance(typeof(T), 42.0, "42", new object()); + using var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + + [Test] + public void TupleRoundtripGeneric() + { + TupleRoundtripGeneric, ValueTuple>(); + } + + static void TupleRoundtripGeneric() + { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + + static PyObject GetPythonIterable() => PythonEngine.Eval("map(lambda x: x, [1,2,3])"); + + [Test] + public void ListDecoderTest() + { + var codec = ListDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + using var pyList = new PyList(items.ToArray()); + + using var pyListType = pyList.GetPythonType(); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); + + //we'd have to copy into a list instance to do this, it would not be lossless. + //lossy converters can be implemented outside of the python.net core library + Assert.IsFalse(codec.CanDecode(pyListType, typeof(List))); + + //convert to list of int + IList intList = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intList); }); + CollectionAssert.AreEqual(intList, new List { 1, 2, 3 }); + + //convert to list of string. This will not work. + //The ListWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python list can be queried without any conversion, + //the IList will report a Count of 3. + IList stringList = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringList); }); + Assert.AreEqual(stringList.Count, 3); + Assert.Throws(typeof(InvalidCastException), () => { var x = stringList[0]; }); + + //can't convert python iterable to list (this will require a copy which isn't lossless) + using var foo = GetPythonIterable(); + using var fooType = foo.GetPythonType(); + Assert.IsFalse(codec.CanDecode(fooType, typeof(IList))); + } + + [Test] + public void SequenceDecoderTest() + { + var codec = SequenceDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + //SequenceConverter can only convert to any ICollection + using var pyList = new PyList(items.ToArray()); + using var listType = pyList.GetPythonType(); + //it can convert a PyList, since PyList satisfies the python sequence protocol + + Assert.IsFalse(codec.CanDecode(listType, typeof(bool))); + Assert.IsFalse(codec.CanDecode(listType, typeof(IList))); + Assert.IsFalse(codec.CanDecode(listType, typeof(System.Collections.IEnumerable))); + Assert.IsFalse(codec.CanDecode(listType, typeof(IEnumerable))); + + Assert.IsTrue(codec.CanDecode(listType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(listType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(listType, typeof(ICollection))); + + //convert to collection of int + ICollection intCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intCollection); }); + CollectionAssert.AreEqual(intCollection, new List { 1, 2, 3 }); + + //no python exception should have occurred during the above conversion and check + Runtime.CheckExceptionOccurred(); + + //convert to collection of string. This will not work. + //The SequenceWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python sequence can be queried without any conversion, + //the IList will report a Count of 3. + ICollection stringCollection = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringCollection); }); + Assert.AreEqual(3, stringCollection.Count()); + Assert.Throws(typeof(InvalidCastException), () => { + string[] array = new string[3]; + stringCollection.CopyTo(array, 0); + }); + + Runtime.CheckExceptionOccurred(); + + //can't convert python iterable to collection (this will require a copy which isn't lossless) + //python iterables do not satisfy the python sequence protocol + var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); + Assert.IsFalse(codec.CanDecode(fooType, typeof(ICollection))); + + //python tuples do satisfy the python sequence protocol + var pyTuple = new PyTuple(items.ToArray()); + var pyTupleType = pyTuple.GetPythonType(); + + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + Assert.IsTrue(codec.CanDecode(pyTupleType, typeof(ICollection))); + + //convert to collection of int + ICollection intCollection2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out intCollection2); }); + CollectionAssert.AreEqual(intCollection2, new List { 1, 2, 3 }); + + //no python exception should have occurred during the above conversion and check + Runtime.CheckExceptionOccurred(); + + //convert to collection of string. This will not work. + //The SequenceWrapper class will throw a python exception when it tries to access any element. + //TryDecode is a lossless conversion so there will be no exception at that point + //interestingly, since the size of the python sequence can be queried without any conversion, + //the IList will report a Count of 3. + ICollection stringCollection2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyTuple, out stringCollection2); }); + Assert.AreEqual(3, stringCollection2.Count()); + Assert.Throws(typeof(InvalidCastException), () => { + string[] array = new string[3]; + stringCollection2.CopyTo(array, 0); + }); + + Runtime.CheckExceptionOccurred(); + + } + + [Test] + public void IterableDecoderTest() + { + var codec = IterableDecoder.Instance; + var items = new List() { new PyInt(1), new PyInt(2), new PyInt(3) }; + + var pyList = new PyList(items.ToArray()); + var pyListType = pyList.GetPythonType(); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(IList))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(System.Collections.IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(ICollection))); + Assert.IsFalse(codec.CanDecode(pyListType, typeof(bool))); + + //ensure a PyList can be converted to a plain IEnumerable + System.Collections.IEnumerable plainEnumerable1 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable1); }); + CollectionAssert.AreEqual(plainEnumerable1.Cast().Select(i => i.ToInt32()), new List { 1, 2, 3 }); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will lead to an empty iterable when decoding. TODO - should it throw? + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(pyListType, typeof(IEnumerable))); + + IEnumerable intEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); + CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); + + Runtime.CheckExceptionOccurred(); + + IEnumerable doubleEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out doubleEnumerable); }); + CollectionAssert.AreEqual(doubleEnumerable, new List { 1, 2, 3 }); + + Runtime.CheckExceptionOccurred(); + + IEnumerable stringEnumerable = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out stringEnumerable); }); + + Assert.Throws(typeof(InvalidCastException), () => { + foreach (string item in stringEnumerable) + { + var x = item; + } + }); + Assert.Throws(typeof(InvalidCastException), () => { + stringEnumerable.Count(); + }); + + Runtime.CheckExceptionOccurred(); + + //ensure a python class which implements the iterator protocol can be converter to a plain IEnumerable + var foo = GetPythonIterable(); + var fooType = foo.GetPythonType(); + System.Collections.IEnumerable plainEnumerable2 = null; + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out plainEnumerable2); }); + CollectionAssert.AreEqual(plainEnumerable2.Cast().Select(i => i.ToInt32()), new List { 1, 2, 3 }); + + //can convert to any generic ienumerable. If the type is not assignable from the python element + //it will be an exception during TryDecode + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + Assert.IsTrue(codec.CanDecode(fooType, typeof(IEnumerable))); + + Assert.DoesNotThrow(() => { codec.TryDecode(pyList, out intEnumerable); }); + CollectionAssert.AreEqual(intEnumerable, new List { 1, 2, 3 }); + } + + // regression for https://github.com/pythonnet/pythonnet/issues/1427 + [Test] + public void PythonRegisteredDecoder_NoStackOverflowOnSystemType() + { + const string PyCode = @" +import clr +import System +from Python.Runtime import PyObjectConversions +from Python.Runtime.Codecs import RawProxyEncoder + + +class ListAsRawEncoder(RawProxyEncoder): + __namespace__ = 'Dummy' + def CanEncode(self, clr_type): + return clr_type.Name == 'IList`1' and clr_type.Namespace == 'System.Collections.Generic' + + +list_encoder = ListAsRawEncoder() +PyObjectConversions.RegisterEncoder(list_encoder) + +system_type = list_encoder.GetType()"; + + PythonEngine.Exec(PyCode); + } + + const string TestExceptionMessage = "Hello World!"; + [Test] + public void ExceptionEncoded() + { + PyObjectConversions.RegisterEncoder(new ValueErrorCodec()); + void CallMe() => throw new ValueErrorWrapper(TestExceptionMessage); + var callMeAction = new Action(CallMe); + using var scope = Py.CreateScope(); + scope.Exec(@" +def call(func): + try: + func() + except ValueError as e: + return str(e) +"); + var callFunc = scope.Get("call"); + string message = callFunc.Invoke(callMeAction.ToPython()).As(); + Assert.AreEqual(TestExceptionMessage, message); + } + + [Test] + public void ExceptionDecoded() + { + PyObjectConversions.RegisterDecoder(new ValueErrorCodec()); + using var scope = Py.CreateScope(); + var error = Assert.Throws(() + => PythonEngine.Exec($"raise ValueError('{TestExceptionMessage}')")); + Assert.AreEqual(TestExceptionMessage, error.Message); + } + + [Test] + public void DateTimeDecoded() + { + using var scope = Py.CreateScope(); + scope.Exec(@" +import clr +from datetime import datetime + + from Python.EmbeddingTest import Codecs, DateTimeDecoder DateTimeDecoder.Setup() diff --git a/src/embed_tests/Events.cs b/src/embed_tests/Events.cs index c216f4214..15842e33e 100644 --- a/src/embed_tests/Events.cs +++ b/src/embed_tests/Events.cs @@ -1,67 +1,67 @@ -using System; -using System.Diagnostics; -using System.Threading; - -using NUnit.Framework; - -using Python.Runtime; - -namespace Python.EmbeddingTest; - -public class Events -{ - [OneTimeSetUp] - public void SetUp() - { - PythonEngine.Initialize(); - } - - [OneTimeTearDown] - public void Dispose() - { - PythonEngine.Shutdown(); - } - - [Test] - public void UsingDoesNotLeak() - { - using var scope = Py.CreateScope(); - scope.Exec(@" -import gc - -from Python.EmbeddingTest import ClassWithEventHandler - -def event_handler(): - pass - -for _ in range(2000): - example = ClassWithEventHandler() - example.LeakEvent += event_handler - example.LeakEvent -= event_handler - del example - -gc.collect() -"); - Runtime.Runtime.TryCollectingGarbage(10); - Assert.AreEqual(0, ClassWithEventHandler.alive); - } -} - -public class ClassWithEventHandler -{ - internal static int alive; - - public event EventHandler LeakEvent; - private Array arr; // dummy array to exacerbate memory leak - - public ClassWithEventHandler() - { - Interlocked.Increment(ref alive); - this.arr = new int[800]; - } - - ~ClassWithEventHandler() - { - Interlocked.Decrement(ref alive); - } -} +using System; +using System.Diagnostics; +using System.Threading; + +using NUnit.Framework; + +using Python.Runtime; + +namespace Python.EmbeddingTest; + +public class Events +{ + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void UsingDoesNotLeak() + { + using var scope = Py.CreateScope(); + scope.Exec(@" +import gc + +from Python.EmbeddingTest import ClassWithEventHandler + +def event_handler(): + pass + +for _ in range(2000): + example = ClassWithEventHandler() + example.LeakEvent += event_handler + example.LeakEvent -= event_handler + del example + +gc.collect() +"); + Runtime.Runtime.TryCollectingGarbage(10); + Assert.AreEqual(0, ClassWithEventHandler.alive); + } +} + +public class ClassWithEventHandler +{ + internal static int alive; + + public event EventHandler LeakEvent; + private Array arr; // dummy array to exacerbate memory leak + + public ClassWithEventHandler() + { + Interlocked.Increment(ref alive); + this.arr = new int[800]; + } + + ~ClassWithEventHandler() + { + Interlocked.Decrement(ref alive); + } +} diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index d2767e664..39462d6b0 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -210,6 +210,230 @@ public void ToBigInteger() CollectionAssert.AreEqual(expected, actual); } + [Test] + public void ToBigIntegerLarge() + { + BigInteger val = BigInteger.Pow(2, 1024) + 3; + var pyInt = new PyInt(val); + Assert.AreEqual(val, pyInt.ToBigInteger()); + val = -val; + pyInt = new PyInt(val); + Assert.AreEqual(val, pyInt.ToBigInteger()); + } + } +} +using System; +using System.Globalization; +using System.Linq; +using System.Numerics; + +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestPyInt + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void TestCtorInt() + { + const int i = 5; + var a = new PyInt(i); + Assert.AreEqual(i, a.ToInt32()); + } + + [Test] + public void TestCtorUInt() + { + const uint i = 5; + var a = new PyInt(i); + Assert.AreEqual(i, a.ToInt32()); + } + + [Test] + public void TestCtorLong() + { + const long i = 5; + var a = new PyInt(i); + Assert.AreEqual(i, a.ToInt32()); + } + + [Test] + public void TestCtorULong() + { + const ulong i = 5; + var a = new PyInt(i); + Assert.AreEqual(i, a.ToInt32()); + } + + [Test] + public void TestCtorShort() + { + const short i = 5; + var a = new PyInt(i); + Assert.AreEqual(i, a.ToInt32()); + } + + [Test] + public void TestCtorUShort() + { + const ushort i = 5; + var a = new PyInt(i); + Assert.AreEqual(i, a.ToInt32()); + } + + [Test] + public void TestCtorByte() + { + const byte i = 5; + var a = new PyInt(i); + Assert.AreEqual(i, a.ToInt32()); + } + + [Test] + public void TestCtorSByte() + { + const sbyte i = 5; + var a = new PyInt(i); + Assert.AreEqual(i, a.ToInt32()); + } + + [Test] + public void TestCtorPyObject() + { + var i = new PyInt(5); + var a = new PyInt(i); + Assert.AreEqual(5, a.ToInt32()); + } + + [Test] + public void TestCtorBadPyObject() + { + var i = new PyString("Foo"); + PyInt a = null; + + var ex = Assert.Throws(() => a = new PyInt(i)); + + StringAssert.StartsWith("object is not an int", ex.Message); + Assert.IsNull(a); + } + + [Test] + public void TestCtorString() + { + const string i = "5"; + var a = new PyInt(i); + Assert.AreEqual(5, a.ToInt32()); + } + + [Test] + public void TestCtorBadString() + { + const string i = "Foo"; + PyInt a = null; + + var ex = Assert.Throws(() => a = new PyInt(i)); + + StringAssert.StartsWith("invalid literal for int", ex.Message); + Assert.IsNull(a); + } + + [Test] + public void TestIsIntTypeTrue() + { + var i = new PyInt(5); + Assert.True(PyInt.IsIntType(i)); + } + + [Test] + public void TestIsIntTypeFalse() + { + var s = new PyString("Foo"); + Assert.False(PyInt.IsIntType(s)); + } + + [Test] + public void TestAsIntGood() + { + var i = new PyInt(5); + var a = PyInt.AsInt(i); + Assert.AreEqual(5, a.ToInt32()); + } + + [Test] + public void TestAsIntBad() + { + var s = new PyString("Foo"); + PyInt a = null; + + var ex = Assert.Throws(() => a = PyInt.AsInt(s)); + StringAssert.StartsWith("invalid literal for int", ex.Message); + Assert.IsNull(a); + } + + [Test] + public void TestConvertToInt32() + { + var a = new PyInt(5); + Assert.IsInstanceOf(typeof(int), a.ToInt32()); + Assert.AreEqual(5, a.ToInt32()); + } + + [Test] + public void TestConvertToInt16() + { + var a = new PyInt(5); + Assert.IsInstanceOf(typeof(short), a.ToInt16()); + Assert.AreEqual(5, a.ToInt16()); + } + + [Test] + public void TestConvertToInt64() + { + long val = 5 + (long)int.MaxValue; + var a = new PyInt(val); + Assert.IsInstanceOf(typeof(long), a.ToInt64()); + Assert.AreEqual(val, a.ToInt64()); + } + + [Test] + public void ToBigInteger() + { + int[] simpleValues = + { + 0, 1, 2, + 0x10, + 0x79, + 0x80, + 0x81, + 0xFF, + 0x123, + 0x8000, + 0x1234, + 0x8001, + 0x4000, + 0xFF, + }; + simpleValues = simpleValues.Concat(simpleValues.Select(v => -v)).ToArray(); + + var expected = simpleValues.Select(v => new BigInteger(v)).ToArray(); + var actual = simpleValues.Select(v => new PyInt(v).ToBigInteger()).ToArray(); + + CollectionAssert.AreEqual(expected, actual); + } + [Test] public void CompareTo() { diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index be91d7f45..df5f5acde 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -1,234 +1,234 @@ -using System; -using NUnit.Framework; -using Python.Runtime; - -namespace Python.EmbeddingTest -{ - public class TestPythonEngineProperties - { - [Test] - public static void GetBuildinfoDoesntCrash() - { - PythonEngine.Initialize(); - using (Py.GIL()) - { - string s = PythonEngine.BuildInfo; - - Assert.True(s.Length > 5); - Assert.True(s.Contains(",")); - } - } - - [Test] - public static void GetCompilerDoesntCrash() - { - PythonEngine.Initialize(); - using (Py.GIL()) - { - string s = PythonEngine.Compiler; - - Assert.True(s.Length > 0); - Assert.True(s.Contains("[")); - Assert.True(s.Contains("]")); - } - } - - [Test] - public static void GetCopyrightDoesntCrash() - { - PythonEngine.Initialize(); - using (Py.GIL()) - { - string s = PythonEngine.Copyright; - - Assert.True(s.Length > 0); - Assert.True(s.Contains("Python Software Foundation")); - } - } - - [Test] - public static void GetPlatformDoesntCrash() - { - PythonEngine.Initialize(); - using (Py.GIL()) - { - string s = PythonEngine.Platform; - - Assert.True(s.Length > 0); - Assert.True(s.Contains("x") || s.Contains("win")); - } - } - - [Test] - public static void GetVersionDoesntCrash() - { - PythonEngine.Initialize(); - using (Py.GIL()) - { - string s = PythonEngine.Version; - - Assert.True(s.Length > 0); - Assert.True(s.Contains(",")); - } - } - - [Test] - public static void GetPythonPathDefault() - { - PythonEngine.Initialize(); - string s = PythonEngine.PythonPath; - - StringAssert.Contains("python", s.ToLower()); - PythonEngine.Shutdown(); - } - - [Test] - public static void GetProgramNameDefault() - { - PythonEngine.Initialize(); - string s = PythonEngine.ProgramName; - - Assert.NotNull(s); - PythonEngine.Shutdown(); - } - - /// - /// Test default behavior of PYTHONHOME. If ENVVAR is set it will - /// return the same value. If not, returns EmptyString. - /// - [Test] - public static void GetPythonHomeDefault() - { - string envPythonHome = Environment.GetEnvironmentVariable("PYTHONHOME") ?? ""; - - PythonEngine.Initialize(); - string enginePythonHome = PythonEngine.PythonHome; - - Assert.AreEqual(envPythonHome, enginePythonHome); - PythonEngine.Shutdown(); - } - - [Test] - public void SetPythonHome() - { - PythonEngine.Initialize(); - var pythonHomeBackup = PythonEngine.PythonHome; - PythonEngine.Shutdown(); - - if (pythonHomeBackup == "") - Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); - - var pythonHome = "/dummypath/"; - - PythonEngine.PythonHome = pythonHome; - PythonEngine.Initialize(); - - Assert.AreEqual(pythonHome, PythonEngine.PythonHome); - PythonEngine.Shutdown(); - - // Restoring valid pythonhome. - PythonEngine.PythonHome = pythonHomeBackup; - } - - [Test] - public void SetPythonHomeTwice() - { - PythonEngine.Initialize(); - var pythonHomeBackup = PythonEngine.PythonHome; - PythonEngine.Shutdown(); - - if (pythonHomeBackup == "") - Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); - - var pythonHome = "/dummypath/"; - - PythonEngine.PythonHome = "/dummypath2/"; - PythonEngine.PythonHome = pythonHome; - PythonEngine.Initialize(); - - Assert.AreEqual(pythonHome, PythonEngine.PythonHome); - PythonEngine.Shutdown(); - - PythonEngine.PythonHome = pythonHomeBackup; - } - - [Test] - [Ignore("Currently buggy in Python")] - public void SetPythonHomeEmptyString() - { - PythonEngine.Initialize(); - - var backup = PythonEngine.PythonHome; - if (backup == "") - { - PythonEngine.Shutdown(); - Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); - } - PythonEngine.PythonHome = ""; - - Assert.AreEqual("", PythonEngine.PythonHome); - - PythonEngine.PythonHome = backup; - PythonEngine.Shutdown(); - } - - [Test] - public void SetProgramName() - { - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - - var programNameBackup = PythonEngine.ProgramName; - - var programName = "FooBar"; - - PythonEngine.ProgramName = programName; - PythonEngine.Initialize(); - - Assert.AreEqual(programName, PythonEngine.ProgramName); - PythonEngine.Shutdown(); - - PythonEngine.ProgramName = programNameBackup; - } - - [Test] - public void SetPythonPath() - { - PythonEngine.Initialize(); - - const string moduleName = "pytest"; - bool importShouldSucceed; - try - { - Py.Import(moduleName); - importShouldSucceed = true; - } - catch - { - importShouldSucceed = false; - } - - string[] paths = Py.Import("sys").GetAttr("path").As(); - string path = string.Join(System.IO.Path.PathSeparator.ToString(), paths); - - // path should not be set to PythonEngine.PythonPath here. - // PythonEngine.PythonPath gets the default module search path, not the full search path. - // The list sys.path is initialized with this value on interpreter startup; - // it can be (and usually is) modified later to change the search path for loading modules. - // See https://docs.python.org/3/c-api/init.html#c.Py_GetPath - // After PythonPath is set, then PythonEngine.PythonPath will correctly return the full search path. - - PythonEngine.Shutdown(); - - PythonEngine.PythonPath = path; - PythonEngine.Initialize(); - - Assert.AreEqual(path, PythonEngine.PythonPath); - if (importShouldSucceed) Py.Import(moduleName); - - PythonEngine.Shutdown(); - } - } -} +using System; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestPythonEngineProperties + { + [Test] + public static void GetBuildinfoDoesntCrash() + { + PythonEngine.Initialize(); + using (Py.GIL()) + { + string s = PythonEngine.BuildInfo; + + Assert.True(s.Length > 5); + Assert.True(s.Contains(",")); + } + } + + [Test] + public static void GetCompilerDoesntCrash() + { + PythonEngine.Initialize(); + using (Py.GIL()) + { + string s = PythonEngine.Compiler; + + Assert.True(s.Length > 0); + Assert.True(s.Contains("[")); + Assert.True(s.Contains("]")); + } + } + + [Test] + public static void GetCopyrightDoesntCrash() + { + PythonEngine.Initialize(); + using (Py.GIL()) + { + string s = PythonEngine.Copyright; + + Assert.True(s.Length > 0); + Assert.True(s.Contains("Python Software Foundation")); + } + } + + [Test] + public static void GetPlatformDoesntCrash() + { + PythonEngine.Initialize(); + using (Py.GIL()) + { + string s = PythonEngine.Platform; + + Assert.True(s.Length > 0); + Assert.True(s.Contains("x") || s.Contains("win")); + } + } + + [Test] + public static void GetVersionDoesntCrash() + { + PythonEngine.Initialize(); + using (Py.GIL()) + { + string s = PythonEngine.Version; + + Assert.True(s.Length > 0); + Assert.True(s.Contains(",")); + } + } + + [Test] + public static void GetPythonPathDefault() + { + PythonEngine.Initialize(); + string s = PythonEngine.PythonPath; + + StringAssert.Contains("python", s.ToLower()); + PythonEngine.Shutdown(); + } + + [Test] + public static void GetProgramNameDefault() + { + PythonEngine.Initialize(); + string s = PythonEngine.ProgramName; + + Assert.NotNull(s); + PythonEngine.Shutdown(); + } + + /// + /// Test default behavior of PYTHONHOME. If ENVVAR is set it will + /// return the same value. If not, returns EmptyString. + /// + [Test] + public static void GetPythonHomeDefault() + { + string envPythonHome = Environment.GetEnvironmentVariable("PYTHONHOME") ?? ""; + + PythonEngine.Initialize(); + string enginePythonHome = PythonEngine.PythonHome; + + Assert.AreEqual(envPythonHome, enginePythonHome); + PythonEngine.Shutdown(); + } + + [Test] + public void SetPythonHome() + { + PythonEngine.Initialize(); + var pythonHomeBackup = PythonEngine.PythonHome; + PythonEngine.Shutdown(); + + if (pythonHomeBackup == "") + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); + + var pythonHome = "/dummypath/"; + + PythonEngine.PythonHome = pythonHome; + PythonEngine.Initialize(); + + Assert.AreEqual(pythonHome, PythonEngine.PythonHome); + PythonEngine.Shutdown(); + + // Restoring valid pythonhome. + PythonEngine.PythonHome = pythonHomeBackup; + } + + [Test] + public void SetPythonHomeTwice() + { + PythonEngine.Initialize(); + var pythonHomeBackup = PythonEngine.PythonHome; + PythonEngine.Shutdown(); + + if (pythonHomeBackup == "") + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); + + var pythonHome = "/dummypath/"; + + PythonEngine.PythonHome = "/dummypath2/"; + PythonEngine.PythonHome = pythonHome; + PythonEngine.Initialize(); + + Assert.AreEqual(pythonHome, PythonEngine.PythonHome); + PythonEngine.Shutdown(); + + PythonEngine.PythonHome = pythonHomeBackup; + } + + [Test] + [Ignore("Currently buggy in Python")] + public void SetPythonHomeEmptyString() + { + PythonEngine.Initialize(); + + var backup = PythonEngine.PythonHome; + if (backup == "") + { + PythonEngine.Shutdown(); + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); + } + PythonEngine.PythonHome = ""; + + Assert.AreEqual("", PythonEngine.PythonHome); + + PythonEngine.PythonHome = backup; + PythonEngine.Shutdown(); + } + + [Test] + public void SetProgramName() + { + if (PythonEngine.IsInitialized) + { + PythonEngine.Shutdown(); + } + + var programNameBackup = PythonEngine.ProgramName; + + var programName = "FooBar"; + + PythonEngine.ProgramName = programName; + PythonEngine.Initialize(); + + Assert.AreEqual(programName, PythonEngine.ProgramName); + PythonEngine.Shutdown(); + + PythonEngine.ProgramName = programNameBackup; + } + + [Test] + public void SetPythonPath() + { + PythonEngine.Initialize(); + + const string moduleName = "pytest"; + bool importShouldSucceed; + try + { + Py.Import(moduleName); + importShouldSucceed = true; + } + catch + { + importShouldSucceed = false; + } + + string[] paths = Py.Import("sys").GetAttr("path").As(); + string path = string.Join(System.IO.Path.PathSeparator.ToString(), paths); + + // path should not be set to PythonEngine.PythonPath here. + // PythonEngine.PythonPath gets the default module search path, not the full search path. + // The list sys.path is initialized with this value on interpreter startup; + // it can be (and usually is) modified later to change the search path for loading modules. + // See https://docs.python.org/3/c-api/init.html#c.Py_GetPath + // After PythonPath is set, then PythonEngine.PythonPath will correctly return the full search path. + + PythonEngine.Shutdown(); + + PythonEngine.PythonPath = path; + PythonEngine.Initialize(); + + Assert.AreEqual(path, PythonEngine.PythonPath); + if (importShouldSucceed) Py.Import(moduleName); + + PythonEngine.Shutdown(); + } + } +} diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj index bde07ecab..28f7de531 100644 --- a/src/perf_tests/Python.PerformanceTests.csproj +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -34,6 +34,10 @@ + + + + diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index f97cc5aec..0dc5465d8 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -14,9 +14,28 @@ namespace Python.PythonTestsRunner { public class PythonTestRunner { + private void AddEnvPath(params string[] paths) + { + // PC에 설정되어 있는 환경 변수를 가져온다. + var envPaths = Environment.GetEnvironmentVariable("PATH").Split(Path.PathSeparator).ToList(); + // 중복 환경 변수가 없으면 list에 넣는다. + envPaths.InsertRange(0, paths.Where(x => x.Length > 0 && !envPaths.Contains(x)).ToArray()); + // 환경 변수를 다시 설정한다. + Environment.SetEnvironmentVariable("PATH", string.Join(Path.PathSeparator.ToString(), envPaths), EnvironmentVariableTarget.Process); + Environment.SetEnvironmentVariable("PYTHONNET_PYDLL", "python38.dll"); + } + [OneTimeSetUp] public void SetUp() { + string python_dir = @"D:\Utility\Python38"; + var PYTHON_HOME = Environment.ExpandEnvironmentVariables(python_dir); + + // 환경 변수 설정 + AddEnvPath(PYTHON_HOME); + + PythonEngine.PythonHome = PYTHON_HOME; + Environment.SetEnvironmentVariable("PYTHONNET_PYDLL", "python38.dll"); PythonEngine.Initialize(); } diff --git a/src/runtime/Converter.cs b/src/runtime/Converter.cs index 50b33e60e..a75fe0b69 100644 --- a/src/runtime/Converter.cs +++ b/src/runtime/Converter.cs @@ -95,6 +95,992 @@ internal static BorrowedReference GetPythonTypeByAlias(Type op) } + internal static NewReference ToPython(T value) + => ToPython(value, typeof(T)); + + private static readonly Func IsTransparentProxy = GetIsTransparentProxy(); + + private static bool Never(object _) => false; + + private static Func GetIsTransparentProxy() + { + var remoting = typeof(int).Assembly.GetType("System.Runtime.Remoting.RemotingServices"); + if (remoting is null) return Never; + + var isProxy = remoting.GetMethod("IsTransparentProxy", new[] { typeof(object) }); + if (isProxy is null) return Never; + + return (Func)Delegate.CreateDelegate( + typeof(Func), isProxy, + throwOnBindFailure: true); + } + + internal static NewReference ToPythonDetectType(object? value) + => value is null ? new NewReference(Runtime.PyNone) : ToPython(value, value.GetType()); + internal static NewReference ToPython(object? value, Type type) + { + if (value is PyObject pyObj) + { + return new NewReference(pyObj); + } + + // Null always converts to None in Python. + if (value == null) + { + return new NewReference(Runtime.PyNone); + } + + if (EncodableByUser(type, value)) + { + var encoded = PyObjectConversions.TryEncode(value, type); + if (encoded != null) { + return new NewReference(encoded); + } + } + + if (type.IsInterface) + { + var ifaceObj = (InterfaceObject)ClassManager.GetClassImpl(type); + return ifaceObj.TryWrapObject(value); + } + + if (type.IsArray || type.IsEnum) + { + return CLRObject.GetReference(value, type); + } + + // it the type is a python subclass of a managed type then return the + // underlying python object rather than construct a new wrapper object. + if (value is IPythonDerivedType pyderived) + { + if (!IsTransparentProxy(pyderived)) + return ClassDerivedObject.ToPython(pyderived); + } + + // ModuleObjects are created in a way that their wrapping them as + // a CLRObject fails, the ClassObject has no tpHandle. Return the + // pyHandle as is, do not convert. + if (value is ModuleObject) + { + throw new NotImplementedException(); + } + + // hmm - from Python, we almost never care what the declared + // type is. we'd rather have the object bound to the actual + // implementing class. + + type = value.GetType(); + + if (type.IsEnum) + { + return CLRObject.GetReference(value, type); + } + + TypeCode tc = Type.GetTypeCode(type); + + switch (tc) + { + case TypeCode.Object: + return CLRObject.GetReference(value, type); + + case TypeCode.String: + return Runtime.PyString_FromString((string)value); + + case TypeCode.Int32: + return Runtime.PyInt_FromInt32((int)value); + + case TypeCode.Boolean: + if ((bool)value) + { + return new NewReference(Runtime.PyTrue); + } + return new NewReference(Runtime.PyFalse); + + case TypeCode.Byte: + return Runtime.PyInt_FromInt32((byte)value); + + case TypeCode.Char: + return Runtime.PyUnicode_FromOrdinal((int)((char)value)); + + case TypeCode.Int16: + return Runtime.PyInt_FromInt32((short)value); + + case TypeCode.Int64: + return Runtime.PyLong_FromLongLong((long)value); + + case TypeCode.Single: + return Runtime.PyFloat_FromDouble((float)value); + + case TypeCode.Double: + return Runtime.PyFloat_FromDouble((double)value); + + case TypeCode.SByte: + return Runtime.PyInt_FromInt32((sbyte)value); + + case TypeCode.UInt16: + return Runtime.PyInt_FromInt32((ushort)value); + + case TypeCode.UInt32: + return Runtime.PyLong_FromUnsignedLongLong((uint)value); + + case TypeCode.UInt64: + return Runtime.PyLong_FromUnsignedLongLong((ulong)value); + + default: + return CLRObject.GetReference(value, type); + } + } + + static bool EncodableByUser(Type type, object value) + { + TypeCode typeCode = Type.GetTypeCode(type); + return type.IsEnum + || typeCode is TypeCode.DateTime or TypeCode.Decimal + || typeCode == TypeCode.Object && value.GetType() != typeof(object) && value is not Type; + } + + /// + /// In a few situations, we don't have any advisory type information + /// when we want to convert an object to Python. + /// + internal static NewReference ToPythonImplicit(object? value) + { + if (value == null) + { + return new NewReference(Runtime.PyNone); + } + + return ToPython(value, objectType); + } + + + /// + /// Return a managed object for the given Python object, taking funny + /// byref types into account. + /// + /// A Python object + /// The desired managed type + /// Receives the managed object + /// If true, call Exceptions.SetError with the reason for failure. + /// True on success + internal static bool ToManaged(BorrowedReference value, Type type, + out object? result, bool setError) + { + if (type.IsByRef) + { + type = type.GetElementType(); + } + return Converter.ToManagedValue(value, type, out result, setError); + } + + internal static bool ToManagedValue(BorrowedReference value, Type obType, + out object? result, bool setError) + { + if (obType == typeof(PyObject)) + { + result = new PyObject(value); + return true; + } + + if (obType.IsSubclassOf(typeof(PyObject)) + && !obType.IsAbstract + && obType.GetConstructor(new[] { typeof(PyObject) }) is { } ctor) + { + var untyped = new PyObject(value); + result = ToPyObjectSubclass(ctor, untyped, setError); + return result is not null; + } + + // Common case: if the Python value is a wrapped managed object + // instance, just return the wrapped object. + result = null; + switch (ManagedType.GetManagedObject(value)) + { + case CLRObject co: + object tmp = co.inst; + if (obType.IsInstanceOfType(tmp)) + { + result = tmp; + return true; + } + if (setError) + { + string typeString = tmp is null ? "null" : tmp.GetType().ToString(); + Exceptions.SetError(Exceptions.TypeError, $"{typeString} value cannot be converted to {obType}"); + } + return false; + + case ClassBase cb: + if (!cb.type.Valid) + { + Exceptions.SetError(Exceptions.TypeError, cb.type.DeletedMessage); + return false; + } + result = cb.type.Value; + return true; + + case null: + break; + + default: + throw new ArgumentException("We should never receive instances of other managed types"); + } + + if (value == Runtime.PyNone && !obType.IsValueType) + { + result = null; + return true; + } + + if (obType.IsGenericType && obType.GetGenericTypeDefinition() == typeof(Nullable<>)) + { + if( value == Runtime.PyNone ) + { + result = null; + return true; + } + // Set type to underlying type + obType = obType.GetGenericArguments()[0]; + } + + if (obType.ContainsGenericParameters) + { + if (setError) + { + Exceptions.SetError(Exceptions.TypeError, $"Cannot create an instance of the open generic type {obType}"); + } + return false; + } + + if (obType.IsArray) + { + return ToArray(value, obType, out result, setError); + } + + // Conversion to 'Object' is done based on some reasonable default + // conversions (Python string -> managed string). + if (obType == objectType) + { + if (Runtime.PyString_CheckExact(value)) + { + return ToPrimitive(value, stringType, out result, setError); + } + + if (Runtime.PyBool_CheckExact(value)) + { + return ToPrimitive(value, boolType, out result, setError); + } + + if (Runtime.PyFloat_CheckExact(value)) + { + return ToPrimitive(value, doubleType, out result, setError); + } + + // give custom codecs a chance to take over conversion + // of ints, sequences, and types derived from primitives + BorrowedReference pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) + { + return true; + } + + if (Runtime.PyString_Check(value)) + { + return ToPrimitive(value, stringType, out result, setError); + } + + if (Runtime.PyBool_Check(value)) + { + return ToPrimitive(value, boolType, out result, setError); + } + + if (Runtime.PyFloat_Check(value)) + { + return ToPrimitive(value, doubleType, out result, setError); + } + + if (Runtime.PyInt_Check(value)) + { + result = new PyInt(value); + return true; + } + + result = new PyObject(value); + return true; + } + + // Conversion to 'Type' is done using the same mappings as above for objects. + if (obType == typeType) + { + if (value == Runtime.PyStringType) + { + result = stringType; + return true; + } + + if (value == Runtime.PyBoolType) + { + result = boolType; + return true; + } + + if (value == Runtime.PyLongType) + { + result = typeof(PyInt); + return true; + } + + if (value == Runtime.PyFloatType) + { + result = doubleType; + return true; + } + + if (value == Runtime.PyListType) + { + result = typeof(PyList); + return true; + } + + if (value == Runtime.PyTupleType) + { + result = typeof(PyTuple); + return true; + } + + if (setError) + { + Exceptions.SetError(Exceptions.TypeError, "value cannot be converted to Type"); + } + + return false; + } + + if (DecodableByUser(obType)) + { + BorrowedReference pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) + { + return true; + } + } + + if (obType == typeof(System.Numerics.BigInteger) + && Runtime.PyInt_Check(value)) + { + using var pyInt = new PyInt(value); + result = pyInt.ToBigInteger(); + return true; + } + + return ToPrimitive(value, obType, out result, setError); + } + + /// + /// Unlike , + /// this method does not have a setError parameter, because it should + /// only be called after . + /// + internal static bool ToManagedExplicit(BorrowedReference value, Type obType, + out object? result) + { + result = null; + + // this method would potentially clean any existing error resulting in information loss + Debug.Assert(Runtime.PyErr_Occurred() == null); + + string? converterName = + IsInteger(obType) ? "__int__" + : IsFloatingNumber(obType) ? "__float__" + : null; + + if (converterName is null) return false; + + Debug.Assert(obType.IsPrimitive); + + using var converter = Runtime.PyObject_GetAttrString(value, converterName); + if (converter.IsNull()) + { + Exceptions.Clear(); + return false; + } + + using var explicitlyCoerced = Runtime.PyObject_CallObject(converter.Borrow(), BorrowedReference.Null); + if (explicitlyCoerced.IsNull()) + { + Exceptions.Clear(); + return false; + } + return ToPrimitive(explicitlyCoerced.Borrow(), obType, out result, false); + } + + static object? ToPyObjectSubclass(ConstructorInfo ctor, PyObject instance, bool setError) + { + try + { + return ctor.Invoke(new object[] { instance }); + } + catch (TargetInvocationException ex) + { + if (setError) + { + Exceptions.SetError(ex.InnerException); + } + return null; + } + catch (SecurityException ex) + { + if (setError) + { + Exceptions.SetError(ex); + } + return null; + } + } + + static bool DecodableByUser(Type type) + { + TypeCode typeCode = Type.GetTypeCode(type); + return type.IsEnum + || typeCode is TypeCode.Object or TypeCode.Decimal or TypeCode.DateTime; + } + + internal delegate bool TryConvertFromPythonDelegate(BorrowedReference pyObj, out object? result); + + internal static int ToInt32(BorrowedReference value) + { + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + throw PythonException.ThrowLastAsClrException(); + } + return checked((int)num); + } + + /// + /// Convert a Python value to an instance of a primitive managed type. + /// + internal static bool ToPrimitive(BorrowedReference value, Type obType, out object? result, bool setError) + { + result = null; + if (obType.IsEnum) + { + if (setError) + { + Exceptions.SetError(Exceptions.TypeError, "since Python.NET 3.0 int can not be converted to Enum implicitly. Use Enum(int_value)"); + } + return false; + } + + TypeCode tc = Type.GetTypeCode(obType); + + switch (tc) + { + case TypeCode.String: + string? st = Runtime.GetManagedString(value); + if (st == null) + { + goto type_error; + } + result = st; + return true; + + case TypeCode.Int32: + { + // Python3 always use PyLong API + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + if (num > Int32.MaxValue || num < Int32.MinValue) + { + goto overflow; + } + result = (int)num; + return true; + } + + case TypeCode.Boolean: + if (value == Runtime.PyTrue) + { + result = true; + return true; + } + if (value == Runtime.PyFalse) + { + result = false; + return true; + } + if (setError) + { + goto type_error; + } + return false; + + case TypeCode.Byte: + { + if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) + { + if (Runtime.PyBytes_Size(value) == 1) + { + IntPtr bytePtr = Runtime.PyBytes_AsString(value); + result = (byte)Marshal.ReadByte(bytePtr); + return true; + } + goto type_error; + } + + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + if (num > Byte.MaxValue || num < Byte.MinValue) + { + goto overflow; + } + result = (byte)num; + return true; + } + + case TypeCode.SByte: + { + if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) + { + if (Runtime.PyBytes_Size(value) == 1) + { + IntPtr bytePtr = Runtime.PyBytes_AsString(value); + result = (sbyte)Marshal.ReadByte(bytePtr); + return true; + } + goto type_error; + } + + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + if (num > SByte.MaxValue || num < SByte.MinValue) + { + goto overflow; + } + result = (sbyte)num; + return true; + } + + case TypeCode.Char: + { + if (Runtime.PyObject_TypeCheck(value, Runtime.PyBytesType)) + { + if (Runtime.PyBytes_Size(value) == 1) + { + IntPtr bytePtr = Runtime.PyBytes_AsString(value); + result = (char)Marshal.ReadByte(bytePtr); + return true; + } + goto type_error; + } + else if (Runtime.PyObject_TypeCheck(value, Runtime.PyUnicodeType)) + { + if (Runtime.PyUnicode_GetLength(value) == 1) + { + IntPtr unicodePtr = Runtime.PyUnicode_AsUnicode(value); + Char[] buff = new Char[1]; + Marshal.Copy(unicodePtr, buff, 0, 1); + result = buff[0]; + return true; + } + goto type_error; + } + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + if (num > Char.MaxValue || num < Char.MinValue) + { + goto overflow; + } + result = (char)num; + return true; + } + + case TypeCode.Int16: + { + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + if (num > Int16.MaxValue || num < Int16.MinValue) + { + goto overflow; + } + result = (short)num; + return true; + } + + case TypeCode.Int64: + { + if (Runtime.Is32Bit) + { + if (!Runtime.PyInt_Check(value)) + { + goto type_error; + } + long? num = Runtime.PyLong_AsLongLong(value); + if (num is null) + { + goto convert_error; + } + result = num.Value; + return true; + } + else + { + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + result = (long)num; + return true; + } + } + + case TypeCode.UInt16: + { + nint num = Runtime.PyLong_AsSignedSize_t(value); + if (num == -1 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + if (num > UInt16.MaxValue || num < UInt16.MinValue) + { + goto overflow; + } + result = (ushort)num; + return true; + } + + case TypeCode.UInt32: + { + nuint num = Runtime.PyLong_AsUnsignedSize_t(value); + if (num == unchecked((nuint)(-1)) && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + if (num > UInt32.MaxValue) + { + goto overflow; + } + result = (uint)num; + return true; + } + + case TypeCode.UInt64: + { + ulong? num = Runtime.PyLong_AsUnsignedLongLong(value); + if (num is null) + { + goto convert_error; + } + result = num.Value; + return true; + } + + case TypeCode.Single: + { + if (!Runtime.PyFloat_Check(value) && !Runtime.PyInt_Check(value)) + { + goto type_error; + } + double num = Runtime.PyFloat_AsDouble(value); + if (num == -1.0 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + if (num > Single.MaxValue || num < Single.MinValue) + { + if (!double.IsInfinity(num)) + { + goto overflow; + } + } + result = (float)num; + return true; + } + + case TypeCode.Double: + { + if (!Runtime.PyFloat_Check(value) && !Runtime.PyInt_Check(value)) + { + goto type_error; + } + double num = Runtime.PyFloat_AsDouble(value); + if (num == -1.0 && Exceptions.ErrorOccurred()) + { + goto convert_error; + } + result = num; + return true; + } + default: + goto type_error; + } + + convert_error: + if (!setError) + { + Exceptions.Clear(); + } + return false; + + type_error: + if (setError) + { + string tpName = Runtime.PyObject_GetTypeName(value); + Exceptions.SetError(Exceptions.TypeError, $"'{tpName}' value cannot be converted to {obType}"); + } + return false; + + overflow: + // C# level overflow error + if (setError) + { + Exceptions.SetError(Exceptions.OverflowError, "value too large to convert"); + } + return false; + } + + private static void SetConversionError(BorrowedReference value, Type target) + { + // PyObject_Repr might clear the error + Runtime.PyErr_Fetch(out var causeType, out var causeVal, out var causeTrace); + + var ob = Runtime.PyObject_Repr(value); + string src = "'object has no repr'"; + if (ob.IsNull()) + { + Exceptions.Clear(); + } + else + { + src = Runtime.GetManagedString(ob.Borrow()) ?? src; + } + ob.Dispose(); + + Runtime.PyErr_Restore(causeType.StealNullable(), causeVal.StealNullable(), causeTrace.StealNullable()); + Exceptions.RaiseTypeError($"Cannot convert {src} to {target}"); + } + + + /// + /// Convert a Python value to a correctly typed managed array instance. + /// The Python value must support the Python iterator protocol or and the + /// items in the sequence must be convertible to the target array type. + /// + private static bool ToArray(BorrowedReference value, Type obType, out object? result, bool setError) + { + Type elementType = obType.GetElementType(); + result = null; + + using var IterObject = Runtime.PyObject_GetIter(value); + if (IterObject.IsNull()) + { + if (setError) + { + SetConversionError(value, obType); + } + else + { + // PyObject_GetIter will have set an error + Exceptions.Clear(); + } + return false; + } + + IList list; + try + { + // MakeGenericType can throw because elementType may not be a valid generic argument even though elementType[] is a valid array type. + // For example, if elementType is a pointer type. + // See https://docs.microsoft.com/en-us/dotnet/api/system.type.makegenerictype#System_Type_MakeGenericType_System_Type + var constructedListType = typeof(List<>).MakeGenericType(elementType); + bool IsSeqObj = Runtime.PySequence_Check(value); + object[] constructorArgs = Array.Empty(); + if (IsSeqObj) + { + var len = Runtime.PySequence_Size(value); + if (len >= 0) + { + if (len <= int.MaxValue) + { + constructorArgs = new object[] { (int)len }; + } + } + else + { + // for the sequences, that explicitly deny calling __len__() + Exceptions.Clear(); + } + } + // CreateInstance can throw even if MakeGenericType succeeded. + // See https://docs.microsoft.com/en-us/dotnet/api/system.activator.createinstance#System_Activator_CreateInstance_System_Type_ + list = (IList)Activator.CreateInstance(constructedListType, args: constructorArgs); + } + catch (Exception e) + { + if (setError) + { + Exceptions.SetError(e); + SetConversionError(value, obType); + } + return false; + } + + while (true) + { + using var item = Runtime.PyIter_Next(IterObject.Borrow()); + if (item.IsNull()) break; + + if (!Converter.ToManaged(item.Borrow(), elementType, out var obj, setError)) + { + return false; + } + + list.Add(obj); + } + + if (Exceptions.ErrorOccurred()) + { + if (!setError) Exceptions.Clear(); + return false; + } + + Array items = Array.CreateInstance(elementType, list.Count); + list.CopyTo(items, 0); + + result = items; + return true; + } + + internal static bool IsFloatingNumber(Type type) => type == typeof(float) || type == typeof(double); + internal static bool IsInteger(Type type) + => type == typeof(Byte) || type == typeof(SByte) + || type == typeof(Int16) || type == typeof(UInt16) + || type == typeof(Int32) || type == typeof(UInt32) + || type == typeof(Int64) || type == typeof(UInt64); + } + + public static class ConverterExtension + { + public static PyObject ToPython(this object? o) + { + if (o is null) return Runtime.None; + return Converter.ToPython(o, o.GetType()).MoveToPyObject(); + } + } +} +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Security; + +namespace Python.Runtime +{ + /// + /// Performs data conversions between managed types and Python types. + /// + [SuppressUnmanagedCodeSecurity] + internal class Converter + { + private Converter() + { + } + + private static readonly Type objectType; + private static readonly Type stringType; + private static readonly Type singleType; + private static readonly Type doubleType; + private static readonly Type int16Type; + private static readonly Type int32Type; + private static readonly Type int64Type; + private static readonly Type boolType; + private static readonly Type typeType; + + static Converter() + { + objectType = typeof(Object); + stringType = typeof(String); + int16Type = typeof(Int16); + int32Type = typeof(Int32); + int64Type = typeof(Int64); + singleType = typeof(Single); + doubleType = typeof(Double); + boolType = typeof(Boolean); + typeType = typeof(Type); + } + + + /// + /// Given a builtin Python type, return the corresponding CLR type. + /// + internal static Type? GetTypeByAlias(BorrowedReference op) + { + if (op == Runtime.PyStringType) + return stringType; + + if (op == Runtime.PyUnicodeType) + return stringType; + + if (op == Runtime.PyLongType) + return int32Type; + + if (op == Runtime.PyLongType) + return int64Type; + + if (op == Runtime.PyFloatType) + return doubleType; + + if (op == Runtime.PyBoolType) + return boolType; + + return null; + } + + internal static BorrowedReference GetPythonTypeByAlias(Type op) + { + if (op == stringType) + return Runtime.PyUnicodeType.Reference; + + if (op == int16Type) + return Runtime.PyLongType.Reference; + + if (op == int32Type) + return Runtime.PyLongType.Reference; + + if (op == int64Type) + return Runtime.PyLongType.Reference; + + if (op == doubleType) + return Runtime.PyFloatType.Reference; + + if (op == singleType) + return Runtime.PyFloatType.Reference; + + if (op == boolType) + return Runtime.PyBoolType.Reference; + + return BorrowedReference.Null; + } + + internal static NewReference ToPython(T value) => ToPython(value, typeof(T)); diff --git a/src/runtime/Finalizer.cs b/src/runtime/Finalizer.cs index 5b5ecfcfc..bd7356605 100644 --- a/src/runtime/Finalizer.cs +++ b/src/runtime/Finalizer.cs @@ -138,6 +138,426 @@ internal void AddFinalizedObject(ref IntPtr obj, int run Debug.Assert(Runtime.Refcount(new BorrowedReference(obj)) > 0); +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { + this._objQueue.Enqueue(new PendingFinalization { + PyObj = obj, RuntimeRun = run, +#if TRACE_ALLOC + StackTrace = stackTrace.ToString(), +#endif + }); + } + obj = IntPtr.Zero; + } + + internal void AddDerivedFinalizedObject(ref IntPtr derived, int run) + { + if (derived == IntPtr.Zero) + throw new ArgumentNullException(nameof(derived)); + + if (!Enable) + { + return; + } + + var pending = new PendingFinalization { PyObj = derived, RuntimeRun = run }; + derived = IntPtr.Zero; + _derivedQueue.Enqueue(pending); + } + + internal void AddFinalizedBuffer(ref Py_buffer buffer) + { + if (buffer.obj == IntPtr.Zero) + throw new ArgumentNullException(nameof(buffer)); + + if (!Enable) + return; + + var pending = buffer; + buffer = default; + _bufferQueue.Enqueue(pending); + } + + internal static void Initialize() + { + Instance.started = true; + } + + internal static void Shutdown() + { + Instance.DisposeAll(); + Instance.started = false; + } + + internal nint DisposeAll() + { + if (_objQueue.IsEmpty && _derivedQueue.IsEmpty && _bufferQueue.IsEmpty) + return 0; + + nint collected = 0; + + BeforeCollect?.Invoke(this, new CollectArgs() + { + ObjectCount = _objQueue.Count + }); +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { +#if FINALIZER_CHECK + ValidateRefCount(); +#endif + Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback); + Debug.Assert(errType.IsNull()); + + int run = Runtime.GetRun(); + + try + { + while (!_objQueue.IsEmpty) + { + if (!_objQueue.TryDequeue(out var obj)) + continue; + + if (obj.RuntimeRun != run) + { + HandleFinalizationException(obj.PyObj, new RuntimeShutdownException(obj.PyObj)); + continue; + } + + IntPtr copyForException = obj.PyObj; + Runtime.XDecref(StolenReference.Take(ref obj.PyObj)); + collected++; + try + { + Runtime.CheckExceptionOccurred(); + } + catch (Exception e) + { + HandleFinalizationException(obj.PyObj, e); + } + } + + while (!_derivedQueue.IsEmpty) + { + if (!_derivedQueue.TryDequeue(out var derived)) + continue; + + if (derived.RuntimeRun != run) + { + HandleFinalizationException(derived.PyObj, new RuntimeShutdownException(derived.PyObj)); + continue; + } + +#pragma warning disable CS0618 // Type or member is obsolete. OK for internal use + PythonDerivedType.Finalize(derived.PyObj); +#pragma warning restore CS0618 // Type or member is obsolete + + collected++; + } + + while (!_bufferQueue.IsEmpty) + { + if (!_bufferQueue.TryDequeue(out var buffer)) + continue; + + Runtime.PyBuffer_Release(ref buffer); + collected++; + } + } + finally + { + // Python requires finalizers to preserve exception: + // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation + Runtime.PyErr_Restore(errType.StealNullable(), errVal.StealNullable(), traceback.StealNullable()); + } + } + return collected; + } + + void HandleFinalizationException(IntPtr obj, Exception cause) + { + var errorArgs = new ErrorArgs(cause); + + ErrorHandler?.Invoke(this, errorArgs); + + if (!errorArgs.Handled) + { + throw new FinalizationException( + "Python object finalization failed", + disposable: obj, innerException: cause); + } + } + +#if FINALIZER_CHECK + private void ValidateRefCount() + { + if (!RefCountValidationEnabled) + { + return; + } + var counter = new Dictionary(); + var holdRefs = new Dictionary(); + var indexer = new Dictionary>(); + foreach (var obj in _objQueue) + { + var handle = obj; + if (!counter.ContainsKey(handle)) + { + counter[handle] = 0; + } + counter[handle]++; + if (!holdRefs.ContainsKey(handle)) + { + holdRefs[handle] = Runtime.Refcount(handle); + } + List objs; + if (!indexer.TryGetValue(handle, out objs)) + { + objs = new List(); + indexer.Add(handle, objs); + } + objs.Add(obj); + } + foreach (var pair in counter) + { + IntPtr handle = pair.Key; + long cnt = pair.Value; + // Tracked handle's ref count is larger than the object's holds + // it may take an unspecified behaviour if it decref in Dispose + if (cnt > holdRefs[handle]) + { + var args = new IncorrectFinalizeArgs() + { + Handle = handle, + ImpactedObjects = indexer[handle] + }; + bool handled = false; + if (IncorrectRefCntResolver != null) + { + var funcList = IncorrectRefCntResolver.GetInvocationList(); + foreach (IncorrectRefCntHandler func in funcList) + { + if (func(this, args)) + { + handled = true; + break; + } + } + } + if (!handled && ThrowIfUnhandleIncorrectRefCount) + { + throw new IncorrectRefCountException(handle); + } + } + // Make sure no other references for PyObjects after this method + indexer[handle].Clear(); + } + indexer.Clear(); + } +#endif + } + + struct PendingFinalization + { + public IntPtr PyObj; + public BorrowedReference Ref => new(PyObj); + public ManagedType? Managed => ManagedType.GetManagedObject(Ref); + public nint RefCount => Runtime.Refcount(Ref); + public int RuntimeRun; +#if TRACE_ALLOC + public string StackTrace; +#endif + } + + public class FinalizationException : Exception + { + public IntPtr Handle { get; } + + /// + /// Gets the object, whose finalization failed. + /// + /// If this function crashes, you can also try , + /// which does not attempt to increase the object reference count. + /// + public PyObject GetObject() => new(new BorrowedReference(this.Handle)); + /// + /// Gets the object, whose finalization failed without incrementing + /// its reference count. This should only ever be called during debugging. + /// When the result is disposed or finalized, the program will crash. + /// + public PyObject DebugGetObject() + { + IntPtr dangerousNoIncRefCopy = this.Handle; + return new(StolenReference.Take(ref dangerousNoIncRefCopy)); + } + + public FinalizationException(string message, IntPtr disposable, Exception innerException) + : base(message, innerException) + { + if (disposable == IntPtr.Zero) throw new ArgumentNullException(nameof(disposable)); + this.Handle = disposable; + } + + protected FinalizationException(string message, IntPtr disposable) + : base(message) + { + if (disposable == IntPtr.Zero) throw new ArgumentNullException(nameof(disposable)); + this.Handle = disposable; + } + } + + public class RuntimeShutdownException : FinalizationException + { + public RuntimeShutdownException(IntPtr disposable) + : base("Python runtime was shut down after this object was created." + + " It is an error to attempt to dispose or to continue using it even after restarting the runtime.", disposable) + { + } + } +} +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Python.Runtime +{ + public class Finalizer + { + public class CollectArgs : EventArgs + { + public int ObjectCount { get; set; } + } + + public class ErrorArgs : EventArgs + { + public ErrorArgs(Exception error) + { + Error = error ?? throw new ArgumentNullException(nameof(error)); + } + public bool Handled { get; set; } + public Exception Error { get; } + } + + public static Finalizer Instance { get; } = new (); + + public event EventHandler? BeforeCollect; + public event EventHandler? ErrorHandler; + + const int DefaultThreshold = 200; + [DefaultValue(DefaultThreshold)] + public int Threshold { get; set; } = DefaultThreshold; + + bool started; + + [DefaultValue(true)] + public bool Enable { get; set; } = true; + + private readonly ConcurrentQueue _objQueue = new(); + private readonly ConcurrentQueue _derivedQueue = new(); + private readonly ConcurrentQueue _bufferQueue = new(); + private int _throttled; + + #region FINALIZER_CHECK + +#if FINALIZER_CHECK + private readonly object _queueLock = new object(); + internal bool RefCountValidationEnabled { get; set; } = true; +#else + internal bool RefCountValidationEnabled { get; set; } = false; +#endif + // Keep these declarations for compat even no FINALIZER_CHECK + internal class IncorrectFinalizeArgs : EventArgs + { + public IncorrectFinalizeArgs(IntPtr handle, IReadOnlyCollection imacted) + { + Handle = handle; + ImpactedObjects = imacted; + } + public IntPtr Handle { get; } + public BorrowedReference Reference => new(Handle); + public IReadOnlyCollection ImpactedObjects { get; } + } + + internal class IncorrectRefCountException : Exception + { + public IntPtr PyPtr { get; internal set; } + string? message; + public override string Message + { + get + { + if (message is not null) return message; + var gil = PythonEngine.AcquireLock(); + try + { + using var pyname = Runtime.PyObject_Str(new BorrowedReference(PyPtr)); + string name = Runtime.GetManagedString(pyname.BorrowOrThrow()) ?? Util.BadStr; + message = $"<{name}> may has a incorrect ref count"; + } + finally + { + PythonEngine.ReleaseLock(gil); + } + return message; + } + } + + internal IncorrectRefCountException(IntPtr ptr) + { + PyPtr = ptr; + + } + } + + internal delegate bool IncorrectRefCntHandler(object sender, IncorrectFinalizeArgs e); + #pragma warning disable 414 + internal event IncorrectRefCntHandler? IncorrectRefCntResolver = null; + #pragma warning restore 414 + internal bool ThrowIfUnhandleIncorrectRefCount { get; set; } = true; + + #endregion + + [ForbidPythonThreads] + public void Collect() => this.DisposeAll(); + + internal void ThrottledCollect() + { + if (!started) throw new InvalidOperationException($"{nameof(PythonEngine)} is not initialized"); + + _throttled = unchecked(this._throttled + 1); + if (!started || !Enable || _throttled < Threshold) return; + _throttled = 0; + this.Collect(); + } + + internal List GetCollectedObjects() + { + return _objQueue.Select(o => o.PyObj).ToList(); + } + + internal void AddFinalizedObject(ref IntPtr obj, int run +#if TRACE_ALLOC + , StackTrace stackTrace +#endif + ) + { + Debug.Assert(obj != IntPtr.Zero); + if (!Enable) + { + return; + } + + Debug.Assert(Runtime.Refcount(new BorrowedReference(obj)) > 0); + #if FINALIZER_CHECK lock (_queueLock) #endif diff --git a/src/runtime/Native/TypeOffset311.cs b/src/runtime/Native/TypeOffset311.cs index de5afacb9..543b9e383 100644 --- a/src/runtime/Native/TypeOffset311.cs +++ b/src/runtime/Native/TypeOffset311.cs @@ -1,141 +1,141 @@ - -// Auto-generated by geninterop.py. -// DO NOT MODIFY BY HAND. - -// Python 3.11: ABI flags: '' - -// ReSharper disable InconsistentNaming -// ReSharper disable IdentifierTypo - -using System; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.InteropServices; - -using Python.Runtime.Native; - -namespace Python.Runtime -{ - [SuppressMessage("Style", "IDE1006:Naming Styles", - Justification = "Following CPython", - Scope = "type")] - - [StructLayout(LayoutKind.Sequential)] - internal class TypeOffset311 : GeneratedTypeOffsets, ITypeOffsets - { - public TypeOffset311() { } - // Auto-generated from PyHeapTypeObject in Python.h - public int ob_refcnt { get; private set; } - public int ob_type { get; private set; } - public int ob_size { get; private set; } - public int tp_name { get; private set; } - public int tp_basicsize { get; private set; } - public int tp_itemsize { get; private set; } - public int tp_dealloc { get; private set; } - public int tp_vectorcall_offset { get; private set; } - public int tp_getattr { get; private set; } - public int tp_setattr { get; private set; } - public int tp_as_async { get; private set; } - public int tp_repr { get; private set; } - public int tp_as_number { get; private set; } - public int tp_as_sequence { get; private set; } - public int tp_as_mapping { get; private set; } - public int tp_hash { get; private set; } - public int tp_call { get; private set; } - public int tp_str { get; private set; } - public int tp_getattro { get; private set; } - public int tp_setattro { get; private set; } - public int tp_as_buffer { get; private set; } - public int tp_flags { get; private set; } - public int tp_doc { get; private set; } - public int tp_traverse { get; private set; } - public int tp_clear { get; private set; } - public int tp_richcompare { get; private set; } - public int tp_weaklistoffset { get; private set; } - public int tp_iter { get; private set; } - public int tp_iternext { get; private set; } - public int tp_methods { get; private set; } - public int tp_members { get; private set; } - public int tp_getset { get; private set; } - public int tp_base { get; private set; } - public int tp_dict { get; private set; } - public int tp_descr_get { get; private set; } - public int tp_descr_set { get; private set; } - public int tp_dictoffset { get; private set; } - public int tp_init { get; private set; } - public int tp_alloc { get; private set; } - public int tp_new { get; private set; } - public int tp_free { get; private set; } - public int tp_is_gc { get; private set; } - public int tp_bases { get; private set; } - public int tp_mro { get; private set; } - public int tp_cache { get; private set; } - public int tp_subclasses { get; private set; } - public int tp_weaklist { get; private set; } - public int tp_del { get; private set; } - public int tp_version_tag { get; private set; } - public int tp_finalize { get; private set; } - public int tp_vectorcall { get; private set; } - public int am_await { get; private set; } - public int am_aiter { get; private set; } - public int am_anext { get; private set; } - public int am_send { get; private set; } - public int nb_add { get; private set; } - public int nb_subtract { get; private set; } - public int nb_multiply { get; private set; } - public int nb_remainder { get; private set; } - public int nb_divmod { get; private set; } - public int nb_power { get; private set; } - public int nb_negative { get; private set; } - public int nb_positive { get; private set; } - public int nb_absolute { get; private set; } - public int nb_bool { get; private set; } - public int nb_invert { get; private set; } - public int nb_lshift { get; private set; } - public int nb_rshift { get; private set; } - public int nb_and { get; private set; } - public int nb_xor { get; private set; } - public int nb_or { get; private set; } - public int nb_int { get; private set; } - public int nb_reserved { get; private set; } - public int nb_float { get; private set; } - public int nb_inplace_add { get; private set; } - public int nb_inplace_subtract { get; private set; } - public int nb_inplace_multiply { get; private set; } - public int nb_inplace_remainder { get; private set; } - public int nb_inplace_power { get; private set; } - public int nb_inplace_lshift { get; private set; } - public int nb_inplace_rshift { get; private set; } - public int nb_inplace_and { get; private set; } - public int nb_inplace_xor { get; private set; } - public int nb_inplace_or { get; private set; } - public int nb_floor_divide { get; private set; } - public int nb_true_divide { get; private set; } - public int nb_inplace_floor_divide { get; private set; } - public int nb_inplace_true_divide { get; private set; } - public int nb_index { get; private set; } - public int nb_matrix_multiply { get; private set; } - public int nb_inplace_matrix_multiply { get; private set; } - public int mp_length { get; private set; } - public int mp_subscript { get; private set; } - public int mp_ass_subscript { get; private set; } - public int sq_length { get; private set; } - public int sq_concat { get; private set; } - public int sq_repeat { get; private set; } - public int sq_item { get; private set; } - public int was_sq_slice { get; private set; } - public int sq_ass_item { get; private set; } - public int was_sq_ass_slice { get; private set; } - public int sq_contains { get; private set; } - public int sq_inplace_concat { get; private set; } - public int sq_inplace_repeat { get; private set; } - public int bf_getbuffer { get; private set; } - public int bf_releasebuffer { get; private set; } - public int name { get; private set; } - public int ht_slots { get; private set; } - public int qualname { get; private set; } - public int ht_cached_keys { get; private set; } - public int ht_module { get; private set; } - public int _ht_tpname { get; private set; } - public int spec_cache_getitem { get; private set; } - } -} + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.11: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset311 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset311() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int am_send { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + public int _ht_tpname { get; private set; } + public int spec_cache_getitem { get; private set; } + } +} diff --git a/src/runtime/PythonEngine.cs b/src/runtime/PythonEngine.cs index 2c4c6c088..19c02b19c 100644 --- a/src/runtime/PythonEngine.cs +++ b/src/runtime/PythonEngine.cs @@ -134,6 +134,678 @@ public static string PythonPath } } + public static Version MinSupportedVersion => new(3, 7); + public static Version MaxSupportedVersion => new(3, 11, int.MaxValue, int.MaxValue); + public static bool IsSupportedVersion(Version version) => version >= MinSupportedVersion && version <= MaxSupportedVersion; + + public static string Version + { + get { return Marshal.PtrToStringAnsi(Runtime.Py_GetVersion()); } + } + + public static string BuildInfo + { + get { return Marshal.PtrToStringAnsi(Runtime.Py_GetBuildInfo()); } + } + + public static string Platform + { + get { return Marshal.PtrToStringAnsi(Runtime.Py_GetPlatform()); } + } + + public static string Copyright + { + get { return Marshal.PtrToStringAnsi(Runtime.Py_GetCopyright()); } + } + + public static string Compiler + { + get { return Marshal.PtrToStringAnsi(Runtime.Py_GetCompiler()); } + } + + /// + /// Set the NoSiteFlag to disable loading the site module. + /// Must be called before Initialize. + /// https://docs.python.org/3/c-api/init.html#c.Py_NoSiteFlag + /// + public static void SetNoSiteFlag() + { + Runtime.SetNoSiteFlag(); + } + + public static int RunSimpleString(string code) + { + return Runtime.PyRun_SimpleString(code); + } + + public static void Initialize() + { + Initialize(setSysArgv: true); + } + + public static void Initialize(bool setSysArgv = true, bool initSigs = false) + { + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs); + } + + /// + /// Initialize Method + /// + /// + /// Initialize the Python runtime. It is safe to call this method + /// more than once, though initialization will only happen on the + /// first call. It is *not* necessary to hold the Python global + /// interpreter lock (GIL) to call this method. + /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. + /// + public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false) + { + if (initialized) + { + return; + } + // Creating the delegateManager MUST happen before Runtime.Initialize + // is called. If it happens afterwards, DelegateManager's CodeGenerator + // throws an exception in its ctor. This exception is eaten somehow + // during an initial "import clr", and the world ends shortly thereafter. + // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). + delegateManager = new DelegateManager(); + Runtime.Initialize(initSigs); + initialized = true; + Exceptions.Clear(); + + // Make sure we clean up properly on app domain unload. + AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; + AppDomain.CurrentDomain.ProcessExit += OnProcessExit; + + if (setSysArgv) + { + Py.SetArgv(args); + } + + // Load the clr.py resource into the clr module + BorrowedReference clr_dict = Runtime.PyModule_GetDict(ImportHook.ClrModuleReference); + + var locals = new PyDict(); + try + { + BorrowedReference module = DefineModule("clr._extras"); + BorrowedReference module_globals = Runtime.PyModule_GetDict(module); + + Assembly assembly = Assembly.GetExecutingAssembly(); + // add the contents of clr.py to the module + string clr_py = assembly.ReadStringResource("clr.py"); + + Exec(clr_py, module_globals, locals.Reference); + + LoadSubmodule(module_globals, "clr.interop", "interop.py"); + + LoadMixins(module_globals); + + // add the imported module to the clr module, and copy the API functions + // and decorators into the main clr module. + Runtime.PyDict_SetItemString(clr_dict, "_extras", module); + + // append version + var version = typeof(PythonEngine) + .Assembly + .GetCustomAttribute() + .InformationalVersion; + using var versionObj = Runtime.PyString_FromString(version); + Runtime.PyDict_SetItemString(clr_dict, "__version__", versionObj.Borrow()); + + using var keys = locals.Keys(); + foreach (PyObject key in keys) + { + if (!key.ToString()!.StartsWith("_")) + { + using PyObject value = locals[key]; + Runtime.PyDict_SetItem(clr_dict, key.Reference, value.Reference); + } + key.Dispose(); + } + } + finally + { + locals.Dispose(); + } + + ImportHook.UpdateCLRModuleDict(); + } + + static BorrowedReference DefineModule(string name) + { + var module = Runtime.PyImport_AddModule(name); + var module_globals = Runtime.PyModule_GetDict(module); + var builtins = Runtime.PyEval_GetBuiltins(); + Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); + return module; + } + + static void LoadSubmodule(BorrowedReference targetModuleDict, string fullName, string resourceName) + { + string? memberName = fullName.AfterLast('.'); + Debug.Assert(memberName != null); + + var module = DefineModule(fullName); + var module_globals = Runtime.PyModule_GetDict(module); + + Assembly assembly = Assembly.GetExecutingAssembly(); + string pyCode = assembly.ReadStringResource(resourceName); + Exec(pyCode, module_globals, module_globals); + + Runtime.PyDict_SetItemString(targetModuleDict, memberName!, module); + } + + static void LoadMixins(BorrowedReference targetModuleDict) + { + foreach (string nested in new[] { "collections" }) + { + LoadSubmodule(targetModuleDict, + fullName: "clr._extras." + nested, + resourceName: typeof(PythonEngine).Namespace + ".Mixins." + nested + ".py"); + } + } + + static void OnDomainUnload(object _, EventArgs __) + { + Shutdown(); + } + + static void OnProcessExit(object _, EventArgs __) + { + Runtime.ProcessIsTerminating = true; + Shutdown(); + } + + /// + /// A helper to perform initialization from the context of an active + /// CPython interpreter process - this bootstraps the managed runtime + /// when it is imported by the CLR extension module. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete(Util.InternalUseOnly)] + public static IntPtr InitExt() + { + try + { + if (Runtime.IsInitialized) + { + var builtins = Runtime.PyEval_GetBuiltins(); + var runtimeError = Runtime.PyDict_GetItemString(builtins, "RuntimeError"); + Exceptions.SetError(runtimeError, "Python.NET runtime is already initialized"); + return IntPtr.Zero; + } + Runtime.HostedInPython = true; + + Initialize(setSysArgv: false); + + Finalizer.Instance.ErrorHandler += AllowLeaksDuringShutdown; + } + catch (PythonException e) + { + e.Restore(); + return IntPtr.Zero; + } + + return Python.Runtime.ImportHook.GetCLRModule() + .DangerousMoveToPointerOrNull(); + } + + private static void AllowLeaksDuringShutdown(object sender, Finalizer.ErrorArgs e) + { + if (e.Error is RuntimeShutdownException) + { + e.Handled = true; + } + } + + /// + /// Shutdown and release resources held by the Python runtime. The + /// Python runtime can no longer be used in the current process + /// after calling the Shutdown method. + /// + public static void Shutdown() + { + if (!initialized) + { + return; + } + + using (Py.GIL()) + { + if (Exceptions.ErrorOccurred()) + { + throw new InvalidOperationException( + "Python error indicator is set", + innerException: PythonException.PeekCurrentOrNull(out _)); + } + } + + // If the shutdown handlers trigger a domain unload, + // don't call shutdown again. + AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; + AppDomain.CurrentDomain.ProcessExit -= OnProcessExit; + + ExecuteShutdownHandlers(); + // Remember to shut down the runtime. + Runtime.Shutdown(); + + initialized = false; + + InteropConfiguration = InteropConfiguration.MakeDefault(); + } + + /// + /// Called when the engine is shut down. + /// + /// Shutdown handlers are run in reverse order they were added, so that + /// resources available when running a shutdown handler are the same as + /// what was available when it was added. + /// + public delegate void ShutdownHandler(); + + static readonly List ShutdownHandlers = new(); + + /// + /// Add a function to be called when the engine is shut down. + /// + /// Shutdown handlers are executed in the opposite order they were + /// added, so that you can be sure that everything that was initialized + /// when you added the handler is still initialized when you need to shut + /// down. + /// + /// If the same shutdown handler is added several times, it will be run + /// several times. + /// + /// Don't add shutdown handlers while running a shutdown handler. + /// + public static void AddShutdownHandler(ShutdownHandler handler) + { + ShutdownHandlers.Add(handler); + } + + /// + /// Remove a shutdown handler. + /// + /// If the same shutdown handler is added several times, only the last + /// one is removed. + /// + /// Don't remove shutdown handlers while running a shutdown handler. + /// + public static void RemoveShutdownHandler(ShutdownHandler handler) + { + for (int index = ShutdownHandlers.Count - 1; index >= 0; --index) + { + if (ShutdownHandlers[index] == handler) + { + ShutdownHandlers.RemoveAt(index); + break; + } + } + } + + /// + /// Run all the shutdown handlers. + /// + /// They're run in opposite order they were added. + /// + static void ExecuteShutdownHandlers() + { + while(ShutdownHandlers.Count > 0) + { + var handler = ShutdownHandlers[ShutdownHandlers.Count - 1]; + ShutdownHandlers.RemoveAt(ShutdownHandlers.Count - 1); + handler(); + } + } + + /// + /// AcquireLock Method + /// + /// + /// Acquire the Python global interpreter lock (GIL). Managed code + /// *must* call this method before using any objects or calling any + /// methods on objects in the Python.Runtime namespace. The only + /// exception is PythonEngine.Initialize, which may be called without + /// first calling AcquireLock. + /// Each call to AcquireLock must be matched by a corresponding call + /// to ReleaseLock, passing the token obtained from AcquireLock. + /// For more information, see the "Extending and Embedding" section + /// of the Python documentation on www.python.org. + /// + internal static PyGILState AcquireLock() + { + return Runtime.PyGILState_Ensure(); + } + + + /// + /// ReleaseLock Method + /// + /// + /// Release the Python global interpreter lock using a token obtained + /// from a previous call to AcquireLock. + /// For more information, see the "Extending and Embedding" section + /// of the Python documentation on www.python.org. + /// + internal static void ReleaseLock(PyGILState gs) + { + Runtime.PyGILState_Release(gs); + } + + + /// + /// BeginAllowThreads Method + /// + /// + /// Release the Python global interpreter lock to allow other threads + /// to run. This is equivalent to the Py_BEGIN_ALLOW_THREADS macro + /// provided by the C Python API. + /// For more information, see the "Extending and Embedding" section + /// of the Python documentation on www.python.org. + /// + public static unsafe IntPtr BeginAllowThreads() + { + return (IntPtr)Runtime.PyEval_SaveThread(); + } + + + /// + /// EndAllowThreads Method + /// + /// + /// Re-aquire the Python global interpreter lock for the current + /// thread. This is equivalent to the Py_END_ALLOW_THREADS macro + /// provided by the C Python API. + /// For more information, see the "Extending and Embedding" section + /// of the Python documentation on www.python.org. + /// + public static unsafe void EndAllowThreads(IntPtr ts) + { + Runtime.PyEval_RestoreThread((PyThreadState*)ts); + } + + public static PyObject Compile(string code, string filename = "", RunFlagType mode = RunFlagType.File) + { + var flag = (int)mode; + NewReference ptr = Runtime.Py_CompileString(code, filename, flag); + PythonException.ThrowIfIsNull(ptr); + return ptr.MoveToPyObject(); + } + + + /// + /// Eval Method + /// + /// + /// Evaluate a Python expression and returns the result. + /// It's a subset of Python eval function. + /// + public static PyObject Eval(string code, PyDict? globals = null, PyObject? locals = null) + { + PyObject result = RunString(code, globals.BorrowNullable(), locals.BorrowNullable(), RunFlagType.Eval); + return result; + } + + + /// + /// Exec Method + /// + /// + /// Run a string containing Python code. + /// It's a subset of Python exec function. + /// + public static void Exec(string code, PyDict? globals = null, PyObject? locals = null) + { + using PyObject result = RunString(code, globals.BorrowNullable(), locals.BorrowNullable(), RunFlagType.File); + if (result.obj != Runtime.PyNone) + { + throw PythonException.ThrowLastAsClrException(); + } + } + /// + /// Exec Method + /// + /// + /// Run a string containing Python code. + /// It's a subset of Python exec function. + /// + internal static void Exec(string code, BorrowedReference globals, BorrowedReference locals = default) + { + using PyObject result = RunString(code, globals: globals, locals: locals, RunFlagType.File); + if (result.obj != Runtime.PyNone) + { + throw PythonException.ThrowLastAsClrException(); + } + } + + /// + /// Gets the Python thread ID. + /// + /// The Python thread ID. + public static ulong GetPythonThreadID() + { + using PyObject threading = Py.Import("threading"); + using PyObject id = threading.InvokeMethod("get_ident"); + return id.As(); + } + + /// + /// Interrupts the execution of a thread. + /// + /// The Python thread ID. + /// The number of thread states modified; this is normally one, but will be zero if the thread id is not found. + public static int Interrupt(ulong pythonThreadID) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return Runtime.PyThreadState_SetAsyncExcLLP64((uint)pythonThreadID, Exceptions.KeyboardInterrupt); + } + + return Runtime.PyThreadState_SetAsyncExcLP64(pythonThreadID, Exceptions.KeyboardInterrupt); + } + + /// + /// RunString Method. Function has been deprecated and will be removed. + /// Use Exec/Eval/RunSimpleString instead. + /// + [Obsolete("RunString is deprecated and will be removed. Use Exec/Eval/RunSimpleString instead.")] + public static PyObject RunString(string code, PyDict? globals = null, PyObject? locals = null) + { + return RunString(code, globals.BorrowNullable(), locals.BorrowNullable(), RunFlagType.File); + } + + /// + /// Internal RunString Method. + /// + /// + /// Run a string containing Python code. Returns the result of + /// executing the code string as a PyObject instance, or null if + /// an exception was raised. + /// + internal static PyObject RunString(string code, BorrowedReference globals, BorrowedReference locals, RunFlagType flag) + { + if (code is null) throw new ArgumentNullException(nameof(code)); + + NewReference tempGlobals = default; + if (globals.IsNull) + { + globals = Runtime.PyEval_GetGlobals(); + if (globals.IsNull) + { + tempGlobals = Runtime.PyDict_New(); + globals = tempGlobals.BorrowOrThrow(); + Runtime.PyDict_SetItem( + globals, PyIdentifier.__builtins__, + Runtime.PyEval_GetBuiltins() + ); + } + } + + if (locals == null) + { + locals = globals; + } + + try + { + NewReference result = Runtime.PyRun_String( + code, flag, globals, locals + ); + PythonException.ThrowIfIsNull(result); + return result.MoveToPyObject(); + } + finally + { + tempGlobals.Dispose(); + } + } + } + + public enum RunFlagType : int + { + Single = 256, + File = 257, /* Py_file_input */ + Eval = 258 + } +} +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + /// + /// This class provides the public interface of the Python runtime. + /// + public class PythonEngine : IDisposable + { + private static DelegateManager? delegateManager; + private static bool initialized; + private static IntPtr _pythonHome = IntPtr.Zero; + private static IntPtr _programName = IntPtr.Zero; + private static IntPtr _pythonPath = IntPtr.Zero; + private static InteropConfiguration interopConfiguration = InteropConfiguration.MakeDefault(); + + public PythonEngine() + { + Initialize(); + } + + public PythonEngine(params string[] args) + { + Initialize(args); + } + + public PythonEngine(IEnumerable args) + { + Initialize(args); + } + + public void Dispose() + { + Shutdown(); + } + + public static bool IsInitialized + { + get { return initialized; } + } + + private static void EnsureInitialized() + { + if (!IsInitialized) + throw new InvalidOperationException( + "Python must be initialized for this operation" + ); + } + + /// Set to true to enable GIL debugging assistance. + public static bool DebugGIL { get; set; } = false; + + internal static DelegateManager DelegateManager + { + get + { + if (delegateManager == null) + { + throw new InvalidOperationException( + "DelegateManager has not yet been initialized using Python.Runtime.PythonEngine.Initialize()."); + } + return delegateManager; + } + } + + public static InteropConfiguration InteropConfiguration + { + get => interopConfiguration; + set + { + if (IsInitialized) + throw new NotSupportedException("Changing interop configuration when engine is running is not supported"); + + interopConfiguration = value ?? throw new ArgumentNullException(nameof(InteropConfiguration)); + } + } + + public static string ProgramName + { + get + { + IntPtr p = Runtime.TryUsingDll(() => Runtime.Py_GetProgramName()); + return UcsMarshaler.PtrToPy3UnicodePy2String(p) ?? ""; + } + set + { + Marshal.FreeHGlobal(_programName); + _programName = Runtime.TryUsingDll( + () => UcsMarshaler.Py3UnicodePy2StringtoPtr(value) + ); + Runtime.Py_SetProgramName(_programName); + } + } + + public static string PythonHome + { + get + { + EnsureInitialized(); + IntPtr p = Runtime.TryUsingDll(() => Runtime.Py_GetPythonHome()); + return UcsMarshaler.PtrToPy3UnicodePy2String(p) ?? ""; + } + set + { + // this value is null in the beginning + Marshal.FreeHGlobal(_pythonHome); + _pythonHome = UcsMarshaler.Py3UnicodePy2StringtoPtr(value); + Runtime.TryUsingDll(() => Runtime.Py_SetPythonHome(_pythonHome)); + } + } + + public static string PythonPath + { + get + { + IntPtr p = Runtime.TryUsingDll(() => Runtime.Py_GetPath()); + return UcsMarshaler.PtrToPy3UnicodePy2String(p) ?? ""; + } + set + { + Marshal.FreeHGlobal(_pythonPath); + _pythonPath = Runtime.TryUsingDll( + () => UcsMarshaler.Py3UnicodePy2StringtoPtr(value) + ); + Runtime.Py_SetPath(_pythonPath); + } + } + public static Version MinSupportedVersion => new(3, 7); public static Version MaxSupportedVersion => new(3, 12, int.MaxValue, int.MaxValue); public static bool IsSupportedVersion(Version version) => version >= MinSupportedVersion && version <= MaxSupportedVersion; diff --git a/src/runtime/PythonTypes/PyInt.cs b/src/runtime/PythonTypes/PyInt.cs index 0d00f5a13..e33de2958 100644 --- a/src/runtime/PythonTypes/PyInt.cs +++ b/src/runtime/PythonTypes/PyInt.cs @@ -3,6 +3,242 @@ using System.Numerics; using System.Runtime.Serialization; +namespace Python.Runtime +{ + /// + /// Represents a Python integer object. + /// See the documentation at https://docs.python.org/3/c-api/long.html + /// + public class PyInt : PyNumber, IFormattable + { + internal PyInt(in StolenReference ptr) : base(ptr) + { + } + + internal PyInt(BorrowedReference reference) : base(reference) + { + if (!Runtime.PyInt_Check(reference)) throw new ArgumentException("object is not an int"); + } + + + /// + /// PyInt Constructor + /// + /// + /// Copy constructor - obtain a PyInt from a generic PyObject. An + /// ArgumentException will be thrown if the given object is not a + /// Python int object. + /// + public PyInt(PyObject o) : base(FromObject(o)) + { + } + + private static BorrowedReference FromObject(PyObject o) + { + if (o is null) throw new ArgumentNullException(nameof(o)); + if (!IsIntType(o)) + { + throw new ArgumentException("object is not an int"); + } + return o.Reference; + } + + /// + /// PyInt Constructor + /// + /// + /// Creates a new Python int from an int32 value. + /// + public PyInt(int value) : base(Runtime.PyInt_FromInt32(value).StealOrThrow()) + { + } + + + /// + /// PyInt Constructor + /// + /// + /// Creates a new Python int from a uint32 value. + /// + public PyInt(uint value) : this((long)value) + { + } + + + /// + /// PyInt Constructor + /// + /// + /// Creates a new Python int from an int64 value. + /// + public PyInt(long value) : base(Runtime.PyInt_FromInt64(value).StealOrThrow()) + { + } + + /// + /// Creates a new Python int from a value. + /// + public PyInt(ulong value) : base(Runtime.PyLong_FromUnsignedLongLong(value).StealOrThrow()) + { + } + + /// + /// PyInt Constructor + /// + /// + /// Creates a new Python int from an int16 value. + /// + public PyInt(short value) : this((int)value) + { + } + + + /// + /// PyInt Constructor + /// + /// + /// Creates a new Python int from a uint16 value. + /// + public PyInt(ushort value) : this((int)value) + { + } + + + /// + /// PyInt Constructor + /// + /// + /// Creates a new Python int from a byte value. + /// + public PyInt(byte value) : this((int)value) + { + } + + + /// + /// PyInt Constructor + /// + /// + /// Creates a new Python int from an sbyte value. + /// + public PyInt(sbyte value) : this((int)value) + { + } + + /// + /// PyInt Constructor + /// + /// + /// Creates a new Python int from a string value. + /// + public PyInt(string value) : base(Runtime.PyLong_FromString(value, 0).StealOrThrow()) + { + } + + public PyInt(BigInteger value) : this(value.ToString(CultureInfo.InvariantCulture)) { } + + protected PyInt(SerializationInfo info, StreamingContext context) + : base(info, context) { } + + + /// + /// IsIntType Method + /// + /// + /// Returns true if the given object is a Python int. + /// + public static bool IsIntType(PyObject value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + return Runtime.PyInt_Check(value.obj); + } + + + /// + /// Convert a Python object to a Python int if possible, raising + /// a PythonException if the conversion is not possible. This is + /// equivalent to the Python expression "int(object)". + /// + public static PyInt AsInt(PyObject value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + var op = Runtime.PyNumber_Long(value.Reference); + PythonException.ThrowIfIsNull(op); + return new PyInt(op.Steal()); + } + + + /// + /// ToInt16 Method + /// + /// + /// Return the value of the Python int object as an int16. + /// + public short ToInt16() + { + return Convert.ToInt16(ToInt32()); + } + + + /// + /// Return the value of the Python int object as an . + /// + public int ToInt32() => Converter.ToInt32(Reference); + + /// + /// ToInt64 Method + /// + /// + /// Return the value of the Python int object as an int64. + /// + public long ToInt64() + { + long? val = Runtime.PyLong_AsLongLong(obj); + if (val is null) + { + throw PythonException.ThrowLastAsClrException(); + } + return val.Value; + } + + public BigInteger ToBigInteger() + { + using var pyHex = Runtime.HexCallable.Invoke(this); + string hex = pyHex.As(); + int offset = 0; + bool neg = false; + if (hex[0] == '-') + { + offset++; + neg = true; + } + byte[] littleEndianBytes = new byte[(hex.Length - offset + 1) / 2 + 1]; + for (; offset < hex.Length; offset++) + { + int littleEndianHexIndex = hex.Length - 1 - offset; + int byteIndex = littleEndianHexIndex / 2; + int isByteTopHalf = littleEndianHexIndex & 1; + int valueShift = isByteTopHalf * 4; + littleEndianBytes[byteIndex] += (byte)(Util.HexToInt(hex[offset]) << valueShift); + } + var result = new BigInteger(littleEndianBytes); + return neg ? -result : result; + } + + public string ToString(string format, IFormatProvider formatProvider) + { + using var _ = Py.GIL(); + return ToBigInteger().ToString(format, formatProvider); + } + + public override TypeCode GetTypeCode() => TypeCode.Int64; + } +} +using System; +using System.Globalization; +using System.Numerics; +using System.Runtime.Serialization; + namespace Python.Runtime { /// diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index cf0c2a03f..8a07fee24 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -1128,6 +1128,1560 @@ public PyBuffer GetBuffer(PyBUF flags = PyBUF.SIMPLE) } + public long Refcount + { + get + { + return Runtime.Refcount(obj); + } + } + + + public override bool TryGetMember(GetMemberBinder binder, out object? result) + { + using var _ = Py.GIL(); + result = CheckNone(this.GetAttr(binder.Name)); + return true; + } + + public override bool TrySetMember(SetMemberBinder binder, object? value) + { + using var _ = Py.GIL(); + using var newVal = Converter.ToPythonDetectType(value); + int r = Runtime.PyObject_SetAttrString(obj, binder.Name, newVal.Borrow()); + if (r < 0) + { + throw PythonException.ThrowLastAsClrException(); + } + return true; + } + + private void GetArgs(object?[] inargs, CallInfo callInfo, out PyTuple args, out PyDict? kwargs) + { + if (callInfo == null || callInfo.ArgumentNames.Count == 0) + { + GetArgs(inargs, out args, out kwargs); + return; + } + + // Support for .net named arguments + var namedArgumentCount = callInfo.ArgumentNames.Count; + var regularArgumentCount = callInfo.ArgumentCount - namedArgumentCount; + + using var argTuple = Runtime.PyTuple_New(regularArgumentCount); + for (int i = 0; i < regularArgumentCount; ++i) + { + AddArgument(argTuple.Borrow(), i, inargs[i]); + } + args = new PyTuple(argTuple.Steal()); + + var namedArgs = new object?[namedArgumentCount * 2]; + for (int i = 0; i < namedArgumentCount; ++i) + { + namedArgs[i * 2] = callInfo.ArgumentNames[i]; + namedArgs[i * 2 + 1] = inargs[regularArgumentCount + i]; + } + kwargs = Py.kw(namedArgs); + } + + private void GetArgs(object?[] inargs, out PyTuple args, out PyDict? kwargs) + { + int arg_count; + for (arg_count = 0; arg_count < inargs.Length && !(inargs[arg_count] is Py.KeywordArguments); ++arg_count) + { + ; + } + using var argtuple = Runtime.PyTuple_New(arg_count); + for (var i = 0; i < arg_count; i++) + { + AddArgument(argtuple.Borrow(), i, inargs[i]); + } + args = new PyTuple(argtuple.Steal()); + + kwargs = null; + for (int i = arg_count; i < inargs.Length; i++) + { + if (inargs[i] is not Py.KeywordArguments kw) + { + throw new ArgumentException("Keyword arguments must come after normal arguments."); + } + if (kwargs == null) + { + kwargs = kw; + } + else + { + kwargs.Update(kw); + } + } + } + + private static void AddArgument(BorrowedReference argtuple, nint i, object? target) + { + using var ptr = GetPythonObject(target); + + if (Runtime.PyTuple_SetItem(argtuple, i, ptr.StealNullable()) < 0) + { + throw PythonException.ThrowLastAsClrException(); + } + } + + private static NewReference GetPythonObject(object? target) + { + if (target is PyObject pyObject) + { + return new NewReference(pyObject); + } + else + { + return Converter.ToPythonDetectType(target); + } + } + + public override bool TryInvokeMember(InvokeMemberBinder binder, object?[] args, out object? result) + { + using var _ = Py.GIL(); + if (this.HasAttr(binder.Name) && this.GetAttr(binder.Name).IsCallable()) + { + PyTuple? pyargs = null; + PyDict? kwargs = null; + try + { + GetArgs(args, binder.CallInfo, out pyargs, out kwargs); + result = CheckNone(InvokeMethod(binder.Name, pyargs, kwargs)); + } + finally + { + pyargs?.Dispose(); + kwargs?.Dispose(); + } + return true; + } + else + { + return base.TryInvokeMember(binder, args, out result); + } + } + + public override bool TryInvoke(InvokeBinder binder, object?[] args, out object? result) + { + using var _ = Py.GIL(); + if (this.IsCallable()) + { + PyTuple? pyargs = null; + PyDict? kwargs = null; + try + { + GetArgs(args, binder.CallInfo, out pyargs, out kwargs); + result = CheckNone(Invoke(pyargs, kwargs)); + } + finally + { + pyargs?.Dispose(); + kwargs?.Dispose(); + } + return true; + } + else + { + return base.TryInvoke(binder, args, out result); + } + } + + public override bool TryConvert(ConvertBinder binder, out object? result) + { + using var _ = Py.GIL(); + // always try implicit conversion first + if (Converter.ToManaged(this.obj, binder.Type, out result, false)) + { + return true; + } + + if (binder.Explicit) + { + Runtime.PyErr_Fetch(out var errType, out var errValue, out var tb); + bool converted = Converter.ToManagedExplicit(Reference, binder.Type, out result); + Runtime.PyErr_Restore(errType.StealNullable(), errValue.StealNullable(), tb.StealNullable()); + return converted; + } + + if (binder.Type == typeof(System.Collections.IEnumerable) && this.IsIterable()) + { + result = new PyIterable(this.Reference); + return true; + } + + return false; + } + + private bool TryCompare(PyObject arg, int op, out object @out) + { + int result = Runtime.PyObject_RichCompareBool(this.obj, arg.obj, op); + @out = result != 0; + if (result < 0) + { + Exceptions.Clear(); + return false; + } + return true; + } + + public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg, out object? result) + { + using var _ = Py.GIL(); + NewReference res; + if (arg is not PyObject) + { + arg = arg.ToPython(); + } + + switch (binder.Operation) + { + case ExpressionType.Add: + res = Runtime.PyNumber_Add(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.AddAssign: + res = Runtime.PyNumber_InPlaceAdd(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.Subtract: + res = Runtime.PyNumber_Subtract(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.SubtractAssign: + res = Runtime.PyNumber_InPlaceSubtract(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.Multiply: + res = Runtime.PyNumber_Multiply(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.MultiplyAssign: + res = Runtime.PyNumber_InPlaceMultiply(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.Divide: + res = Runtime.PyNumber_TrueDivide(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.DivideAssign: + res = Runtime.PyNumber_InPlaceTrueDivide(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.And: + res = Runtime.PyNumber_And(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.AndAssign: + res = Runtime.PyNumber_InPlaceAnd(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.ExclusiveOr: + res = Runtime.PyNumber_Xor(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.ExclusiveOrAssign: + res = Runtime.PyNumber_InPlaceXor(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.GreaterThan: + return this.TryCompare((PyObject)arg, Runtime.Py_GT, out result); + case ExpressionType.GreaterThanOrEqual: + return this.TryCompare((PyObject)arg, Runtime.Py_GE, out result); + case ExpressionType.LeftShift: + res = Runtime.PyNumber_Lshift(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.LeftShiftAssign: + res = Runtime.PyNumber_InPlaceLshift(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.LessThan: + return this.TryCompare((PyObject)arg, Runtime.Py_LT, out result); + case ExpressionType.LessThanOrEqual: + return this.TryCompare((PyObject)arg, Runtime.Py_LE, out result); + case ExpressionType.Modulo: + res = Runtime.PyNumber_Remainder(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.ModuloAssign: + res = Runtime.PyNumber_InPlaceRemainder(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.NotEqual: + return this.TryCompare((PyObject)arg, Runtime.Py_NE, out result); + case ExpressionType.Equal: + return this.TryCompare((PyObject)arg, Runtime.Py_EQ, out result); + case ExpressionType.Or: + res = Runtime.PyNumber_Or(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.OrAssign: + res = Runtime.PyNumber_InPlaceOr(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.Power: + res = Runtime.PyNumber_Power(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.RightShift: + res = Runtime.PyNumber_Rshift(this.obj, ((PyObject)arg).obj); + break; + case ExpressionType.RightShiftAssign: + res = Runtime.PyNumber_InPlaceRshift(this.obj, ((PyObject)arg).obj); + break; + default: + result = null; + return false; + } + Exceptions.ErrorCheck(res.BorrowNullable()); + result = CheckNone(new PyObject(res.Borrow())); + return true; + } + + public static bool operator ==(PyObject? a, PyObject? b) + { + if (a is null && b is null) + { + return true; + } + if (a is null || b is null) + { + return false; + } + + using var _ = Py.GIL(); + int result = Runtime.PyObject_RichCompareBool(a.obj, b.obj, Runtime.Py_EQ); + if (result < 0) throw PythonException.ThrowLastAsClrException(); + return result != 0; + } + + public static bool operator !=(PyObject? a, PyObject? b) + { + if (a is null && b is null) + { + return false; + } + if (a is null || b is null) + { + return true; + } + + using var _ = Py.GIL(); + int result = Runtime.PyObject_RichCompareBool(a.obj, b.obj, Runtime.Py_NE); + if (result < 0) throw PythonException.ThrowLastAsClrException(); + return result != 0; + } + + // Workaround for https://bugzilla.xamarin.com/show_bug.cgi?id=41509 + // See https://github.com/pythonnet/pythonnet/pull/219 + internal static object? CheckNone(PyObject pyObj) + { + if (pyObj != null) + { + if (pyObj.obj == Runtime.PyNone) + { + return null; + } + } + + return pyObj; + } + + public override bool TryUnaryOperation(UnaryOperationBinder binder, out object? result) + { + using var _ = Py.GIL(); + int r; + NewReference res; + switch (binder.Operation) + { + case ExpressionType.Negate: + res = Runtime.PyNumber_Negative(this.obj); + break; + case ExpressionType.UnaryPlus: + res = Runtime.PyNumber_Positive(this.obj); + break; + case ExpressionType.OnesComplement: + res = Runtime.PyNumber_Invert(this.obj); + break; + case ExpressionType.Not: + r = Runtime.PyObject_Not(this.obj); + result = r == 1; + if (r == -1) Exceptions.Clear(); + return r != -1; + case ExpressionType.IsFalse: + r = Runtime.PyObject_IsTrue(this.obj); + result = r == 0; + if (r == -1) Exceptions.Clear(); + return r != -1; + case ExpressionType.IsTrue: + r = Runtime.PyObject_IsTrue(this.obj); + result = r == 1; + if (r == -1) Exceptions.Clear(); + return r != -1; + case ExpressionType.Decrement: + case ExpressionType.Increment: + default: + result = null; + return false; + } + result = CheckNone(new PyObject(res.StealOrThrow())); + return true; + } + + /// + /// Returns the enumeration of all dynamic member names. + /// + /// + /// This method exists for debugging purposes only. + /// + /// A sequence that contains dynamic member names. + public override IEnumerable GetDynamicMemberNames() + { + using var _ = Py.GIL(); + return Dir().Select(pyObj => pyObj.ToString()!).ToArray(); + } + + void ISerializable.GetObjectData(SerializationInfo info, StreamingContext context) + => GetObjectData(info, context); + protected virtual void GetObjectData(SerializationInfo info, StreamingContext context) + { +#pragma warning disable CS0618 // Type or member is obsolete + Runtime.XIncref(this); +#pragma warning restore CS0618 // Type or member is obsolete + info.AddValue("h", rawPtr.ToInt64()); + info.AddValue("r", run); + } + + protected PyObject(SerializationInfo info, StreamingContext context) + { + rawPtr = (IntPtr)info.GetInt64("h"); + run = info.GetInt32("r"); + if (IsDisposed) GC.SuppressFinalize(this); + } + } + + internal static class PyObjectExtensions + { + internal static NewReference NewReferenceOrNull(this PyObject? self) + => self is null || self.IsDisposed ? default : new NewReference(self); + + internal static BorrowedReference BorrowNullable(this PyObject? self) + => self is null ? default : self.Reference; + } +} +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Dynamic; +using System.Linq; +using System.Linq.Expressions; +using System.Runtime.Serialization; +using System.Threading; + +namespace Python.Runtime +{ + /// + /// Represents a generic Python object. The methods of this class are + /// generally equivalent to the Python "abstract object API". See + /// PY2: https://docs.python.org/2/c-api/object.html + /// PY3: https://docs.python.org/3/c-api/object.html + /// for details. + /// + [Serializable] + [DebuggerDisplay("{" + nameof(DebuggerDisplay) + ",nq}")] + public partial class PyObject : DynamicObject, IDisposable, ISerializable + { +#if TRACE_ALLOC + /// + /// Trace stack for PyObject's construction + /// + public StackTrace Traceback { get; } = new StackTrace(1); +#endif + + protected IntPtr rawPtr = IntPtr.Zero; + internal readonly int run = Runtime.GetRun(); + + internal BorrowedReference obj => new (rawPtr); + + public static PyObject None => new (Runtime.PyNone); + internal BorrowedReference Reference => new (rawPtr); + + /// + /// PyObject Constructor + /// + /// + /// Creates a new PyObject from an IntPtr object reference. Note that + /// the PyObject instance assumes ownership of the object reference + /// and the reference will be DECREFed when the PyObject is garbage + /// collected or explicitly disposed. + /// + [Obsolete] + internal PyObject(IntPtr ptr) + { + if (ptr == IntPtr.Zero) throw new ArgumentNullException(nameof(ptr)); + + rawPtr = ptr; + Finalizer.Instance.ThrottledCollect(); + } + + [Obsolete("for testing purposes only")] + internal PyObject(IntPtr ptr, bool skipCollect) + { + if (ptr == IntPtr.Zero) throw new ArgumentNullException(nameof(ptr)); + + rawPtr = ptr; + if (!skipCollect) + Finalizer.Instance.ThrottledCollect(); + } + + /// + /// Creates new pointing to the same object as + /// the . Increments refcount, allowing + /// to have ownership over its own reference. + /// + internal PyObject(BorrowedReference reference) + { + if (reference.IsNull) throw new ArgumentNullException(nameof(reference)); + + rawPtr = new NewReference(reference).DangerousMoveToPointer(); + Finalizer.Instance.ThrottledCollect(); + } + + internal PyObject(BorrowedReference reference, bool skipCollect) + { + if (reference.IsNull) throw new ArgumentNullException(nameof(reference)); + + rawPtr = new NewReference(reference).DangerousMoveToPointer(); + if (!skipCollect) + Finalizer.Instance.ThrottledCollect(); + } + + internal PyObject(in StolenReference reference) + { + if (reference == null) throw new ArgumentNullException(nameof(reference)); + + rawPtr = reference.DangerousGetAddressOrNull(); + Finalizer.Instance.ThrottledCollect(); + } + + // Ensure that encapsulated Python object is decref'ed appropriately + // when the managed wrapper is garbage-collected. + ~PyObject() + { + if (!IsDisposed) + { + +#if TRACE_ALLOC + CheckRun(); +#endif + + Interlocked.Increment(ref Runtime._collected); + + Finalizer.Instance.AddFinalizedObject(ref rawPtr, run +#if TRACE_ALLOC + , Traceback +#endif + ); + } + + Dispose(false); + } + + + /// + /// Gets the native handle of the underlying Python object. This + /// value is generally for internal use by the PythonNet runtime. + /// + [Obsolete] + public IntPtr Handle + { + get { return rawPtr; } + } + + + /// + /// Gets raw Python proxy for this object (bypasses all conversions, + /// except null <==> None) + /// + /// + /// Given an arbitrary managed object, return a Python instance that + /// reflects the managed object. + /// + public static PyObject FromManagedObject(object ob) + { + // Special case: if ob is null, we return None. + if (ob == null) + { + return new PyObject(Runtime.PyNone); + } + return CLRObject.GetReference(ob).MoveToPyObject(); + } + + /// + /// Creates new from a nullable reference. + /// When is null, null is returned. + /// + internal static PyObject? FromNullableReference(BorrowedReference reference) + => reference.IsNull ? null : new PyObject(reference); + + + /// + /// AsManagedObject Method + /// + /// + /// Return a managed object of the given type, based on the + /// value of the Python object. + /// + public object? AsManagedObject(Type t) + { + if (!Converter.ToManaged(obj, t, out var result, true)) + { + throw new InvalidCastException("cannot convert object to target type", + PythonException.FetchCurrentOrNull(out _)); + } + return result; + } + + /// + /// Return a managed object of the given type, based on the + /// value of the Python object. + /// + public T As() => (T)this.AsManagedObject(typeof(T))!; + + internal bool IsDisposed => rawPtr == IntPtr.Zero; + + void CheckDisposed() + { + if (IsDisposed) throw new ObjectDisposedException(nameof(PyObject)); + } + + protected virtual void Dispose(bool disposing) + { + if (IsDisposed) + { + return; + } + + if (Runtime.Py_IsInitialized() == 0 && Runtime._Py_IsFinalizing() != true) + { + throw new InvalidOperationException("Python runtime must be initialized"); + } + + nint refcount = Runtime.Refcount(this.obj); + Debug.Assert(refcount > 0, "Object refcount is 0 or less"); + + if (refcount == 1) + { + Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback); + + try + { + Runtime.XDecref(StolenReference.Take(ref rawPtr)); + Runtime.CheckExceptionOccurred(); + } + finally + { + // Python requires finalizers to preserve exception: + // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation + Runtime.PyErr_Restore(errType.StealNullable(), errVal.StealNullable(), traceback.StealNullable()); + } + } + else + { + Runtime.XDecref(StolenReference.Take(ref rawPtr)); + } + + this.rawPtr = IntPtr.Zero; + } + + /// + /// The Dispose method provides a way to explicitly release the + /// Python object represented by a PyObject instance. It is a good + /// idea to call Dispose on PyObjects that wrap resources that are + /// limited or need strict lifetime control. Otherwise, references + /// to Python objects will not be released until a managed garbage + /// collection occurs. + /// + public void Dispose() + { + GC.SuppressFinalize(this); + Dispose(true); + + } + + internal StolenReference Steal() + { + GC.SuppressFinalize(this); + return StolenReference.Take(ref this.rawPtr); + } + + [Obsolete("Test use only")] + internal void Leak() + { + Debug.Assert(!IsDisposed); + GC.SuppressFinalize(this); + rawPtr = IntPtr.Zero; + } + + internal IntPtr DangerousGetAddressOrNull() => rawPtr; + + internal void CheckRun() + { + if (run != Runtime.GetRun()) + throw new RuntimeShutdownException(rawPtr); + } + + internal BorrowedReference GetPythonTypeReference() + => Runtime.PyObject_TYPE(obj); + + /// + /// GetPythonType Method + /// + /// + /// Returns the Python type of the object. This method is equivalent + /// to the Python expression: type(object). + /// + public PyType GetPythonType() + { + var tp = Runtime.PyObject_Type(Reference); + return new PyType(tp.StealOrThrow(), prevalidated: true); + } + + + /// + /// TypeCheck Method + /// + /// + /// Returns true if the object o is of type typeOrClass or a subtype + /// of typeOrClass. + /// + public bool TypeCheck(PyType typeOrClass) + { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + + return Runtime.PyObject_TypeCheck(obj, typeOrClass.obj); + } + + internal PyType PyType => this.GetPythonType(); + + + /// + /// HasAttr Method + /// + /// + /// Returns true if the object has an attribute with the given name. + /// + public bool HasAttr(string name) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + + return Runtime.PyObject_HasAttrString(Reference, name) != 0; + } + + + /// + /// HasAttr Method + /// + /// + /// Returns true if the object has an attribute with the given name, + /// where name is a PyObject wrapping a string or unicode object. + /// + public bool HasAttr(PyObject name) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + + return Runtime.PyObject_HasAttr(Reference, name.Reference) != 0; + } + + + /// + /// GetAttr Method + /// + /// + /// Returns the named attribute of the Python object, or raises a + /// PythonException if the attribute access fails. + /// + public PyObject GetAttr(string name) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + + using var op = Runtime.PyObject_GetAttrString(obj, name); + return new PyObject(op.StealOrThrow()); + } + + + /// + /// Returns the named attribute of the Python object, or the given + /// default object if the attribute access throws AttributeError. + /// + /// + /// This method ignores any AttrubiteError(s), even ones + /// not raised due to missing requested attribute. + /// + /// For example, if attribute getter calls other Python code, and + /// that code happens to cause AttributeError elsewhere, it will be ignored + /// and value will be returned instead. + /// + /// Name of the attribute. + /// The object to return on AttributeError. + [Obsolete("See remarks")] + public PyObject GetAttr(string name, PyObject _default) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + + using var op = Runtime.PyObject_GetAttrString(obj, name); + if (op.IsNull()) + { + if (Exceptions.ExceptionMatches(Exceptions.AttributeError)) + { + Runtime.PyErr_Clear(); + return _default; + } + else + { + throw PythonException.ThrowLastAsClrException(); + } + } + return new PyObject(op.Steal()); + } + + + /// + /// GetAttr Method + /// + /// + /// Returns the named attribute of the Python object or raises a + /// PythonException if the attribute access fails. The name argument + /// is a PyObject wrapping a Python string or unicode object. + /// + public PyObject GetAttr(PyObject name) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + + using var op = Runtime.PyObject_GetAttr(obj, name.obj); + return new PyObject(op.StealOrThrow()); + } + + + /// + /// Returns the named attribute of the Python object, or the given + /// default object if the attribute access throws AttributeError. + /// + /// + /// This method ignores any AttrubiteError(s), even ones + /// not raised due to missing requested attribute. + /// + /// For example, if attribute getter calls other Python code, and + /// that code happens to cause AttributeError elsewhere, it will be ignored + /// and value will be returned instead. + /// + /// Name of the attribute. Must be of Python type 'str'. + /// The object to return on AttributeError. + [Obsolete("See remarks")] + public PyObject GetAttr(PyObject name, PyObject _default) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + + using var op = Runtime.PyObject_GetAttr(obj, name.obj); + if (op.IsNull()) + { + if (Exceptions.ExceptionMatches(Exceptions.AttributeError)) + { + Runtime.PyErr_Clear(); + return _default; + } + else + { + throw PythonException.ThrowLastAsClrException(); + } + } + return new PyObject(op.Steal()); + } + + + /// + /// SetAttr Method + /// + /// + /// Set an attribute of the object with the given name and value. This + /// method throws a PythonException if the attribute set fails. + /// + public void SetAttr(string name, PyObject value) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (value == null) throw new ArgumentNullException(nameof(value)); + + int r = Runtime.PyObject_SetAttrString(obj, name, value.obj); + if (r < 0) + { + throw PythonException.ThrowLastAsClrException(); + } + } + + + /// + /// SetAttr Method + /// + /// + /// Set an attribute of the object with the given name and value, + /// where the name is a Python string or unicode object. This method + /// throws a PythonException if the attribute set fails. + /// + public void SetAttr(PyObject name, PyObject value) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (value == null) throw new ArgumentNullException(nameof(value)); + + int r = Runtime.PyObject_SetAttr(obj, name.obj, value.obj); + if (r < 0) + { + throw PythonException.ThrowLastAsClrException(); + } + } + + + /// + /// DelAttr Method + /// + /// + /// Delete the named attribute of the Python object. This method + /// throws a PythonException if the attribute set fails. + /// + public void DelAttr(string name) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + + int r = Runtime.PyObject_DelAttrString(obj, name); + if (r < 0) + { + throw PythonException.ThrowLastAsClrException(); + } + } + + + /// + /// DelAttr Method + /// + /// + /// Delete the named attribute of the Python object, where name is a + /// PyObject wrapping a Python string or unicode object. This method + /// throws a PythonException if the attribute set fails. + /// + public void DelAttr(PyObject name) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + + int r = Runtime.PyObject_DelAttr(obj, name.obj); + if (r < 0) + { + throw PythonException.ThrowLastAsClrException(); + } + } + + + /// + /// GetItem Method + /// + /// + /// For objects that support the Python sequence or mapping protocols, + /// return the item at the given object index. This method raises a + /// PythonException if the indexing operation fails. + /// + public virtual PyObject GetItem(PyObject key) + { + if (key == null) throw new ArgumentNullException(nameof(key)); + + using var op = Runtime.PyObject_GetItem(obj, key.obj); + if (op.IsNull()) + { + throw PythonException.ThrowLastAsClrException(); + } + return new PyObject(op.Steal()); + } + + + /// + /// GetItem Method + /// + /// + /// For objects that support the Python sequence or mapping protocols, + /// return the item at the given string index. This method raises a + /// PythonException if the indexing operation fails. + /// + public virtual PyObject GetItem(string key) + { + if (key == null) throw new ArgumentNullException(nameof(key)); + + using var pyKey = new PyString(key); + return GetItem(pyKey); + } + + + /// + /// GetItem Method + /// + /// + /// For objects that support the Python sequence or mapping protocols, + /// return the item at the given numeric index. This method raises a + /// PythonException if the indexing operation fails. + /// + public virtual PyObject GetItem(int index) + { + using var key = new PyInt(index); + return GetItem(key); + } + + + /// + /// SetItem Method + /// + /// + /// For objects that support the Python sequence or mapping protocols, + /// set the item at the given object index to the given value. This + /// method raises a PythonException if the set operation fails. + /// + public virtual void SetItem(PyObject key, PyObject value) + { + if (key == null) throw new ArgumentNullException(nameof(key)); + if (value == null) throw new ArgumentNullException(nameof(value)); + + int r = Runtime.PyObject_SetItem(obj, key.obj, value.obj); + if (r < 0) + { + throw PythonException.ThrowLastAsClrException(); + } + } + + + /// + /// SetItem Method + /// + /// + /// For objects that support the Python sequence or mapping protocols, + /// set the item at the given string index to the given value. This + /// method raises a PythonException if the set operation fails. + /// + public virtual void SetItem(string key, PyObject value) + { + if (key == null) throw new ArgumentNullException(nameof(key)); + if (value == null) throw new ArgumentNullException(nameof(value)); + + using var pyKey = new PyString(key); + SetItem(pyKey, value); + } + + + /// + /// SetItem Method + /// + /// + /// For objects that support the Python sequence or mapping protocols, + /// set the item at the given numeric index to the given value. This + /// method raises a PythonException if the set operation fails. + /// + public virtual void SetItem(int index, PyObject value) + { + if (value == null) throw new ArgumentNullException(nameof(value)); + + using var pyindex = new PyInt(index); + SetItem(pyindex, value); + } + + + /// + /// DelItem Method + /// + /// + /// For objects that support the Python sequence or mapping protocols, + /// delete the item at the given object index. This method raises a + /// PythonException if the delete operation fails. + /// + public virtual void DelItem(PyObject key) + { + if (key == null) throw new ArgumentNullException(nameof(key)); + + int r = Runtime.PyObject_DelItem(obj, key.obj); + if (r < 0) + { + throw PythonException.ThrowLastAsClrException(); + } + } + + + /// + /// DelItem Method + /// + /// + /// For objects that support the Python sequence or mapping protocols, + /// delete the item at the given string index. This method raises a + /// PythonException if the delete operation fails. + /// + public virtual void DelItem(string key) + { + if (key == null) throw new ArgumentNullException(nameof(key)); + + using var pyKey = new PyString(key); + DelItem(pyKey); + } + + + /// + /// DelItem Method + /// + /// + /// For objects that support the Python sequence or mapping protocols, + /// delete the item at the given numeric index. This method raises a + /// PythonException if the delete operation fails. + /// + public virtual void DelItem(int index) + { + using var pyindex = new PyInt(index); + DelItem(pyindex); + } + + + /// + /// Returns the length for objects that support the Python sequence + /// protocol. + /// + public virtual long Length() + { + var s = Runtime.PyObject_Size(Reference); + if (s < 0) + { + throw PythonException.ThrowLastAsClrException(); + } + return s; + } + + + /// + /// String Indexer + /// + /// + /// Provides a shorthand for the string versions of the GetItem and + /// SetItem methods. + /// + public virtual PyObject this[string key] + { + get { return GetItem(key); } + set { SetItem(key, value); } + } + + + /// + /// PyObject Indexer + /// + /// + /// Provides a shorthand for the object versions of the GetItem and + /// SetItem methods. + /// + public virtual PyObject this[PyObject key] + { + get { return GetItem(key); } + set { SetItem(key, value); } + } + + + /// + /// Numeric Indexer + /// + /// + /// Provides a shorthand for the numeric versions of the GetItem and + /// SetItem methods. + /// + public virtual PyObject this[int index] + { + get { return GetItem(index); } + set { SetItem(index, value); } + } + + + /// + /// Return a new (Python) iterator for the object. This is equivalent + /// to the Python expression "iter(object)". + /// + /// Thrown if the object can not be iterated. + public PyIter GetIterator() => PyIter.GetIter(this); + + /// + /// Invoke Method + /// + /// + /// Invoke the callable object with the given arguments, passed as a + /// PyObject[]. A PythonException is raised if the invocation fails. + /// + public PyObject Invoke(params PyObject[] args) + { + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + + var t = new PyTuple(args); + using var r = Runtime.PyObject_Call(obj, t.obj, null); + t.Dispose(); + return new PyObject(r.StealOrThrow()); + } + + + /// + /// Invoke Method + /// + /// + /// Invoke the callable object with the given arguments, passed as a + /// Python tuple. A PythonException is raised if the invocation fails. + /// + public PyObject Invoke(PyTuple args) + { + if (args == null) throw new ArgumentNullException(nameof(args)); + + using var r = Runtime.PyObject_Call(obj, args.obj, null); + return new PyObject(r.StealOrThrow()); + } + + + /// + /// Invoke Method + /// + /// + /// Invoke the callable object with the given positional and keyword + /// arguments. A PythonException is raised if the invocation fails. + /// + public PyObject Invoke(PyObject[] args, PyDict? kw) + { + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + + using var t = new PyTuple(args); + using var r = Runtime.PyObject_Call(obj, t.obj, kw is null ? null : kw.obj); + return new PyObject(r.StealOrThrow()); + } + + + /// + /// Invoke Method + /// + /// + /// Invoke the callable object with the given positional and keyword + /// arguments. A PythonException is raised if the invocation fails. + /// + public PyObject Invoke(PyTuple args, PyDict? kw) + { + if (args == null) throw new ArgumentNullException(nameof(args)); + + using var r = Runtime.PyObject_Call(obj, args.obj, kw is null ? null : kw.obj); + return new PyObject(r.StealOrThrow()); + } + + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(string name, params PyObject[] args) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args); + method.Dispose(); + return result; + } + + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(string name, PyTuple args) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args); + method.Dispose(); + return result; + } + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(PyObject name, params PyObject[] args) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args); + method.Dispose(); + return result; + } + + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(PyObject name, PyTuple args) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args); + method.Dispose(); + return result; + } + + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments + /// and keyword arguments. Keyword args are passed as a PyDict object. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(string name, PyObject[] args, PyDict? kw) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args, kw); + method.Dispose(); + return result; + } + + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments + /// and keyword arguments. Keyword args are passed as a PyDict object. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(string name, PyTuple args, PyDict? kw) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args, kw); + method.Dispose(); + return result; + } + + + /// + /// IsInstance Method + /// + /// + /// Return true if the object is an instance of the given Python type + /// or class. This method always succeeds. + /// + public bool IsInstance(PyObject typeOrClass) + { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + + int r = Runtime.PyObject_IsInstance(obj, typeOrClass.obj); + if (r < 0) + { + Runtime.PyErr_Clear(); + return false; + } + return r != 0; + } + + + /// + /// Return true if the object is identical to or derived from the + /// given Python type or class. This method always succeeds. + /// + public bool IsSubclass(PyObject typeOrClass) + { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + + return IsSubclass(typeOrClass.Reference); + } + + internal bool IsSubclass(BorrowedReference typeOrClass) + { + if (typeOrClass.IsNull) throw new ArgumentNullException(nameof(typeOrClass)); + + int r = Runtime.PyObject_IsSubclass(Reference, typeOrClass); + if (r < 0) + { + Runtime.PyErr_Clear(); + return false; + } + return r != 0; + } + + + /// + /// IsCallable Method + /// + /// + /// Returns true if the object is a callable object. This method + /// always succeeds. + /// + public bool IsCallable() + { + return Runtime.PyCallable_Check(obj) != 0; + } + + + /// + /// IsIterable Method + /// + /// + /// Returns true if the object is iterable object. This method + /// always succeeds. + /// + public bool IsIterable() + { + return Runtime.PyObject_IsIterable(obj); + } + + + /// + /// IsTrue Method + /// + /// + /// Return true if the object is true according to Python semantics. + /// This method always succeeds. + /// + public bool IsTrue() + { + return Runtime.PyObject_IsTrue(obj) != 0; + } + + /// + /// Return true if the object is None + /// + public bool IsNone() => CheckNone(this) == null; + + /// + /// Dir Method + /// + /// + /// Return a list of the names of the attributes of the object. This + /// is equivalent to the Python expression "dir(object)". + /// + public PyList Dir() + { + using var r = Runtime.PyObject_Dir(obj); + return new PyList(r.StealOrThrow()); + } + + + /// + /// Repr Method + /// + /// + /// Return a string representation of the object. This method is + /// the managed equivalent of the Python expression "repr(object)". + /// + public string? Repr() + { + using var strval = Runtime.PyObject_Repr(obj); + return Runtime.GetManagedString(strval.BorrowOrThrow()); + } + + + /// + /// ToString Method + /// + /// + /// Return the string representation of the object. This method is + /// the managed equivalent of the Python expression "str(object)". + /// + public override string? ToString() + { + using var _ = Py.GIL(); + using var strval = Runtime.PyObject_Str(obj); + return Runtime.GetManagedString(strval.BorrowOrThrow()); + } + + ManagedType? InternalManagedObject => ManagedType.GetManagedObject(this.Reference); + + string? DebuggerDisplay + { + get + { + if (DebugUtil.HaveInterpreterLock()) + return this.ToString(); + var obj = this.InternalManagedObject; + return obj is { } + ? obj.ToString() + : $"pyobj at 0x{this.rawPtr:X} (get Py.GIL to see more info)"; + } + } + + + /// + /// Equals Method + /// + /// + /// Return true if this object is equal to the given object. This + /// method is based on Python equality semantics. + /// + public override bool Equals(object o) + { + using var _ = Py.GIL(); + return Equals(o as PyObject); + } + + public virtual bool Equals(PyObject? other) + { + if (other is null) return false; + + if (obj == other.obj) + { + return true; + } + int result = Runtime.PyObject_RichCompareBool(obj, other.obj, Runtime.Py_EQ); + if (result < 0) throw PythonException.ThrowLastAsClrException(); + return result != 0; + } + + + /// + /// GetHashCode Method + /// + /// + /// Return a hashcode based on the Python object. This returns the + /// hash as computed by Python, equivalent to the Python expression + /// "hash(obj)". + /// + public override int GetHashCode() + { + using var _ = Py.GIL(); + nint pyHash = Runtime.PyObject_Hash(obj); + if (pyHash == -1 && Exceptions.ErrorOccurred()) + { + throw PythonException.ThrowLastAsClrException(); + } + return pyHash.GetHashCode(); + } + + /// + /// GetBuffer Method. This Method only works for objects that have a buffer (like "bytes", "bytearray" or "array.array") + /// + /// + /// Send a request to the PyObject to fill in view as specified by flags. If the PyObject cannot provide a buffer of the exact type, it MUST raise PyExc_BufferError, set view->obj to NULL and return -1. + /// On success, fill in view, set view->obj to a new reference to exporter and return 0. In the case of chained buffer providers that redirect requests to a single object, view->obj MAY refer to this object instead of exporter(See Buffer Object Structures). + /// Successful calls to must be paired with calls to , similar to malloc() and free(). Thus, after the consumer is done with the buffer, must be called exactly once. + /// + public PyBuffer GetBuffer(PyBUF flags = PyBUF.SIMPLE) + { + CheckDisposed(); + return new PyBuffer(this, flags); + } + + public long Refcount { get diff --git a/src/runtime/PythonTypes/PyType.cs b/src/runtime/PythonTypes/PyType.cs index 28bda5d3e..036243616 100644 --- a/src/runtime/PythonTypes/PyType.cs +++ b/src/runtime/PythonTypes/PyType.cs @@ -4,6 +4,168 @@ using Python.Runtime.Native; +namespace Python.Runtime +{ + [Serializable] + public class PyType : PyObject + { + /// Creates heap type object from the . + public PyType(TypeSpec spec, PyTuple? bases = null) : base(FromSpec(spec, bases)) { } + /// Wraps an existing type object. + public PyType(PyObject o) : base(FromObject(o)) { } + + internal PyType(PyType o) + : base(o is not null ? o.Reference : throw new ArgumentNullException(nameof(o))) + { + } + + internal PyType(BorrowedReference reference, bool prevalidated = false) : base(reference) + { + if (prevalidated) return; + + if (!Runtime.PyType_Check(this)) + throw new ArgumentException("object is not a type"); + } + + internal PyType(in StolenReference reference, bool prevalidated = false) : base(reference) + { + if (prevalidated) return; + + if (!Runtime.PyType_Check(this)) + throw new ArgumentException("object is not a type"); + } + + protected PyType(SerializationInfo info, StreamingContext context) : base(info, context) { } + + internal new static PyType? FromNullableReference(BorrowedReference reference) + => reference == null + ? null + : new PyType(new NewReference(reference).Steal()); + + internal static PyType FromReference(BorrowedReference reference) + => FromNullableReference(reference) ?? throw new ArgumentNullException(nameof(reference)); + + public string Name + { + get + { + var namePtr = new StrPtr + { + RawPointer = Util.ReadIntPtr(this, TypeOffset.tp_name), + }; + return namePtr.ToString(System.Text.Encoding.UTF8)!; + } + } + + /// Returns true when type is fully initialized + public bool IsReady => Flags.HasFlag(TypeFlags.Ready); + + internal TypeFlags Flags + { + get => (TypeFlags)Util.ReadCLong(this, TypeOffset.tp_flags); + set => Util.WriteCLong(this, TypeOffset.tp_flags, (long)value); + } + + internal PyDict Dict => new(Util.ReadRef(this, TypeOffset.tp_dict)); + + internal PyTuple MRO => new(GetMRO(this)); + + /// Checks if specified object is a Python type. + public static bool IsType(PyObject value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + + return Runtime.PyType_Check(value.obj); + } + /// Checks if specified object is a Python type. + internal static bool IsType(BorrowedReference value) + { + return Runtime.PyType_Check(value); + } + + /// + /// Gets , which represents the specified CLR type. + /// + public static PyType Get(Type clrType) + { + if (clrType is null) + { + throw new ArgumentNullException(nameof(clrType)); + } + + return new PyType(ClassManager.GetClass(clrType)); + } + + internal BorrowedReference BaseReference + { + get => GetBase(Reference); + set => Runtime.ReplaceReference(this, TypeOffset.tp_base, new NewReference(value).Steal()); + } + + internal IntPtr GetSlot(TypeSlotID slot) + { + IntPtr result = Runtime.PyType_GetSlot(this.Reference, slot); + return Exceptions.ErrorCheckIfNull(result); + } + + internal static TypeFlags GetFlags(BorrowedReference type) + { + Debug.Assert(TypeOffset.tp_flags > 0); + return (TypeFlags)Util.ReadCLong(type, TypeOffset.tp_flags); + } + internal static void SetFlags(BorrowedReference type, TypeFlags flags) + { + Debug.Assert(TypeOffset.tp_flags > 0); + Util.WriteCLong(type, TypeOffset.tp_flags, (long)flags); + } + + internal static BorrowedReference GetBase(BorrowedReference type) + { + Debug.Assert(IsType(type)); + return Util.ReadRef(type, TypeOffset.tp_base); + } + + internal static BorrowedReference GetBases(BorrowedReference type) + { + Debug.Assert(IsType(type)); + return Util.ReadRef(type, TypeOffset.tp_bases); + } + + internal static BorrowedReference GetMRO(BorrowedReference type) + { + Debug.Assert(IsType(type)); + return Util.ReadRef(type, TypeOffset.tp_mro); + } + + private static BorrowedReference FromObject(PyObject o) + { + if (o is null) throw new ArgumentNullException(nameof(o)); + if (!IsType(o)) throw new ArgumentException("object is not a type"); + + return o.Reference; + } + + private static StolenReference FromSpec(TypeSpec spec, PyTuple? bases = null) + { + if (spec is null) throw new ArgumentNullException(nameof(spec)); + + if ((spec.Flags & TypeFlags.HeapType) == 0) + throw new NotSupportedException("Only heap types are supported"); + + using var nativeSpec = new NativeTypeSpec(spec); + var basesRef = bases is null ? default : bases.Reference; + var result = Runtime.PyType_FromSpecWithBases(in nativeSpec, basesRef); + // Runtime.PyErr_Print(); + return result.StealOrThrow(); + } + } +} +using System; +using System.Diagnostics; +using System.Runtime.Serialization; + +using Python.Runtime.Native; + namespace Python.Runtime { [Serializable] diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index d769ebdc0..0576fd8bf 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -1,1797 +1,3643 @@ -using System; -using System.Diagnostics; -using System.Diagnostics.Contracts; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading; -using System.Collections.Generic; -using Python.Runtime.Native; -using System.Linq; -using static System.FormattableString; - -namespace Python.Runtime -{ - /// - /// Encapsulates the low-level Python C API. Note that it is - /// the responsibility of the caller to have acquired the GIL - /// before calling any of these methods. - /// - public unsafe partial class Runtime - { - public static string? PythonDLL - { - get => _PythonDll; - set - { - if (_isInitialized) - throw new InvalidOperationException("This property must be set before runtime is initialized"); - _PythonDll = value; - } - } - - static string? _PythonDll = GetDefaultDllName(); - private static string? GetDefaultDllName() - { - string dll = Environment.GetEnvironmentVariable("PYTHONNET_PYDLL"); - if (dll is not null) return dll; - - string verString = Environment.GetEnvironmentVariable("PYTHONNET_PYVER"); - if (!Version.TryParse(verString, out var version)) return null; - - return GetDefaultDllName(version); - } - - private static string GetDefaultDllName(Version version) - { - string prefix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "" : "lib"; - string suffix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? Invariant($"{version.Major}{version.Minor}") - : Invariant($"{version.Major}.{version.Minor}"); - string ext = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".dll" - : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ".dylib" - : ".so"; - return prefix + "python" + suffix + ext; - } - - private static bool _isInitialized = false; - internal static bool IsInitialized => _isInitialized; - private static bool _typesInitialized = false; - internal static bool TypeManagerInitialized => _typesInitialized; - internal static readonly bool Is32Bit = IntPtr.Size == 4; - - // Available in newer .NET Core versions (>= 5) as IntPtr.MaxValue etc. - internal static readonly long IntPtrMaxValue = Is32Bit ? Int32.MaxValue : Int64.MaxValue; - internal static readonly long IntPtrMinValue = Is32Bit ? Int32.MinValue : Int64.MinValue; - internal static readonly ulong UIntPtrMaxValue = Is32Bit ? UInt32.MaxValue : UInt64.MaxValue; - - // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - - internal static Version InteropVersion { get; } - = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version; - - public static int MainManagedThreadId { get; private set; } - - private static readonly List _pyRefs = new (); - - internal static Version PyVersion - { - get - { - var versionTuple = PySys_GetObject("version_info"); - var major = Converter.ToInt32(PyTuple_GetItem(versionTuple, 0)); - var minor = Converter.ToInt32(PyTuple_GetItem(versionTuple, 1)); - var micro = Converter.ToInt32(PyTuple_GetItem(versionTuple, 2)); - return new Version(major, minor, micro); - } - } - - const string RunSysPropName = "__pythonnet_run__"; - static int run = 0; - - internal static int GetRun() - { - int runNumber = run; - Debug.Assert(runNumber > 0, "This must only be called after Runtime is initialized at least once"); - return runNumber; - } - - internal static bool HostedInPython; - internal static bool ProcessIsTerminating; - - /// - /// Initialize the runtime... - /// - /// Always call this method from the Main thread. After the - /// first call to this method, the main thread has acquired the GIL. - internal static void Initialize(bool initSigs = false) - { - if (_isInitialized) - { - return; - } - _isInitialized = true; - - bool interpreterAlreadyInitialized = TryUsingDll( - () => Py_IsInitialized() != 0 - ); - if (!interpreterAlreadyInitialized) - { - Py_InitializeEx(initSigs ? 1 : 0); - - NewRun(); - - if (PyEval_ThreadsInitialized() == 0) - { - PyEval_InitThreads(); - } - RuntimeState.Save(); - } - else - { - if (!HostedInPython) - { - PyGILState_Ensure(); - } - - BorrowedReference pyRun = PySys_GetObject(RunSysPropName); - if (pyRun != null) - { - run = checked((int)PyLong_AsSignedSize_t(pyRun)); - } - else - { - NewRun(); - } - } - MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; - - Finalizer.Initialize(); - - InitPyMembers(); - - ABI.Initialize(PyVersion); - - InternString.Initialize(); - - GenericUtil.Reset(); - ClassManager.Reset(); - ClassDerivedObject.Reset(); - TypeManager.Initialize(); - CLRObject.creationBlocked = false; - _typesInitialized = true; - - // Initialize modules that depend on the runtime class. - AssemblyManager.Initialize(); - OperatorMethod.Initialize(); - if (RuntimeData.HasStashData()) - { - RuntimeData.RestoreRuntimeData(); - } - else - { - PyCLRMetaType = MetaType.Initialize(); - ImportHook.Initialize(); - } - Exceptions.Initialize(); - - // Need to add the runtime directory to sys.path so that we - // can find built-in assemblies like System.Data, et. al. - string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); - BorrowedReference path = PySys_GetObject("path"); - using var item = PyString_FromString(rtdir); - if (PySequence_Contains(path, item.Borrow()) == 0) - { - PyList_Append(path, item.Borrow()); - } - AssemblyManager.UpdatePath(); - - clrInterop = GetModuleLazy("clr.interop"); - inspect = GetModuleLazy("inspect"); - hexCallable = new(() => new PyString("%x").GetAttr("__mod__")); - } - - static void NewRun() - { - run++; - using var pyRun = PyLong_FromLongLong(run); - PySys_SetObject(RunSysPropName, pyRun.BorrowOrThrow()); - } - - private static void InitPyMembers() - { - using (var builtinsOwned = PyImport_ImportModule("builtins")) - { - var builtins = builtinsOwned.Borrow(); - SetPyMember(out PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented").StealNullable()); - - SetPyMember(out PyBaseObjectType, PyObject_GetAttrString(builtins, "object").StealNullable()); - - SetPyMember(out _PyNone, PyObject_GetAttrString(builtins, "None").StealNullable()); - SetPyMember(out _PyTrue, PyObject_GetAttrString(builtins, "True").StealNullable()); - SetPyMember(out _PyFalse, PyObject_GetAttrString(builtins, "False").StealNullable()); - - SetPyMemberTypeOf(out PyBoolType, _PyTrue!); - SetPyMemberTypeOf(out PyNoneType, _PyNone!); - - SetPyMemberTypeOf(out PyMethodType, PyObject_GetAttrString(builtins, "len").StealNullable()); - - // For some arcane reason, builtins.__dict__.__setitem__ is *not* - // a wrapper_descriptor, even though dict.__setitem__ is. - // - // object.__init__ seems safe, though. - SetPyMemberTypeOf(out PyWrapperDescriptorType, PyObject_GetAttrString(PyBaseObjectType, "__init__").StealNullable()); - - SetPyMember(out PySuper_Type, PyObject_GetAttrString(builtins, "super").StealNullable()); - } - - SetPyMemberTypeOf(out PyStringType, PyString_FromString("string").StealNullable()); - - SetPyMemberTypeOf(out PyUnicodeType, PyString_FromString("unicode").StealNullable()); - - SetPyMemberTypeOf(out PyBytesType, EmptyPyBytes().StealNullable()); - - SetPyMemberTypeOf(out PyTupleType, PyTuple_New(0).StealNullable()); - - SetPyMemberTypeOf(out PyListType, PyList_New(0).StealNullable()); - - SetPyMemberTypeOf(out PyDictType, PyDict_New().StealNullable()); - - SetPyMemberTypeOf(out PyLongType, PyInt_FromInt32(0).StealNullable()); - - SetPyMemberTypeOf(out PyFloatType, PyFloat_FromDouble(0).StealNullable()); - - _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); - { - using var sys = PyImport_ImportModule("sys"); - SetPyMemberTypeOf(out PyModuleType, sys.StealNullable()); - } - } - - private static NativeFunc* Get_PyObject_NextNotImplemented() - { - using var pyType = SlotHelper.CreateObjectType(); - return Util.ReadPtr(pyType.Borrow(), TypeOffset.tp_iternext); - } - - internal static void Shutdown() - { - if (Py_IsInitialized() == 0 || !_isInitialized) - { - return; - } - _isInitialized = false; - - var state = PyGILState_Ensure(); - - if (!HostedInPython && !ProcessIsTerminating) - { - // avoid saving dead objects - TryCollectingGarbage(runs: 3); - - RuntimeData.Stash(); - } - - AssemblyManager.Shutdown(); - OperatorMethod.Shutdown(); - ImportHook.Shutdown(); - - ClearClrModules(); - RemoveClrRootModule(); - - TryCollectingGarbage(MaxCollectRetriesOnShutdown, forceBreakLoops: true, - obj: true, derived: false, buffer: false); - CLRObject.creationBlocked = true; - - NullGCHandles(ExtensionType.loadedExtensions); - ClassManager.RemoveClasses(); - TypeManager.RemoveTypes(); - _typesInitialized = false; - - MetaType.Release(); - PyCLRMetaType.Dispose(); - PyCLRMetaType = null!; - - Exceptions.Shutdown(); - PythonEngine.InteropConfiguration.Dispose(); - DisposeLazyObject(clrInterop); - DisposeLazyObject(inspect); - DisposeLazyObject(hexCallable); - PyObjectConversions.Reset(); - - PyGC_Collect(); - bool everythingSeemsCollected = TryCollectingGarbage(MaxCollectRetriesOnShutdown); - Debug.Assert(everythingSeemsCollected); - - Finalizer.Shutdown(); - InternString.Shutdown(); - - ResetPyMembers(); - - if (!HostedInPython) - { - GC.Collect(); - GC.WaitForPendingFinalizers(); - PyGILState_Release(state); - // Then release the GIL for good, if there is somehting to release - // Use the unchecked version as the checked version calls `abort()` - // if the current state is NULL. - if (_PyThreadState_UncheckedGet() != (PyThreadState*)0) - { - PyEval_SaveThread(); - } - - ExtensionType.loadedExtensions.Clear(); - CLRObject.reflectedObjects.Clear(); - } - else - { - PyGILState_Release(state); - } - } - - const int MaxCollectRetriesOnShutdown = 20; - internal static int _collected; - static bool TryCollectingGarbage(int runs, bool forceBreakLoops, - bool obj = true, bool derived = true, bool buffer = true) - { - if (runs <= 0) throw new ArgumentOutOfRangeException(nameof(runs)); - - for (int attempt = 0; attempt < runs; attempt++) - { - Interlocked.Exchange(ref _collected, 0); - nint pyCollected = 0; - for (int i = 0; i < 2; i++) - { - GC.Collect(); - GC.WaitForPendingFinalizers(); - pyCollected += PyGC_Collect(); - pyCollected += Finalizer.Instance.DisposeAll(disposeObj: obj, - disposeDerived: derived, - disposeBuffer: buffer); - } - if (Volatile.Read(ref _collected) == 0 && pyCollected == 0) - { - if (attempt + 1 == runs) return true; - } - else if (forceBreakLoops) - { - NullGCHandles(CLRObject.reflectedObjects); - CLRObject.reflectedObjects.Clear(); - } - } - return false; - } - /// - /// Alternates .NET and Python GC runs in an attempt to collect all garbage - /// - /// Total number of GC loops to run - /// true if a steady state was reached upon the requested number of tries (e.g. on the last try no objects were collected). - [ForbidPythonThreads] - public static bool TryCollectingGarbage(int runs) - => TryCollectingGarbage(runs, forceBreakLoops: false); - - static void DisposeLazyObject(Lazy pyObject) - { - if (pyObject.IsValueCreated) - { - pyObject.Value.Dispose(); - } - } - - private static Lazy GetModuleLazy(string moduleName) - => moduleName is null - ? throw new ArgumentNullException(nameof(moduleName)) - : new Lazy(() => PyModule.Import(moduleName), isThreadSafe: false); - - private static void SetPyMember(out PyObject obj, StolenReference value) - { - // XXX: For current usages, value should not be null. - if (value == null) - { - throw PythonException.ThrowLastAsClrException(); - } - obj = new PyObject(value); - _pyRefs.Add(obj); - } - - private static void SetPyMemberTypeOf(out PyType obj, PyObject value) - { - var type = PyObject_Type(value); - obj = new PyType(type.StealOrThrow(), prevalidated: true); - _pyRefs.Add(obj); - } - - private static void SetPyMemberTypeOf(out PyObject obj, StolenReference value) - { - if (value == null) - { - throw PythonException.ThrowLastAsClrException(); - } - var @ref = new BorrowedReference(value.Pointer); - var type = PyObject_Type(@ref); - XDecref(value.AnalyzerWorkaround()); - SetPyMember(out obj, type.StealNullable()); - } - - private static void ResetPyMembers() - { - foreach (var pyObj in _pyRefs) - pyObj.Dispose(); - _pyRefs.Clear(); - } - - private static void ClearClrModules() - { - var modules = PyImport_GetModuleDict(); - using var items = PyDict_Items(modules); - nint length = PyList_Size(items.BorrowOrThrow()); - if (length < 0) throw PythonException.ThrowLastAsClrException(); - for (nint i = 0; i < length; i++) - { - var item = PyList_GetItem(items.Borrow(), i); - var name = PyTuple_GetItem(item, 0); - var module = PyTuple_GetItem(item, 1); - if (ManagedType.IsInstanceOfManagedType(module)) - { - PyDict_DelItem(modules, name); - } - } - } - - private static void RemoveClrRootModule() - { - var modules = PyImport_GetModuleDict(); - PyDictTryDelItem(modules, "clr"); - PyDictTryDelItem(modules, "clr._extra"); - } - - private static void PyDictTryDelItem(BorrowedReference dict, string key) - { - if (PyDict_DelItemString(dict, key) == 0) - { - return; - } - if (!PythonException.CurrentMatches(Exceptions.KeyError)) - { - throw PythonException.ThrowLastAsClrException(); - } - PyErr_Clear(); - } - - private static void NullGCHandles(IEnumerable objects) - { - foreach (IntPtr objWithGcHandle in objects.ToArray()) - { - var @ref = new BorrowedReference(objWithGcHandle); - ManagedType.TryFreeGCHandle(@ref); - } - } - -#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. - // these objects are initialized in Initialize rather than in constructor - internal static PyObject PyBaseObjectType; - internal static PyObject PyModuleType; - internal static PyObject PySuper_Type; - internal static PyType PyCLRMetaType; - internal static PyObject PyMethodType; - internal static PyObject PyWrapperDescriptorType; - - internal static PyObject PyUnicodeType; - internal static PyObject PyStringType; - internal static PyObject PyTupleType; - internal static PyObject PyListType; - internal static PyObject PyDictType; - internal static PyObject PyLongType; - internal static PyObject PyFloatType; - internal static PyType PyBoolType; - internal static PyType PyNoneType; - internal static BorrowedReference PyTypeType => new(Delegates.PyType_Type); - - internal static PyObject PyBytesType; - internal static NativeFunc* _PyObject_NextNotImplemented; - - internal static PyObject PyNotImplemented; - internal const int Py_LT = 0; - internal const int Py_LE = 1; - internal const int Py_EQ = 2; - internal const int Py_NE = 3; - internal const int Py_GT = 4; - internal const int Py_GE = 5; - - internal static BorrowedReference PyTrue => _PyTrue; - static PyObject _PyTrue; - internal static BorrowedReference PyFalse => _PyFalse; - static PyObject _PyFalse; - internal static BorrowedReference PyNone => _PyNone; - private static PyObject _PyNone; - - private static Lazy inspect; - internal static PyObject InspectModule => inspect.Value; - - private static Lazy clrInterop; - internal static PyObject InteropModule => clrInterop.Value; - - private static Lazy hexCallable; - internal static PyObject HexCallable => hexCallable.Value; -#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. - - internal static BorrowedReference CLRMetaType => PyCLRMetaType; - - public static PyObject None => new(_PyNone); - - /// - /// Check if any Python Exceptions occurred. - /// If any exist throw new PythonException. - /// - /// - /// Can be used instead of `obj == IntPtr.Zero` for example. - /// - internal static void CheckExceptionOccurred() - { - if (PyErr_Occurred() != null) - { - throw PythonException.ThrowLastAsClrException(); - } - } - - internal static Type[]? PythonArgsToTypeArray(BorrowedReference arg) - { - return PythonArgsToTypeArray(arg, false); - } - - internal static Type[]? PythonArgsToTypeArray(BorrowedReference arg, bool mangleObjects) - { - // Given a PyObject * that is either a single type object or a - // tuple of (managed or unmanaged) type objects, return a Type[] - // containing the CLR Type objects that map to those types. - BorrowedReference args = arg; - NewReference newArgs = default; - - if (!PyTuple_Check(arg)) - { - newArgs = PyTuple_New(1); - args = newArgs.Borrow(); - PyTuple_SetItem(args, 0, arg); - } - - var n = PyTuple_Size(args); - var types = new Type[n]; - Type? t = null; - - for (var i = 0; i < n; i++) - { - BorrowedReference op = PyTuple_GetItem(args, i); - if (mangleObjects && (!PyType_Check(op))) - { - op = PyObject_TYPE(op); - } - ManagedType? mt = ManagedType.GetManagedObject(op); - - if (mt is ClassBase b) - { - var _type = b.type; - t = _type.Valid ? _type.Value : null; - } - else if (mt is CLRObject ob) - { - var inst = ob.inst; - if (inst is Type ty) - { - t = ty; - } - } - else - { - t = Converter.GetTypeByAlias(op); - } - - if (t == null) - { - types = null; - break; - } - types[i] = t; - } - newArgs.Dispose(); - return types; - } - - /// - /// Managed exports of the Python C API. Where appropriate, we do - /// some optimization to avoid managed <--> unmanaged transitions - /// (mostly for heavily used methods). - /// - [Obsolete("Use NewReference or PyObject constructor instead")] - internal static unsafe void XIncref(BorrowedReference op) - { - Py_IncRef(op); - return; - } - - internal static unsafe void XDecref(StolenReference op) - { -#if DEBUG - Debug.Assert(op == null || Refcount(new BorrowedReference(op.Pointer)) > 0); - Debug.Assert(_isInitialized || Py_IsInitialized() != 0 || _Py_IsFinalizing() != false); -#endif - if (op == null) return; - Py_DecRef(op.AnalyzerWorkaround()); - return; - } - - [Pure] - internal static unsafe nint Refcount(BorrowedReference op) - { - if (op == null) - { - return 0; - } - var p = (nint*)(op.DangerousGetAddress() + ABI.RefCountOffset); - return *p; - } - [Pure] - internal static int Refcount32(BorrowedReference op) => checked((int)Refcount(op)); - - internal static void TryUsingDll(Action op) => - TryUsingDll(() => { op(); return 0; }); - - /// - /// Call specified function, and handle PythonDLL-related failures. - /// - internal static T TryUsingDll(Func op) - { - try - { - return op(); - } - catch (TypeInitializationException loadFailure) - { - var delegatesLoadFailure = loadFailure; - // failure to load Delegates type might have been the cause - // of failure to load some higher-level type - while (delegatesLoadFailure.InnerException is TypeInitializationException nested) - { - delegatesLoadFailure = nested; - } - - if (delegatesLoadFailure.InnerException is BadPythonDllException badDll) - { - throw badDll; - } - - throw; - } - } - - /// - /// Export of Macro Py_XIncRef. Use XIncref instead. - /// Limit this function usage for Testing and Py_Debug builds - /// - /// PyObject Ptr - - internal static void Py_IncRef(BorrowedReference ob) => Delegates.Py_IncRef(ob); - - /// - /// Export of Macro Py_XDecRef. Use XDecref instead. - /// Limit this function usage for Testing and Py_Debug builds - /// - /// PyObject Ptr - - internal static void Py_DecRef(StolenReference ob) => Delegates.Py_DecRef(ob); - - - internal static void Py_Initialize() => Delegates.Py_Initialize(); - - - internal static void Py_InitializeEx(int initsigs) => Delegates.Py_InitializeEx(initsigs); - - - internal static int Py_IsInitialized() => Delegates.Py_IsInitialized(); - - - internal static void Py_Finalize() => Delegates.Py_Finalize(); - - - internal static PyThreadState* Py_NewInterpreter() => Delegates.Py_NewInterpreter(); - - - internal static void Py_EndInterpreter(PyThreadState* threadState) => Delegates.Py_EndInterpreter(threadState); - - - internal static PyThreadState* PyThreadState_New(PyInterpreterState* istate) => Delegates.PyThreadState_New(istate); - - - internal static PyThreadState* PyThreadState_Get() => Delegates.PyThreadState_Get(); - - - internal static PyThreadState* _PyThreadState_UncheckedGet() => Delegates._PyThreadState_UncheckedGet(); - - - internal static int PyGILState_Check() => Delegates.PyGILState_Check(); - internal static PyGILState PyGILState_Ensure() => Delegates.PyGILState_Ensure(); - - - internal static void PyGILState_Release(PyGILState gs) => Delegates.PyGILState_Release(gs); - - - - internal static PyThreadState* PyGILState_GetThisThreadState() => Delegates.PyGILState_GetThisThreadState(); - - - internal static void PyEval_InitThreads() => Delegates.PyEval_InitThreads(); - - - internal static int PyEval_ThreadsInitialized() => Delegates.PyEval_ThreadsInitialized(); - - - internal static void PyEval_AcquireLock() => Delegates.PyEval_AcquireLock(); - - - internal static void PyEval_ReleaseLock() => Delegates.PyEval_ReleaseLock(); - - - internal static void PyEval_AcquireThread(PyThreadState* tstate) => Delegates.PyEval_AcquireThread(tstate); - - - internal static void PyEval_ReleaseThread(PyThreadState* tstate) => Delegates.PyEval_ReleaseThread(tstate); - - - internal static PyThreadState* PyEval_SaveThread() => Delegates.PyEval_SaveThread(); - - - internal static void PyEval_RestoreThread(PyThreadState* tstate) => Delegates.PyEval_RestoreThread(tstate); - - - internal static BorrowedReference PyEval_GetBuiltins() => Delegates.PyEval_GetBuiltins(); - - - internal static BorrowedReference PyEval_GetGlobals() => Delegates.PyEval_GetGlobals(); - - - internal static BorrowedReference PyEval_GetLocals() => Delegates.PyEval_GetLocals(); - - - internal static IntPtr Py_GetProgramName() => Delegates.Py_GetProgramName(); - - - internal static void Py_SetProgramName(IntPtr name) => Delegates.Py_SetProgramName(name); - - - internal static IntPtr Py_GetPythonHome() => Delegates.Py_GetPythonHome(); - - - internal static void Py_SetPythonHome(IntPtr home) => Delegates.Py_SetPythonHome(home); - - - internal static IntPtr Py_GetPath() => Delegates.Py_GetPath(); - - - internal static void Py_SetPath(IntPtr home) => Delegates.Py_SetPath(home); - - - internal static IntPtr Py_GetVersion() => Delegates.Py_GetVersion(); - - - internal static IntPtr Py_GetPlatform() => Delegates.Py_GetPlatform(); - - - internal static IntPtr Py_GetCopyright() => Delegates.Py_GetCopyright(); - - - internal static IntPtr Py_GetCompiler() => Delegates.Py_GetCompiler(); - - - internal static IntPtr Py_GetBuildInfo() => Delegates.Py_GetBuildInfo(); - - const PyCompilerFlags Utf8String = PyCompilerFlags.IGNORE_COOKIE | PyCompilerFlags.SOURCE_IS_UTF8; - - internal static int PyRun_SimpleString(string code) - { - using var codePtr = new StrPtr(code); - return Delegates.PyRun_SimpleStringFlags(codePtr, Utf8String); - } - - internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals) - { - using var codePtr = new StrPtr(code); - return Delegates.PyRun_StringFlags(codePtr, st, globals, locals, Utf8String); - } - - internal static NewReference PyEval_EvalCode(BorrowedReference co, BorrowedReference globals, BorrowedReference locals) => Delegates.PyEval_EvalCode(co, globals, locals); - - /// - /// Return value: New reference. - /// This is a simplified interface to Py_CompileStringFlags() below, leaving flags set to NULL. - /// - internal static NewReference Py_CompileString(string str, string file, int start) - { - using var strPtr = new StrPtr(str); - - using var fileObj = new PyString(file); - return Delegates.Py_CompileStringObject(strPtr, fileObj, start, Utf8String, -1); - } - - internal static NewReference PyImport_ExecCodeModule(string name, BorrowedReference code) - { - using var namePtr = new StrPtr(name); - return Delegates.PyImport_ExecCodeModule(namePtr, code); - } - - //==================================================================== - // Python abstract object API - //==================================================================== - - /// - /// A macro-like method to get the type of a Python object. This is - /// designed to be lean and mean in IL & avoid managed <-> unmanaged - /// transitions. Note that this does not incref the type object. - /// - internal static unsafe BorrowedReference PyObject_TYPE(BorrowedReference op) - { - IntPtr address = op.DangerousGetAddressOrNull(); - if (address == IntPtr.Zero) - { - return BorrowedReference.Null; - } - Debug.Assert(TypeOffset.ob_type > 0); - BorrowedReference* typePtr = (BorrowedReference*)(address + TypeOffset.ob_type); - return *typePtr; - } - internal static NewReference PyObject_Type(BorrowedReference o) - => Delegates.PyObject_Type(o); - - internal static string PyObject_GetTypeName(BorrowedReference op) - { - Debug.Assert(TypeOffset.tp_name > 0); - Debug.Assert(op != null); - BorrowedReference pyType = PyObject_TYPE(op); - IntPtr ppName = Util.ReadIntPtr(pyType, TypeOffset.tp_name); - return Marshal.PtrToStringAnsi(ppName); - } - - /// - /// Test whether the Python object is an iterable. - /// - internal static bool PyObject_IsIterable(BorrowedReference ob) - { - var ob_type = PyObject_TYPE(ob); - return Util.ReadIntPtr(ob_type, TypeOffset.tp_iter) != IntPtr.Zero; - } - - internal static int PyObject_HasAttrString(BorrowedReference pointer, string name) - { - using var namePtr = new StrPtr(name); - return Delegates.PyObject_HasAttrString(pointer, namePtr); - } - - internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, string name) - { - using var namePtr = new StrPtr(name); - return Delegates.PyObject_GetAttrString(pointer, namePtr); - } - - internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, StrPtr name) - => Delegates.PyObject_GetAttrString(pointer, name); - - - internal static int PyObject_DelAttr(BorrowedReference @object, BorrowedReference name) => Delegates.PyObject_SetAttr(@object, name, null); - internal static int PyObject_DelAttrString(BorrowedReference @object, string name) - { - using var namePtr = new StrPtr(name); - return Delegates.PyObject_SetAttrString(@object, namePtr, null); - } - internal static int PyObject_SetAttrString(BorrowedReference @object, string name, BorrowedReference value) - { - using var namePtr = new StrPtr(name); - return Delegates.PyObject_SetAttrString(@object, namePtr, value); - } - - internal static int PyObject_HasAttr(BorrowedReference pointer, BorrowedReference name) => Delegates.PyObject_HasAttr(pointer, name); - - - internal static NewReference PyObject_GetAttr(BorrowedReference pointer, IntPtr name) - => Delegates.PyObject_GetAttr(pointer, new BorrowedReference(name)); - internal static NewReference PyObject_GetAttr(BorrowedReference o, BorrowedReference name) => Delegates.PyObject_GetAttr(o, name); - - - internal static int PyObject_SetAttr(BorrowedReference o, BorrowedReference name, BorrowedReference value) => Delegates.PyObject_SetAttr(o, name, value); - - - internal static NewReference PyObject_GetItem(BorrowedReference o, BorrowedReference key) => Delegates.PyObject_GetItem(o, key); - - - internal static int PyObject_SetItem(BorrowedReference o, BorrowedReference key, BorrowedReference value) => Delegates.PyObject_SetItem(o, key, value); - - - internal static int PyObject_DelItem(BorrowedReference o, BorrowedReference key) => Delegates.PyObject_DelItem(o, key); - - - internal static NewReference PyObject_GetIter(BorrowedReference op) => Delegates.PyObject_GetIter(op); - - - internal static NewReference PyObject_Call(BorrowedReference pointer, BorrowedReference args, BorrowedReference kw) => Delegates.PyObject_Call(pointer, args, kw); - - internal static NewReference PyObject_CallObject(BorrowedReference callable, BorrowedReference args) => Delegates.PyObject_CallObject(callable, args); - internal static IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args) - => Delegates.PyObject_CallObject(new BorrowedReference(pointer), new BorrowedReference(args)) - .DangerousMoveToPointerOrNull(); - - - internal static int PyObject_RichCompareBool(BorrowedReference value1, BorrowedReference value2, int opid) => Delegates.PyObject_RichCompareBool(value1, value2, opid); - - - internal static int PyObject_IsInstance(BorrowedReference ob, BorrowedReference type) => Delegates.PyObject_IsInstance(ob, type); - - - internal static int PyObject_IsSubclass(BorrowedReference ob, BorrowedReference type) => Delegates.PyObject_IsSubclass(ob, type); - - internal static void PyObject_ClearWeakRefs(BorrowedReference ob) => Delegates.PyObject_ClearWeakRefs(ob); - - internal static BorrowedReference PyObject_GetWeakRefList(BorrowedReference ob) - { - Debug.Assert(ob != null); - var type = PyObject_TYPE(ob); - int offset = Util.ReadInt32(type, TypeOffset.tp_weaklistoffset); - if (offset <= 0) return BorrowedReference.Null; - Debug.Assert(offset > 0); - return Util.ReadRef(ob, offset); - } - - - internal static int PyCallable_Check(BorrowedReference o) => Delegates.PyCallable_Check(o); - - - internal static int PyObject_IsTrue(IntPtr pointer) => PyObject_IsTrue(new BorrowedReference(pointer)); - internal static int PyObject_IsTrue(BorrowedReference pointer) => Delegates.PyObject_IsTrue(pointer); - - - internal static int PyObject_Not(BorrowedReference o) => Delegates.PyObject_Not(o); - - internal static nint PyObject_Size(BorrowedReference pointer) => Delegates.PyObject_Size(pointer); - - - internal static nint PyObject_Hash(BorrowedReference op) => Delegates.PyObject_Hash(op); - - - internal static NewReference PyObject_Repr(BorrowedReference pointer) - { - AssertNoErorSet(); - - return Delegates.PyObject_Repr(pointer); - } - - - internal static NewReference PyObject_Str(BorrowedReference pointer) - { - AssertNoErorSet(); - - return Delegates.PyObject_Str(pointer); - } - - [Conditional("DEBUG")] - internal static void AssertNoErorSet() - { - if (Exceptions.ErrorOccurred()) - throw new InvalidOperationException( - "Can't call with exception set", - PythonException.FetchCurrent()); - } - - - internal static NewReference PyObject_Dir(BorrowedReference pointer) => Delegates.PyObject_Dir(pointer); - - internal static void _Py_NewReference(BorrowedReference ob) - { - if (Delegates._Py_NewReference != null) - Delegates._Py_NewReference(ob); - } - - internal static bool? _Py_IsFinalizing() - { - if (Delegates._Py_IsFinalizing != null) - return Delegates._Py_IsFinalizing() != 0; - else - return null; ; - } - - //==================================================================== - // Python buffer API - //==================================================================== - - - internal static int PyObject_GetBuffer(BorrowedReference exporter, out Py_buffer view, int flags) => Delegates.PyObject_GetBuffer(exporter, out view, flags); - - - internal static void PyBuffer_Release(ref Py_buffer view) => Delegates.PyBuffer_Release(ref view); - - - internal static nint PyBuffer_SizeFromFormat(string format) - { - using var formatPtr = new StrPtr(format, Encoding.ASCII); - return Delegates.PyBuffer_SizeFromFormat(formatPtr); - } - - internal static int PyBuffer_IsContiguous(ref Py_buffer view, char order) => Delegates.PyBuffer_IsContiguous(ref view, order); - - - internal static IntPtr PyBuffer_GetPointer(ref Py_buffer view, nint[] indices) => Delegates.PyBuffer_GetPointer(ref view, indices); - - - internal static int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort) => Delegates.PyBuffer_FromContiguous(ref view, buf, len, fort); - - - internal static int PyBuffer_ToContiguous(IntPtr buf, ref Py_buffer src, IntPtr len, char order) => Delegates.PyBuffer_ToContiguous(buf, ref src, len, order); - - - internal static void PyBuffer_FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, char order) => Delegates.PyBuffer_FillContiguousStrides(ndims, shape, strides, itemsize, order); - - - internal static int PyBuffer_FillInfo(ref Py_buffer view, BorrowedReference exporter, IntPtr buf, IntPtr len, int _readonly, int flags) => Delegates.PyBuffer_FillInfo(ref view, exporter, buf, len, _readonly, flags); - - //==================================================================== - // Python number API - //==================================================================== - - - internal static NewReference PyNumber_Long(BorrowedReference ob) => Delegates.PyNumber_Long(ob); - - - internal static NewReference PyNumber_Float(BorrowedReference ob) => Delegates.PyNumber_Float(ob); - - - internal static bool PyNumber_Check(BorrowedReference ob) => Delegates.PyNumber_Check(ob); - - internal static bool PyInt_Check(BorrowedReference ob) - => PyObject_TypeCheck(ob, PyLongType); - - internal static bool PyInt_CheckExact(BorrowedReference ob) - => PyObject_TypeCheckExact(ob, PyLongType); - - internal static bool PyBool_Check(BorrowedReference ob) - => PyObject_TypeCheck(ob, PyBoolType); - internal static bool PyBool_CheckExact(BorrowedReference ob) - => PyObject_TypeCheckExact(ob, PyBoolType); - - internal static NewReference PyInt_FromInt32(int value) => PyLong_FromLongLong(value); - - internal static NewReference PyInt_FromInt64(long value) => PyLong_FromLongLong(value); - - internal static NewReference PyLong_FromLongLong(long value) => Delegates.PyLong_FromLongLong(value); - - - internal static NewReference PyLong_FromUnsignedLongLong(ulong value) => Delegates.PyLong_FromUnsignedLongLong(value); - - - internal static NewReference PyLong_FromString(string value, int radix) - { - using var valPtr = new StrPtr(value); - return Delegates.PyLong_FromString(valPtr, IntPtr.Zero, radix); - } - - - - internal static nuint PyLong_AsUnsignedSize_t(BorrowedReference value) => Delegates.PyLong_AsUnsignedSize_t(value); - - internal static nint PyLong_AsSignedSize_t(BorrowedReference value) => Delegates.PyLong_AsSignedSize_t(value); - - internal static long? PyLong_AsLongLong(BorrowedReference value) - { - long result = Delegates.PyLong_AsLongLong(value); - if (result == -1 && Exceptions.ErrorOccurred()) - { - return null; - } - return result; - } - - internal static ulong? PyLong_AsUnsignedLongLong(BorrowedReference value) - { - ulong result = Delegates.PyLong_AsUnsignedLongLong(value); - if (result == unchecked((ulong)-1) && Exceptions.ErrorOccurred()) - { - return null; - } - return result; - } - - internal static bool PyFloat_Check(BorrowedReference ob) - => PyObject_TypeCheck(ob, PyFloatType); - internal static bool PyFloat_CheckExact(BorrowedReference ob) - => PyObject_TypeCheckExact(ob, PyFloatType); - - /// - /// Return value: New reference. - /// Create a Python integer from the pointer p. The pointer value can be retrieved from the resulting value using PyLong_AsVoidPtr(). - /// - internal static NewReference PyLong_FromVoidPtr(IntPtr p) => Delegates.PyLong_FromVoidPtr(p); - - /// - /// Convert a Python integer pylong to a C void pointer. If pylong cannot be converted, an OverflowError will be raised. This is only assured to produce a usable void pointer for values created with PyLong_FromVoidPtr(). - /// - - internal static IntPtr PyLong_AsVoidPtr(BorrowedReference ob) => Delegates.PyLong_AsVoidPtr(ob); - - - internal static NewReference PyFloat_FromDouble(double value) => Delegates.PyFloat_FromDouble(value); - - - internal static NewReference PyFloat_FromString(BorrowedReference value) => Delegates.PyFloat_FromString(value); - - - internal static double PyFloat_AsDouble(BorrowedReference ob) => Delegates.PyFloat_AsDouble(ob); - - - internal static NewReference PyNumber_Add(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Add(o1, o2); - - - internal static NewReference PyNumber_Subtract(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Subtract(o1, o2); - - - internal static NewReference PyNumber_Multiply(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Multiply(o1, o2); - - - internal static NewReference PyNumber_TrueDivide(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_TrueDivide(o1, o2); - - - internal static NewReference PyNumber_And(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_And(o1, o2); - - - internal static NewReference PyNumber_Xor(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Xor(o1, o2); - - - internal static NewReference PyNumber_Or(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Or(o1, o2); - - - internal static NewReference PyNumber_Lshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Lshift(o1, o2); - - - internal static NewReference PyNumber_Rshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Rshift(o1, o2); - - - internal static NewReference PyNumber_Power(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Power(o1, o2); - - - internal static NewReference PyNumber_Remainder(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Remainder(o1, o2); - - - internal static NewReference PyNumber_InPlaceAdd(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceAdd(o1, o2); - - - internal static NewReference PyNumber_InPlaceSubtract(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceSubtract(o1, o2); - - - internal static NewReference PyNumber_InPlaceMultiply(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceMultiply(o1, o2); - - - internal static NewReference PyNumber_InPlaceTrueDivide(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceTrueDivide(o1, o2); - - - internal static NewReference PyNumber_InPlaceAnd(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceAnd(o1, o2); - - - internal static NewReference PyNumber_InPlaceXor(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceXor(o1, o2); - - - internal static NewReference PyNumber_InPlaceOr(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceOr(o1, o2); - - - internal static NewReference PyNumber_InPlaceLshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceLshift(o1, o2); - - - internal static NewReference PyNumber_InPlaceRshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceRshift(o1, o2); - - - internal static NewReference PyNumber_InPlacePower(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlacePower(o1, o2); - - - internal static NewReference PyNumber_InPlaceRemainder(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceRemainder(o1, o2); - - - internal static NewReference PyNumber_Negative(BorrowedReference o1) => Delegates.PyNumber_Negative(o1); - - - internal static NewReference PyNumber_Positive(BorrowedReference o1) => Delegates.PyNumber_Positive(o1); - - - internal static NewReference PyNumber_Invert(BorrowedReference o1) => Delegates.PyNumber_Invert(o1); - - - //==================================================================== - // Python sequence API - //==================================================================== - - - internal static bool PySequence_Check(BorrowedReference pointer) => Delegates.PySequence_Check(pointer); - - internal static NewReference PySequence_GetItem(BorrowedReference pointer, nint index) => Delegates.PySequence_GetItem(pointer, index); - internal static int PySequence_SetItem(BorrowedReference pointer, nint index, BorrowedReference value) => Delegates.PySequence_SetItem(pointer, index, value); - - internal static int PySequence_DelItem(BorrowedReference pointer, nint index) => Delegates.PySequence_DelItem(pointer, index); - - internal static NewReference PySequence_GetSlice(BorrowedReference pointer, nint i1, nint i2) => Delegates.PySequence_GetSlice(pointer, i1, i2); - - internal static int PySequence_SetSlice(BorrowedReference pointer, nint i1, nint i2, BorrowedReference v) => Delegates.PySequence_SetSlice(pointer, i1, i2, v); - - internal static int PySequence_DelSlice(BorrowedReference pointer, nint i1, nint i2) => Delegates.PySequence_DelSlice(pointer, i1, i2); - - internal static nint PySequence_Size(BorrowedReference pointer) => Delegates.PySequence_Size(pointer); - - internal static int PySequence_Contains(BorrowedReference pointer, BorrowedReference item) => Delegates.PySequence_Contains(pointer, item); - - - internal static NewReference PySequence_Concat(BorrowedReference pointer, BorrowedReference other) => Delegates.PySequence_Concat(pointer, other); - - internal static NewReference PySequence_Repeat(BorrowedReference pointer, nint count) => Delegates.PySequence_Repeat(pointer, count); - - - internal static nint PySequence_Index(BorrowedReference pointer, BorrowedReference item) => Delegates.PySequence_Index(pointer, item); - - private static nint PySequence_Count(BorrowedReference pointer, BorrowedReference value) => Delegates.PySequence_Count(pointer, value); - - - internal static NewReference PySequence_Tuple(BorrowedReference pointer) => Delegates.PySequence_Tuple(pointer); - - - internal static NewReference PySequence_List(BorrowedReference pointer) => Delegates.PySequence_List(pointer); - - - //==================================================================== - // Python string API - //==================================================================== - internal static bool PyString_Check(BorrowedReference ob) - => PyObject_TypeCheck(ob, PyStringType); - internal static bool PyString_CheckExact(BorrowedReference ob) - => PyObject_TypeCheckExact(ob, PyStringType); - - internal static NewReference PyString_FromString(string value) - { - int byteorder = BitConverter.IsLittleEndian ? -1 : 1; - int* byteorderPtr = &byteorder; - fixed(char* ptr = value) - return Delegates.PyUnicode_DecodeUTF16( - (IntPtr)ptr, - value.Length * sizeof(Char), - IntPtr.Zero, - (IntPtr)byteorderPtr - ); - } - - - internal static NewReference EmptyPyBytes() - { - byte* bytes = stackalloc byte[1]; - bytes[0] = 0; - return Delegates.PyBytes_FromString((IntPtr)bytes); - } - - internal static NewReference PyByteArray_FromStringAndSize(IntPtr strPtr, nint len) => Delegates.PyByteArray_FromStringAndSize(strPtr, len); - internal static NewReference PyByteArray_FromStringAndSize(string s) - { - using var ptr = new StrPtr(s); - return PyByteArray_FromStringAndSize(ptr.RawPointer, checked((nint)ptr.ByteCount)); - } - - internal static IntPtr PyBytes_AsString(BorrowedReference ob) - { - Debug.Assert(ob != null); - return Delegates.PyBytes_AsString(ob); - } - - internal static nint PyBytes_Size(BorrowedReference op) => Delegates.PyBytes_Size(op); - - internal static IntPtr PyUnicode_AsUTF8(BorrowedReference unicode) => Delegates.PyUnicode_AsUTF8(unicode); - - /// Length in code points - internal static nint PyUnicode_GetLength(BorrowedReference ob) => Delegates.PyUnicode_GetLength(ob); - - - internal static NewReference PyUnicode_AsUTF16String(BorrowedReference ob) => Delegates.PyUnicode_AsUTF16String(ob); - - internal static int PyUnicode_ReadChar(BorrowedReference ob, nint index) => Delegates.PyUnicode_ReadChar(ob, index); - - - - internal static NewReference PyUnicode_FromOrdinal(int c) => Delegates.PyUnicode_FromOrdinal(c); - - internal static NewReference PyUnicode_InternFromString(string s) - { - using var ptr = new StrPtr(s); - return Delegates.PyUnicode_InternFromString(ptr); - } - - internal static int PyUnicode_Compare(BorrowedReference left, BorrowedReference right) => Delegates.PyUnicode_Compare(left, right); - - internal static string ToString(BorrowedReference op) - { - using var strval = PyObject_Str(op); - return GetManagedStringFromUnicodeObject(strval.BorrowOrThrow())!; - } - - /// - /// Function to access the internal PyUnicode/PyString object and - /// convert it to a managed string with the correct encoding. - /// - /// - /// We can't easily do this through through the CustomMarshaler's on - /// the returns because will have access to the IntPtr but not size. - /// - /// For PyUnicodeType, we can't convert with Marshal.PtrToStringUni - /// since it only works for UCS2. - /// - /// PyStringType or PyUnicodeType object to convert - /// Managed String - internal static string? GetManagedString(in BorrowedReference op) - { - var type = PyObject_TYPE(op); - - if (type == PyUnicodeType) - { - return GetManagedStringFromUnicodeObject(op); - } - - return null; - } - - static string GetManagedStringFromUnicodeObject(BorrowedReference op) - { -#if DEBUG - var type = PyObject_TYPE(op); - Debug.Assert(type == PyUnicodeType); -#endif - using var bytes = PyUnicode_AsUTF16String(op); - if (bytes.IsNull()) - { - throw PythonException.ThrowLastAsClrException(); - } - int bytesLength = checked((int)PyBytes_Size(bytes.Borrow())); - char* codePoints = (char*)PyBytes_AsString(bytes.Borrow()); - return new string(codePoints, - startIndex: 1, // skip BOM - length: bytesLength / 2 - 1); // utf16 - BOM - } - - - //==================================================================== - // Python dictionary API - //==================================================================== - - internal static bool PyDict_Check(BorrowedReference ob) - { - return PyObject_TYPE(ob) == PyDictType; - } - - - internal static NewReference PyDict_New() => Delegates.PyDict_New(); - - /// - /// Return NULL if the key is not present, but without setting an exception. - /// - internal static BorrowedReference PyDict_GetItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItem(pointer, key); - - internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key) - { - using var keyStr = new StrPtr(key); - return Delegates.PyDict_GetItemString(pointer, keyStr); - } - - internal static BorrowedReference PyDict_GetItemWithError(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItemWithError(pointer, key); - - /// - /// Return 0 on success or -1 on failure. - /// - internal static int PyDict_SetItem(BorrowedReference dict, BorrowedReference key, BorrowedReference value) => Delegates.PyDict_SetItem(dict, key, value); - - /// - /// Return 0 on success or -1 on failure. - /// - internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value) - { - using var keyPtr = new StrPtr(key); - return Delegates.PyDict_SetItemString(dict, keyPtr, value); - } - - internal static int PyDict_DelItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_DelItem(pointer, key); - - - internal static int PyDict_DelItemString(BorrowedReference pointer, string key) - { - using var keyPtr = new StrPtr(key); - return Delegates.PyDict_DelItemString(pointer, keyPtr); - } - - internal static int PyMapping_HasKey(BorrowedReference pointer, BorrowedReference key) => Delegates.PyMapping_HasKey(pointer, key); - - - internal static NewReference PyDict_Keys(BorrowedReference pointer) => Delegates.PyDict_Keys(pointer); - - internal static NewReference PyDict_Values(BorrowedReference pointer) => Delegates.PyDict_Values(pointer); - - internal static NewReference PyDict_Items(BorrowedReference pointer) => Delegates.PyDict_Items(pointer); - - - internal static NewReference PyDict_Copy(BorrowedReference pointer) => Delegates.PyDict_Copy(pointer); - - - internal static int PyDict_Update(BorrowedReference pointer, BorrowedReference other) => Delegates.PyDict_Update(pointer, other); - - - internal static void PyDict_Clear(BorrowedReference pointer) => Delegates.PyDict_Clear(pointer); - - internal static nint PyDict_Size(BorrowedReference pointer) => Delegates.PyDict_Size(pointer); - - - internal static NewReference PySet_New(BorrowedReference iterable) => Delegates.PySet_New(iterable); - - - internal static int PySet_Add(BorrowedReference set, BorrowedReference key) => Delegates.PySet_Add(set, key); - - /// - /// Return 1 if found, 0 if not found, and -1 if an error is encountered. - /// - - internal static int PySet_Contains(BorrowedReference anyset, BorrowedReference key) => Delegates.PySet_Contains(anyset, key); - - //==================================================================== - // Python list API - //==================================================================== - - internal static bool PyList_Check(BorrowedReference ob) - { - return PyObject_TYPE(ob) == PyListType; - } - - internal static NewReference PyList_New(nint size) => Delegates.PyList_New(size); - - internal static BorrowedReference PyList_GetItem(BorrowedReference pointer, nint index) => Delegates.PyList_GetItem(pointer, index); - - internal static int PyList_SetItem(BorrowedReference pointer, nint index, StolenReference value) => Delegates.PyList_SetItem(pointer, index, value); - - internal static int PyList_Insert(BorrowedReference pointer, nint index, BorrowedReference value) => Delegates.PyList_Insert(pointer, index, value); - - - internal static int PyList_Append(BorrowedReference pointer, BorrowedReference value) => Delegates.PyList_Append(pointer, value); - - - internal static int PyList_Reverse(BorrowedReference pointer) => Delegates.PyList_Reverse(pointer); - - - internal static int PyList_Sort(BorrowedReference pointer) => Delegates.PyList_Sort(pointer); - - private static NewReference PyList_GetSlice(BorrowedReference pointer, nint start, nint end) => Delegates.PyList_GetSlice(pointer, start, end); - - private static int PyList_SetSlice(BorrowedReference pointer, nint start, nint end, BorrowedReference value) => Delegates.PyList_SetSlice(pointer, start, end, value); - - - internal static nint PyList_Size(BorrowedReference pointer) => Delegates.PyList_Size(pointer); - - //==================================================================== - // Python tuple API - //==================================================================== - - internal static bool PyTuple_Check(BorrowedReference ob) - { - return PyObject_TYPE(ob) == PyTupleType; - } - internal static NewReference PyTuple_New(nint size) => Delegates.PyTuple_New(size); - - internal static BorrowedReference PyTuple_GetItem(BorrowedReference pointer, nint index) => Delegates.PyTuple_GetItem(pointer, index); - - internal static int PyTuple_SetItem(BorrowedReference pointer, nint index, BorrowedReference value) - { - var newRef = new NewReference(value); - return PyTuple_SetItem(pointer, index, newRef.Steal()); - } - - internal static int PyTuple_SetItem(BorrowedReference pointer, nint index, StolenReference value) => Delegates.PyTuple_SetItem(pointer, index, value); - - internal static NewReference PyTuple_GetSlice(BorrowedReference pointer, nint start, nint end) => Delegates.PyTuple_GetSlice(pointer, start, end); - - internal static nint PyTuple_Size(BorrowedReference pointer) => Delegates.PyTuple_Size(pointer); - - - //==================================================================== - // Python iterator API - //==================================================================== - internal static bool PyIter_Check(BorrowedReference ob) - { - if (Delegates.PyIter_Check != null) - return Delegates.PyIter_Check(ob) != 0; - var ob_type = PyObject_TYPE(ob); - var tp_iternext = (NativeFunc*)Util.ReadIntPtr(ob_type, TypeOffset.tp_iternext); - return tp_iternext != (NativeFunc*)0 && tp_iternext != _PyObject_NextNotImplemented; - } - internal static NewReference PyIter_Next(BorrowedReference pointer) => Delegates.PyIter_Next(pointer); - - - //==================================================================== - // Python module API - //==================================================================== - - - internal static NewReference PyModule_New(string name) - { - using var namePtr = new StrPtr(name); - return Delegates.PyModule_New(namePtr); - } - - internal static BorrowedReference PyModule_GetDict(BorrowedReference module) => Delegates.PyModule_GetDict(module); - - internal static NewReference PyImport_Import(BorrowedReference name) => Delegates.PyImport_Import(name); - - /// The module to add the object to. - /// The key that will refer to the object. - /// The object to add to the module. - /// Return -1 on error, 0 on success. - internal static int PyModule_AddObject(BorrowedReference module, string name, StolenReference value) - { - using var namePtr = new StrPtr(name); - IntPtr valueAddr = value.DangerousGetAddressOrNull(); - int res = Delegates.PyModule_AddObject(module, namePtr, valueAddr); - // We can't just exit here because the reference is stolen only on success. - if (res != 0) - { - XDecref(StolenReference.TakeNullable(ref valueAddr)); - } - return res; - - } - - /// - /// Return value: New reference. - /// - - internal static NewReference PyImport_ImportModule(string name) - { - using var namePtr = new StrPtr(name); - return Delegates.PyImport_ImportModule(namePtr); - } - - internal static NewReference PyImport_ReloadModule(BorrowedReference module) => Delegates.PyImport_ReloadModule(module); - - - internal static BorrowedReference PyImport_AddModule(string name) - { - using var namePtr = new StrPtr(name); - return Delegates.PyImport_AddModule(namePtr); - } - - internal static BorrowedReference PyImport_GetModuleDict() => Delegates.PyImport_GetModuleDict(); - - - internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath) - { - var marshaler = StrArrayMarshaler.GetInstance(null); - var argvPtr = marshaler.MarshalManagedToNative(argv); - try - { - Delegates.PySys_SetArgvEx(argc, argvPtr, updatepath); - } - finally - { - marshaler.CleanUpNativeData(argvPtr); - } - } - - /// - /// Return value: Borrowed reference. - /// Return the object name from the sys module or NULL if it does not exist, without setting an exception. - /// - - internal static BorrowedReference PySys_GetObject(string name) - { - using var namePtr = new StrPtr(name); - return Delegates.PySys_GetObject(namePtr); - } - - internal static int PySys_SetObject(string name, BorrowedReference ob) - { - using var namePtr = new StrPtr(name); - return Delegates.PySys_SetObject(namePtr, ob); - } - - - //==================================================================== - // Python type object API - //==================================================================== - internal static bool PyType_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyTypeType); - - - internal static void PyType_Modified(BorrowedReference type) => Delegates.PyType_Modified(type); - internal static bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2) - { - Debug.Assert(t1 != null && t2 != null); - return Delegates.PyType_IsSubtype(t1, t2); - } - - internal static bool PyObject_TypeCheckExact(BorrowedReference ob, BorrowedReference tp) - => PyObject_TYPE(ob) == tp; - internal static bool PyObject_TypeCheck(BorrowedReference ob, BorrowedReference tp) - { - BorrowedReference t = PyObject_TYPE(ob); - return (t == tp) || PyType_IsSubtype(t, tp); - } - - internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedReference ofType) - { - return (type == ofType) || PyType_IsSubtype(type, ofType); - } - - - internal static NewReference PyType_GenericNew(BorrowedReference type, BorrowedReference args, BorrowedReference kw) => Delegates.PyType_GenericNew(type, args, kw); - - internal static NewReference PyType_GenericAlloc(BorrowedReference type, nint n) => Delegates.PyType_GenericAlloc(type, n); - - internal static IntPtr PyType_GetSlot(BorrowedReference type, TypeSlotID slot) => Delegates.PyType_GetSlot(type, slot); - internal static NewReference PyType_FromSpecWithBases(in NativeTypeSpec spec, BorrowedReference bases) => Delegates.PyType_FromSpecWithBases(in spec, bases); - - /// - /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type�s base class. Return 0 on success, or return -1 and sets an exception on error. - /// - - internal static int PyType_Ready(BorrowedReference type) => Delegates.PyType_Ready(type); - - - internal static BorrowedReference _PyType_Lookup(BorrowedReference type, BorrowedReference name) => Delegates._PyType_Lookup(type, name); - - - internal static NewReference PyObject_GenericGetAttr(BorrowedReference obj, BorrowedReference name) => Delegates.PyObject_GenericGetAttr(obj, name); - - - internal static int PyObject_GenericSetAttr(BorrowedReference obj, BorrowedReference name, BorrowedReference value) => Delegates.PyObject_GenericSetAttr(obj, name, value); - - internal static NewReference PyObject_GenericGetDict(BorrowedReference o) => PyObject_GenericGetDict(o, IntPtr.Zero); - internal static NewReference PyObject_GenericGetDict(BorrowedReference o, IntPtr context) => Delegates.PyObject_GenericGetDict(o, context); - - internal static void PyObject_GC_Del(StolenReference ob) => Delegates.PyObject_GC_Del(ob); - - - internal static bool PyObject_GC_IsTracked(BorrowedReference ob) - { - if (PyVersion >= new Version(3, 9)) - return Delegates.PyObject_GC_IsTracked(ob) != 0; - - throw new NotSupportedException("Requires Python 3.9"); - } - - internal static void PyObject_GC_Track(BorrowedReference ob) => Delegates.PyObject_GC_Track(ob); - - internal static void PyObject_GC_UnTrack(BorrowedReference ob) => Delegates.PyObject_GC_UnTrack(ob); - - internal static void _PyObject_Dump(BorrowedReference ob) => Delegates._PyObject_Dump(ob); - - //==================================================================== - // Python memory API - //==================================================================== - - internal static IntPtr PyMem_Malloc(long size) - { - return PyMem_Malloc(new IntPtr(size)); - } - - - private static IntPtr PyMem_Malloc(nint size) => Delegates.PyMem_Malloc(size); - - private static IntPtr PyMem_Realloc(IntPtr ptr, nint size) => Delegates.PyMem_Realloc(ptr, size); - - - internal static void PyMem_Free(IntPtr ptr) => Delegates.PyMem_Free(ptr); - - - //==================================================================== - // Python exception API - //==================================================================== - - - internal static void PyErr_SetString(BorrowedReference ob, string message) - { - using var msgPtr = new StrPtr(message); - Delegates.PyErr_SetString(ob, msgPtr); - } - - internal static void PyErr_SetObject(BorrowedReference type, BorrowedReference exceptionObject) => Delegates.PyErr_SetObject(type, exceptionObject); - - internal static int PyErr_ExceptionMatches(BorrowedReference exception) => Delegates.PyErr_ExceptionMatches(exception); - - - internal static int PyErr_GivenExceptionMatches(BorrowedReference given, BorrowedReference typeOrTypes) => Delegates.PyErr_GivenExceptionMatches(given, typeOrTypes); - - - internal static void PyErr_NormalizeException(ref NewReference type, ref NewReference val, ref NewReference tb) => Delegates.PyErr_NormalizeException(ref type, ref val, ref tb); - - - internal static BorrowedReference PyErr_Occurred() => Delegates.PyErr_Occurred(); - - - internal static void PyErr_Fetch(out NewReference type, out NewReference val, out NewReference tb) => Delegates.PyErr_Fetch(out type, out val, out tb); - - - internal static void PyErr_Restore(StolenReference type, StolenReference val, StolenReference tb) => Delegates.PyErr_Restore(type, val, tb); - - - internal static void PyErr_Clear() => Delegates.PyErr_Clear(); - - - internal static void PyErr_Print() => Delegates.PyErr_Print(); - - - internal static NewReference PyException_GetCause(BorrowedReference ex) - => Delegates.PyException_GetCause(ex); - internal static NewReference PyException_GetTraceback(BorrowedReference ex) - => Delegates.PyException_GetTraceback(ex); - - /// - /// Set the cause associated with the exception to cause. Use NULL to clear it. There is no type check to make sure that cause is either an exception instance or None. This steals a reference to cause. - /// - internal static void PyException_SetCause(BorrowedReference ex, StolenReference cause) - => Delegates.PyException_SetCause(ex, cause); - internal static int PyException_SetTraceback(BorrowedReference ex, BorrowedReference tb) - => Delegates.PyException_SetTraceback(ex, tb); - - //==================================================================== - // Cell API - //==================================================================== - - - internal static NewReference PyCell_Get(BorrowedReference cell) => Delegates.PyCell_Get(cell); - - - internal static int PyCell_Set(BorrowedReference cell, BorrowedReference value) => Delegates.PyCell_Set(cell, value); - - internal static nint PyGC_Collect() => Delegates.PyGC_Collect(); - internal static void Py_CLEAR(BorrowedReference ob, int offset) => ReplaceReference(ob, offset, default); - internal static void Py_CLEAR(ref T? ob) - where T: PyObject - { - ob?.Dispose(); - ob = null; - } - - internal static void ReplaceReference(BorrowedReference ob, int offset, StolenReference newValue) - { - IntPtr raw = Util.ReadIntPtr(ob, offset); - Util.WriteNullableRef(ob, offset, newValue); - XDecref(StolenReference.TakeNullable(ref raw)); - } - - //==================================================================== - // Python Capsules API - //==================================================================== - - - internal static NewReference PyCapsule_New(IntPtr pointer, IntPtr name, IntPtr destructor) - => Delegates.PyCapsule_New(pointer, name, destructor); - - internal static IntPtr PyCapsule_GetPointer(BorrowedReference capsule, IntPtr name) - { - return Delegates.PyCapsule_GetPointer(capsule, name); - } - - internal static int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer) => Delegates.PyCapsule_SetPointer(capsule, pointer); - - //==================================================================== - // Miscellaneous - //==================================================================== - - - internal static int PyThreadState_SetAsyncExcLLP64(uint id, BorrowedReference exc) => Delegates.PyThreadState_SetAsyncExcLLP64(id, exc); - - internal static int PyThreadState_SetAsyncExcLP64(ulong id, BorrowedReference exc) => Delegates.PyThreadState_SetAsyncExcLP64(id, exc); - - - internal static void SetNoSiteFlag() - { - TryUsingDll(() => - { - *Delegates.Py_NoSiteFlag = 1; - return *Delegates.Py_NoSiteFlag; - }); - } - } - - internal class BadPythonDllException : MissingMethodException - { - public BadPythonDllException(string message, Exception innerException) - : base(message, innerException) { } - } -} +using System; +using System.Diagnostics; +using System.Diagnostics.Contracts; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; +using System.Collections.Generic; +using Python.Runtime.Native; +using System.Linq; +using static System.FormattableString; + +namespace Python.Runtime +{ + /// + /// Encapsulates the low-level Python C API. Note that it is + /// the responsibility of the caller to have acquired the GIL + /// before calling any of these methods. + /// + public unsafe partial class Runtime + { + public static string? PythonDLL + { + get => _PythonDll; + set + { + if (_isInitialized) + throw new InvalidOperationException("This property must be set before runtime is initialized"); + _PythonDll = value; + } + } + + static string? _PythonDll = GetDefaultDllName(); + private static string? GetDefaultDllName() + { + string dll = Environment.GetEnvironmentVariable("PYTHONNET_PYDLL"); + if (dll is not null) return dll; + + string verString = Environment.GetEnvironmentVariable("PYTHONNET_PYVER"); + if (!Version.TryParse(verString, out var version)) return null; + + return GetDefaultDllName(version); + } + + private static string GetDefaultDllName(Version version) + { + string prefix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "" : "lib"; + string suffix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Invariant($"{version.Major}{version.Minor}") + : Invariant($"{version.Major}.{version.Minor}"); + string ext = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".dll" + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ".dylib" + : ".so"; + return prefix + "python" + suffix + ext; + } + + private static bool _isInitialized = false; + internal static bool IsInitialized => _isInitialized; + private static bool _typesInitialized = false; + internal static bool TypeManagerInitialized => _typesInitialized; + internal static readonly bool Is32Bit = IntPtr.Size == 4; + + // Available in newer .NET Core versions (>= 5) as IntPtr.MaxValue etc. + internal static readonly long IntPtrMaxValue = Is32Bit ? Int32.MaxValue : Int64.MaxValue; + internal static readonly long IntPtrMinValue = Is32Bit ? Int32.MinValue : Int64.MinValue; + internal static readonly ulong UIntPtrMaxValue = Is32Bit ? UInt32.MaxValue : UInt64.MaxValue; + + // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; + + internal static Version InteropVersion { get; } + = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version; + + public static int MainManagedThreadId { get; private set; } + + private static readonly List _pyRefs = new (); + + internal static Version PyVersion + { + get + { + var versionTuple = PySys_GetObject("version_info"); + var major = Converter.ToInt32(PyTuple_GetItem(versionTuple, 0)); + var minor = Converter.ToInt32(PyTuple_GetItem(versionTuple, 1)); + var micro = Converter.ToInt32(PyTuple_GetItem(versionTuple, 2)); + return new Version(major, minor, micro); + } + } + + const string RunSysPropName = "__pythonnet_run__"; + static int run = 0; + + internal static int GetRun() + { + int runNumber = run; + Debug.Assert(runNumber > 0, "This must only be called after Runtime is initialized at least once"); + return runNumber; + } + + internal static bool HostedInPython; + internal static bool ProcessIsTerminating; + + /// + /// Initialize the runtime... + /// + /// Always call this method from the Main thread. After the + /// first call to this method, the main thread has acquired the GIL. + internal static void Initialize(bool initSigs = false) + { + if (_isInitialized) + { + return; + } + _isInitialized = true; + + bool interpreterAlreadyInitialized = TryUsingDll( + () => Py_IsInitialized() != 0 + ); + if (!interpreterAlreadyInitialized) + { + Py_InitializeEx(initSigs ? 1 : 0); + + NewRun(); + + if (PyEval_ThreadsInitialized() == 0) + { + PyEval_InitThreads(); + } + RuntimeState.Save(); + } + else + { + if (!HostedInPython) + { + PyGILState_Ensure(); + } + + BorrowedReference pyRun = PySys_GetObject(RunSysPropName); + if (pyRun != null) + { + run = checked((int)PyLong_AsSignedSize_t(pyRun)); + } + else + { + NewRun(); + } + } + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; + + Finalizer.Initialize(); + + InitPyMembers(); + + ABI.Initialize(PyVersion); + + InternString.Initialize(); + + GenericUtil.Reset(); + ClassManager.Reset(); + ClassDerivedObject.Reset(); + TypeManager.Initialize(); + _typesInitialized = true; + + // Initialize modules that depend on the runtime class. + AssemblyManager.Initialize(); + OperatorMethod.Initialize(); + if (RuntimeData.HasStashData()) + { + RuntimeData.RestoreRuntimeData(); + } + else + { + PyCLRMetaType = MetaType.Initialize(); + ImportHook.Initialize(); + } + Exceptions.Initialize(); + + // Need to add the runtime directory to sys.path so that we + // can find built-in assemblies like System.Data, et. al. + string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); + BorrowedReference path = PySys_GetObject("path"); + using var item = PyString_FromString(rtdir); + if (PySequence_Contains(path, item.Borrow()) == 0) + { + PyList_Append(path, item.Borrow()); + } + AssemblyManager.UpdatePath(); + + clrInterop = GetModuleLazy("clr.interop"); + inspect = GetModuleLazy("inspect"); + hexCallable = new(() => new PyString("%x").GetAttr("__mod__")); + } + + static void NewRun() + { + run++; + using var pyRun = PyLong_FromLongLong(run); + PySys_SetObject(RunSysPropName, pyRun.BorrowOrThrow()); + } + + private static void InitPyMembers() + { + using (var builtinsOwned = PyImport_ImportModule("builtins")) + { + var builtins = builtinsOwned.Borrow(); + SetPyMember(out PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented").StealNullable()); + + SetPyMember(out PyBaseObjectType, PyObject_GetAttrString(builtins, "object").StealNullable()); + + SetPyMember(out _PyNone, PyObject_GetAttrString(builtins, "None").StealNullable()); + SetPyMember(out _PyTrue, PyObject_GetAttrString(builtins, "True").StealNullable()); + SetPyMember(out _PyFalse, PyObject_GetAttrString(builtins, "False").StealNullable()); + + SetPyMemberTypeOf(out PyBoolType, _PyTrue!); + SetPyMemberTypeOf(out PyNoneType, _PyNone!); + + SetPyMemberTypeOf(out PyMethodType, PyObject_GetAttrString(builtins, "len").StealNullable()); + + // For some arcane reason, builtins.__dict__.__setitem__ is *not* + // a wrapper_descriptor, even though dict.__setitem__ is. + // + // object.__init__ seems safe, though. + SetPyMemberTypeOf(out PyWrapperDescriptorType, PyObject_GetAttrString(PyBaseObjectType, "__init__").StealNullable()); + + SetPyMember(out PySuper_Type, PyObject_GetAttrString(builtins, "super").StealNullable()); + } + + SetPyMemberTypeOf(out PyStringType, PyString_FromString("string").StealNullable()); + + SetPyMemberTypeOf(out PyUnicodeType, PyString_FromString("unicode").StealNullable()); + + SetPyMemberTypeOf(out PyBytesType, EmptyPyBytes().StealNullable()); + + SetPyMemberTypeOf(out PyTupleType, PyTuple_New(0).StealNullable()); + + SetPyMemberTypeOf(out PyListType, PyList_New(0).StealNullable()); + + SetPyMemberTypeOf(out PyDictType, PyDict_New().StealNullable()); + + SetPyMemberTypeOf(out PyLongType, PyInt_FromInt32(0).StealNullable()); + + SetPyMemberTypeOf(out PyFloatType, PyFloat_FromDouble(0).StealNullable()); + + _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); + { + using var sys = PyImport_ImportModule("sys"); + SetPyMemberTypeOf(out PyModuleType, sys.StealNullable()); + } + } + + private static NativeFunc* Get_PyObject_NextNotImplemented() + { + using var pyType = SlotHelper.CreateObjectType(); + return Util.ReadPtr(pyType.Borrow(), TypeOffset.tp_iternext); + } + + internal static void Shutdown() + { + if (Py_IsInitialized() == 0 || !_isInitialized) + { + return; + } + _isInitialized = false; + + var state = PyGILState_Ensure(); + + if (!HostedInPython && !ProcessIsTerminating) + { + // avoid saving dead objects + TryCollectingGarbage(runs: 3); + + RuntimeData.Stash(); + } + + AssemblyManager.Shutdown(); + OperatorMethod.Shutdown(); + ImportHook.Shutdown(); + + ClearClrModules(); + RemoveClrRootModule(); + + NullGCHandles(ExtensionType.loadedExtensions); + ClassManager.RemoveClasses(); + TypeManager.RemoveTypes(); + _typesInitialized = false; + + MetaType.Release(); + PyCLRMetaType.Dispose(); + PyCLRMetaType = null!; + + Exceptions.Shutdown(); + PythonEngine.InteropConfiguration.Dispose(); + DisposeLazyObject(clrInterop); + DisposeLazyObject(inspect); + DisposeLazyObject(hexCallable); + PyObjectConversions.Reset(); + + PyGC_Collect(); + bool everythingSeemsCollected = TryCollectingGarbage(MaxCollectRetriesOnShutdown, + forceBreakLoops: true); + Debug.Assert(everythingSeemsCollected); + + Finalizer.Shutdown(); + InternString.Shutdown(); + + ResetPyMembers(); + + if (!HostedInPython) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + PyGILState_Release(state); + // Then release the GIL for good, if there is somehting to release + // Use the unchecked version as the checked version calls `abort()` + // if the current state is NULL. + if (_PyThreadState_UncheckedGet() != (PyThreadState*)0) + { + PyEval_SaveThread(); + } + + ExtensionType.loadedExtensions.Clear(); + CLRObject.reflectedObjects.Clear(); + } + else + { + PyGILState_Release(state); + } + } + + const int MaxCollectRetriesOnShutdown = 20; + internal static int _collected; + static bool TryCollectingGarbage(int runs, bool forceBreakLoops) + { + if (runs <= 0) throw new ArgumentOutOfRangeException(nameof(runs)); + + for (int attempt = 0; attempt < runs; attempt++) + { + Interlocked.Exchange(ref _collected, 0); + nint pyCollected = 0; + for (int i = 0; i < 2; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + pyCollected += PyGC_Collect(); + pyCollected += Finalizer.Instance.DisposeAll(); + } + if (Volatile.Read(ref _collected) == 0 && pyCollected == 0) + { + if (attempt + 1 == runs) return true; + } + else if (forceBreakLoops) + { + NullGCHandles(CLRObject.reflectedObjects); + CLRObject.reflectedObjects.Clear(); + } + } + return false; + } + /// + /// Alternates .NET and Python GC runs in an attempt to collect all garbage + /// + /// Total number of GC loops to run + /// true if a steady state was reached upon the requested number of tries (e.g. on the last try no objects were collected). + [ForbidPythonThreads] + public static bool TryCollectingGarbage(int runs) + => TryCollectingGarbage(runs, forceBreakLoops: false); + + static void DisposeLazyObject(Lazy pyObject) + { + if (pyObject.IsValueCreated) + { + pyObject.Value.Dispose(); + } + } + + private static Lazy GetModuleLazy(string moduleName) + => moduleName is null + ? throw new ArgumentNullException(nameof(moduleName)) + : new Lazy(() => PyModule.Import(moduleName), isThreadSafe: false); + + private static void SetPyMember(out PyObject obj, StolenReference value) + { + // XXX: For current usages, value should not be null. + if (value == null) + { + throw PythonException.ThrowLastAsClrException(); + } + obj = new PyObject(value); + _pyRefs.Add(obj); + } + + private static void SetPyMemberTypeOf(out PyType obj, PyObject value) + { + var type = PyObject_Type(value); + obj = new PyType(type.StealOrThrow(), prevalidated: true); + _pyRefs.Add(obj); + } + + private static void SetPyMemberTypeOf(out PyObject obj, StolenReference value) + { + if (value == null) + { + throw PythonException.ThrowLastAsClrException(); + } + var @ref = new BorrowedReference(value.Pointer); + var type = PyObject_Type(@ref); + XDecref(value.AnalyzerWorkaround()); + SetPyMember(out obj, type.StealNullable()); + } + + private static void ResetPyMembers() + { + foreach (var pyObj in _pyRefs) + pyObj.Dispose(); + _pyRefs.Clear(); + } + + private static void ClearClrModules() + { + var modules = PyImport_GetModuleDict(); + using var items = PyDict_Items(modules); + nint length = PyList_Size(items.BorrowOrThrow()); + if (length < 0) throw PythonException.ThrowLastAsClrException(); + for (nint i = 0; i < length; i++) + { + var item = PyList_GetItem(items.Borrow(), i); + var name = PyTuple_GetItem(item, 0); + var module = PyTuple_GetItem(item, 1); + if (ManagedType.IsInstanceOfManagedType(module)) + { + PyDict_DelItem(modules, name); + } + } + } + + private static void RemoveClrRootModule() + { + var modules = PyImport_GetModuleDict(); + PyDictTryDelItem(modules, "clr"); + PyDictTryDelItem(modules, "clr._extra"); + } + + private static void PyDictTryDelItem(BorrowedReference dict, string key) + { + if (PyDict_DelItemString(dict, key) == 0) + { + return; + } + if (!PythonException.CurrentMatches(Exceptions.KeyError)) + { + throw PythonException.ThrowLastAsClrException(); + } + PyErr_Clear(); + } + + private static void NullGCHandles(IEnumerable objects) + { + foreach (IntPtr objWithGcHandle in objects.ToArray()) + { + var @ref = new BorrowedReference(objWithGcHandle); + ManagedType.TryFreeGCHandle(@ref); + } + } + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + // these objects are initialized in Initialize rather than in constructor + internal static PyObject PyBaseObjectType; + internal static PyObject PyModuleType; + internal static PyObject PySuper_Type; + internal static PyType PyCLRMetaType; + internal static PyObject PyMethodType; + internal static PyObject PyWrapperDescriptorType; + + internal static PyObject PyUnicodeType; + internal static PyObject PyStringType; + internal static PyObject PyTupleType; + internal static PyObject PyListType; + internal static PyObject PyDictType; + internal static PyObject PyLongType; + internal static PyObject PyFloatType; + internal static PyType PyBoolType; + internal static PyType PyNoneType; + internal static BorrowedReference PyTypeType => new(Delegates.PyType_Type); + + internal static PyObject PyBytesType; + internal static NativeFunc* _PyObject_NextNotImplemented; + + internal static PyObject PyNotImplemented; + internal const int Py_LT = 0; + internal const int Py_LE = 1; + internal const int Py_EQ = 2; + internal const int Py_NE = 3; + internal const int Py_GT = 4; + internal const int Py_GE = 5; + + internal static BorrowedReference PyTrue => _PyTrue; + static PyObject _PyTrue; + internal static BorrowedReference PyFalse => _PyFalse; + static PyObject _PyFalse; + internal static BorrowedReference PyNone => _PyNone; + private static PyObject _PyNone; + + private static Lazy inspect; + internal static PyObject InspectModule => inspect.Value; + + private static Lazy clrInterop; + internal static PyObject InteropModule => clrInterop.Value; + + private static Lazy hexCallable; + internal static PyObject HexCallable => hexCallable.Value; +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + + internal static BorrowedReference CLRMetaType => PyCLRMetaType; + + public static PyObject None => new(_PyNone); + + /// + /// Check if any Python Exceptions occurred. + /// If any exist throw new PythonException. + /// + /// + /// Can be used instead of `obj == IntPtr.Zero` for example. + /// + internal static void CheckExceptionOccurred() + { + if (PyErr_Occurred() != null) + { + throw PythonException.ThrowLastAsClrException(); + } + } + + internal static Type[]? PythonArgsToTypeArray(BorrowedReference arg) + { + return PythonArgsToTypeArray(arg, false); + } + + internal static Type[]? PythonArgsToTypeArray(BorrowedReference arg, bool mangleObjects) + { + // Given a PyObject * that is either a single type object or a + // tuple of (managed or unmanaged) type objects, return a Type[] + // containing the CLR Type objects that map to those types. + BorrowedReference args = arg; + NewReference newArgs = default; + + if (!PyTuple_Check(arg)) + { + newArgs = PyTuple_New(1); + args = newArgs.Borrow(); + PyTuple_SetItem(args, 0, arg); + } + + var n = PyTuple_Size(args); + var types = new Type[n]; + Type? t = null; + + for (var i = 0; i < n; i++) + { + BorrowedReference op = PyTuple_GetItem(args, i); + if (mangleObjects && (!PyType_Check(op))) + { + op = PyObject_TYPE(op); + } + ManagedType? mt = ManagedType.GetManagedObject(op); + + if (mt is ClassBase b) + { + var _type = b.type; + t = _type.Valid ? _type.Value : null; + } + else if (mt is CLRObject ob) + { + var inst = ob.inst; + if (inst is Type ty) + { + t = ty; + } + } + else + { + t = Converter.GetTypeByAlias(op); + } + + if (t == null) + { + types = null; + break; + } + types[i] = t; + } + newArgs.Dispose(); + return types; + } + + /// + /// Managed exports of the Python C API. Where appropriate, we do + /// some optimization to avoid managed <--> unmanaged transitions + /// (mostly for heavily used methods). + /// + [Obsolete("Use NewReference or PyObject constructor instead")] + internal static unsafe void XIncref(BorrowedReference op) + { +#if !CUSTOM_INCDEC_REF + Py_IncRef(op); + return; +#else + var p = (void*)op; + if ((void*)0 != p) + { + if (Is32Bit) + { + (*(int*)p)++; + } + else + { + (*(long*)p)++; + } + } +#endif + } + + internal static unsafe void XDecref(StolenReference op) + { +#if DEBUG + Debug.Assert(op == null || Refcount(new BorrowedReference(op.Pointer)) > 0); + Debug.Assert(_isInitialized || Py_IsInitialized() != 0 || _Py_IsFinalizing() != false); +#endif +#if !CUSTOM_INCDEC_REF + if (op == null) return; + Py_DecRef(op.AnalyzerWorkaround()); + return; +#else + var p = (void*)op; + if ((void*)0 != p) + { + if (Is32Bit) + { + --(*(int*)p); + } + else + { + --(*(long*)p); + } + if ((*(int*)p) == 0) + { + // PyObject_HEAD: struct _typeobject *ob_type + void* t = Is32Bit + ? (void*)(*((uint*)p + 1)) + : (void*)(*((ulong*)p + 1)); + // PyTypeObject: destructor tp_dealloc + void* f = Is32Bit + ? (void*)(*((uint*)t + 6)) + : (void*)(*((ulong*)t + 6)); + if ((void*)0 == f) + { + return; + } + NativeCall.Void_Call_1(new IntPtr(f), op); + } + } +#endif + } + + [Pure] + internal static unsafe nint Refcount(BorrowedReference op) + { + if (op == null) + { + return 0; + } + var p = (nint*)(op.DangerousGetAddress() + ABI.RefCountOffset); + return *p; + } + [Pure] + internal static int Refcount32(BorrowedReference op) => checked((int)Refcount(op)); + + internal static void TryUsingDll(Action op) => + TryUsingDll(() => { op(); return 0; }); + + /// + /// Call specified function, and handle PythonDLL-related failures. + /// + internal static T TryUsingDll(Func op) + { + try + { + return op(); + } + catch (TypeInitializationException loadFailure) + { + var delegatesLoadFailure = loadFailure; + // failure to load Delegates type might have been the cause + // of failure to load some higher-level type + while (delegatesLoadFailure.InnerException is TypeInitializationException nested) + { + delegatesLoadFailure = nested; + } + + if (delegatesLoadFailure.InnerException is BadPythonDllException badDll) + { + throw badDll; + } + + throw; + } + } + + /// + /// Export of Macro Py_XIncRef. Use XIncref instead. + /// Limit this function usage for Testing and Py_Debug builds + /// + /// PyObject Ptr + + internal static void Py_IncRef(BorrowedReference ob) => Delegates.Py_IncRef(ob); + + /// + /// Export of Macro Py_XDecRef. Use XDecref instead. + /// Limit this function usage for Testing and Py_Debug builds + /// + /// PyObject Ptr + + internal static void Py_DecRef(StolenReference ob) => Delegates.Py_DecRef(ob); + + + internal static void Py_Initialize() => Delegates.Py_Initialize(); + + + internal static void Py_InitializeEx(int initsigs) => Delegates.Py_InitializeEx(initsigs); + + + internal static int Py_IsInitialized() => Delegates.Py_IsInitialized(); + + + internal static void Py_Finalize() => Delegates.Py_Finalize(); + + + internal static PyThreadState* Py_NewInterpreter() => Delegates.Py_NewInterpreter(); + + + internal static void Py_EndInterpreter(PyThreadState* threadState) => Delegates.Py_EndInterpreter(threadState); + + + internal static PyThreadState* PyThreadState_New(PyInterpreterState* istate) => Delegates.PyThreadState_New(istate); + + + internal static PyThreadState* PyThreadState_Get() => Delegates.PyThreadState_Get(); + + + internal static PyThreadState* _PyThreadState_UncheckedGet() => Delegates._PyThreadState_UncheckedGet(); + + + internal static int PyGILState_Check() => Delegates.PyGILState_Check(); + internal static PyGILState PyGILState_Ensure() => Delegates.PyGILState_Ensure(); + + + internal static void PyGILState_Release(PyGILState gs) => Delegates.PyGILState_Release(gs); + + + + internal static PyThreadState* PyGILState_GetThisThreadState() => Delegates.PyGILState_GetThisThreadState(); + + + public static int Py_Main(int argc, string[] argv) + { + var marshaler = StrArrayMarshaler.GetInstance(null); + var argvPtr = marshaler.MarshalManagedToNative(argv); + try + { + return Delegates.Py_Main(argc, argvPtr); + } + finally + { + marshaler.CleanUpNativeData(argvPtr); + } + } + + internal static void PyEval_InitThreads() => Delegates.PyEval_InitThreads(); + + + internal static int PyEval_ThreadsInitialized() => Delegates.PyEval_ThreadsInitialized(); + + + internal static void PyEval_AcquireLock() => Delegates.PyEval_AcquireLock(); + + + internal static void PyEval_ReleaseLock() => Delegates.PyEval_ReleaseLock(); + + + internal static void PyEval_AcquireThread(PyThreadState* tstate) => Delegates.PyEval_AcquireThread(tstate); + + + internal static void PyEval_ReleaseThread(PyThreadState* tstate) => Delegates.PyEval_ReleaseThread(tstate); + + + internal static PyThreadState* PyEval_SaveThread() => Delegates.PyEval_SaveThread(); + + + internal static void PyEval_RestoreThread(PyThreadState* tstate) => Delegates.PyEval_RestoreThread(tstate); + + + internal static BorrowedReference PyEval_GetBuiltins() => Delegates.PyEval_GetBuiltins(); + + + internal static BorrowedReference PyEval_GetGlobals() => Delegates.PyEval_GetGlobals(); + + + internal static BorrowedReference PyEval_GetLocals() => Delegates.PyEval_GetLocals(); + + + internal static IntPtr Py_GetProgramName() => Delegates.Py_GetProgramName(); + + + internal static void Py_SetProgramName(IntPtr name) => Delegates.Py_SetProgramName(name); + + + internal static IntPtr Py_GetPythonHome() => Delegates.Py_GetPythonHome(); + + + internal static void Py_SetPythonHome(IntPtr home) => Delegates.Py_SetPythonHome(home); + + + internal static IntPtr Py_GetPath() => Delegates.Py_GetPath(); + + + internal static void Py_SetPath(IntPtr home) => Delegates.Py_SetPath(home); + + + internal static IntPtr Py_GetVersion() => Delegates.Py_GetVersion(); + + + internal static IntPtr Py_GetPlatform() => Delegates.Py_GetPlatform(); + + + internal static IntPtr Py_GetCopyright() => Delegates.Py_GetCopyright(); + + + internal static IntPtr Py_GetCompiler() => Delegates.Py_GetCompiler(); + + + internal static IntPtr Py_GetBuildInfo() => Delegates.Py_GetBuildInfo(); + + const PyCompilerFlags Utf8String = PyCompilerFlags.IGNORE_COOKIE | PyCompilerFlags.SOURCE_IS_UTF8; + + internal static int PyRun_SimpleString(string code) + { + using var codePtr = new StrPtr(code, Encoding.UTF8); + return Delegates.PyRun_SimpleStringFlags(codePtr, Utf8String); + } + + internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals) + { + using var codePtr = new StrPtr(code, Encoding.UTF8); + return Delegates.PyRun_StringFlags(codePtr, st, globals, locals, Utf8String); + } + + internal static NewReference PyEval_EvalCode(BorrowedReference co, BorrowedReference globals, BorrowedReference locals) => Delegates.PyEval_EvalCode(co, globals, locals); + + /// + /// Return value: New reference. + /// This is a simplified interface to Py_CompileStringFlags() below, leaving flags set to NULL. + /// + internal static NewReference Py_CompileString(string str, string file, int start) + { + using var strPtr = new StrPtr(str, Encoding.UTF8); + using var fileObj = new PyString(file); + return Delegates.Py_CompileStringObject(strPtr, fileObj, start, Utf8String, -1); + } + + internal static NewReference PyImport_ExecCodeModule(string name, BorrowedReference code) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyImport_ExecCodeModule(namePtr, code); + } + + //==================================================================== + // Python abstract object API + //==================================================================== + + /// + /// A macro-like method to get the type of a Python object. This is + /// designed to be lean and mean in IL & avoid managed <-> unmanaged + /// transitions. Note that this does not incref the type object. + /// + internal static unsafe BorrowedReference PyObject_TYPE(BorrowedReference op) + { + IntPtr address = op.DangerousGetAddressOrNull(); + if (address == IntPtr.Zero) + { + return BorrowedReference.Null; + } + Debug.Assert(TypeOffset.ob_type > 0); + BorrowedReference* typePtr = (BorrowedReference*)(address + TypeOffset.ob_type); + return *typePtr; + } + internal static NewReference PyObject_Type(BorrowedReference o) + => Delegates.PyObject_Type(o); + + internal static string PyObject_GetTypeName(BorrowedReference op) + { + Debug.Assert(TypeOffset.tp_name > 0); + Debug.Assert(op != null); + BorrowedReference pyType = PyObject_TYPE(op); + IntPtr ppName = Util.ReadIntPtr(pyType, TypeOffset.tp_name); + return Marshal.PtrToStringAnsi(ppName); + } + + /// + /// Test whether the Python object is an iterable. + /// + internal static bool PyObject_IsIterable(BorrowedReference ob) + { + var ob_type = PyObject_TYPE(ob); + return Util.ReadIntPtr(ob_type, TypeOffset.tp_iter) != IntPtr.Zero; + } + + internal static int PyObject_HasAttrString(BorrowedReference pointer, string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_HasAttrString(pointer, namePtr); + } + + internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_GetAttrString(pointer, namePtr); + } + + internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, StrPtr name) + => Delegates.PyObject_GetAttrString(pointer, name); + + + internal static int PyObject_DelAttr(BorrowedReference @object, BorrowedReference name) => Delegates.PyObject_SetAttr(@object, name, null); + internal static int PyObject_DelAttrString(BorrowedReference @object, string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_SetAttrString(@object, namePtr, null); + } + internal static int PyObject_SetAttrString(BorrowedReference @object, string name, BorrowedReference value) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyObject_SetAttrString(@object, namePtr, value); + } + + internal static int PyObject_HasAttr(BorrowedReference pointer, BorrowedReference name) => Delegates.PyObject_HasAttr(pointer, name); + + + internal static NewReference PyObject_GetAttr(BorrowedReference pointer, IntPtr name) + => Delegates.PyObject_GetAttr(pointer, new BorrowedReference(name)); + internal static NewReference PyObject_GetAttr(BorrowedReference o, BorrowedReference name) => Delegates.PyObject_GetAttr(o, name); + + + internal static int PyObject_SetAttr(BorrowedReference o, BorrowedReference name, BorrowedReference value) => Delegates.PyObject_SetAttr(o, name, value); + + + internal static NewReference PyObject_GetItem(BorrowedReference o, BorrowedReference key) => Delegates.PyObject_GetItem(o, key); + + + internal static int PyObject_SetItem(BorrowedReference o, BorrowedReference key, BorrowedReference value) => Delegates.PyObject_SetItem(o, key, value); + + + internal static int PyObject_DelItem(BorrowedReference o, BorrowedReference key) => Delegates.PyObject_DelItem(o, key); + + + internal static NewReference PyObject_GetIter(BorrowedReference op) => Delegates.PyObject_GetIter(op); + + + internal static NewReference PyObject_Call(BorrowedReference pointer, BorrowedReference args, BorrowedReference kw) => Delegates.PyObject_Call(pointer, args, kw); + + internal static NewReference PyObject_CallObject(BorrowedReference callable, BorrowedReference args) => Delegates.PyObject_CallObject(callable, args); + internal static IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args) + => Delegates.PyObject_CallObject(new BorrowedReference(pointer), new BorrowedReference(args)) + .DangerousMoveToPointerOrNull(); + + + internal static int PyObject_RichCompareBool(BorrowedReference value1, BorrowedReference value2, int opid) => Delegates.PyObject_RichCompareBool(value1, value2, opid); + + + internal static int PyObject_IsInstance(BorrowedReference ob, BorrowedReference type) => Delegates.PyObject_IsInstance(ob, type); + + + internal static int PyObject_IsSubclass(BorrowedReference ob, BorrowedReference type) => Delegates.PyObject_IsSubclass(ob, type); + + internal static void PyObject_ClearWeakRefs(BorrowedReference ob) => Delegates.PyObject_ClearWeakRefs(ob); + + internal static BorrowedReference PyObject_GetWeakRefList(BorrowedReference ob) + { + Debug.Assert(ob != null); + var type = PyObject_TYPE(ob); + int offset = Util.ReadInt32(type, TypeOffset.tp_weaklistoffset); + if (offset == 0) return BorrowedReference.Null; + Debug.Assert(offset > 0); + return Util.ReadRef(ob, offset); + } + + + internal static int PyCallable_Check(BorrowedReference o) => Delegates.PyCallable_Check(o); + + + internal static int PyObject_IsTrue(IntPtr pointer) => PyObject_IsTrue(new BorrowedReference(pointer)); + internal static int PyObject_IsTrue(BorrowedReference pointer) => Delegates.PyObject_IsTrue(pointer); + + + internal static int PyObject_Not(BorrowedReference o) => Delegates.PyObject_Not(o); + + internal static nint PyObject_Size(BorrowedReference pointer) => Delegates.PyObject_Size(pointer); + + + internal static nint PyObject_Hash(BorrowedReference op) => Delegates.PyObject_Hash(op); + + + internal static NewReference PyObject_Repr(BorrowedReference pointer) + { + AssertNoErorSet(); + + return Delegates.PyObject_Repr(pointer); + } + + + internal static NewReference PyObject_Str(BorrowedReference pointer) + { + AssertNoErorSet(); + + return Delegates.PyObject_Str(pointer); + } + + [Conditional("DEBUG")] + internal static void AssertNoErorSet() + { + if (Exceptions.ErrorOccurred()) + throw new InvalidOperationException( + "Can't call with exception set", + PythonException.FetchCurrent()); + } + + + internal static NewReference PyObject_Dir(BorrowedReference pointer) => Delegates.PyObject_Dir(pointer); + + internal static void _Py_NewReference(BorrowedReference ob) + { + if (Delegates._Py_NewReference != null) + Delegates._Py_NewReference(ob); + } + + internal static bool? _Py_IsFinalizing() + { + if (Delegates._Py_IsFinalizing != null) + return Delegates._Py_IsFinalizing() != 0; + else + return null; ; + } + + //==================================================================== + // Python buffer API + //==================================================================== + + + internal static int PyObject_GetBuffer(BorrowedReference exporter, out Py_buffer view, int flags) => Delegates.PyObject_GetBuffer(exporter, out view, flags); + + + internal static void PyBuffer_Release(ref Py_buffer view) => Delegates.PyBuffer_Release(ref view); + + + internal static nint PyBuffer_SizeFromFormat(string format) + { + using var formatPtr = new StrPtr(format, Encoding.ASCII); + return Delegates.PyBuffer_SizeFromFormat(formatPtr); + } + + internal static int PyBuffer_IsContiguous(ref Py_buffer view, char order) => Delegates.PyBuffer_IsContiguous(ref view, order); + + + internal static IntPtr PyBuffer_GetPointer(ref Py_buffer view, nint[] indices) => Delegates.PyBuffer_GetPointer(ref view, indices); + + + internal static int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort) => Delegates.PyBuffer_FromContiguous(ref view, buf, len, fort); + + + internal static int PyBuffer_ToContiguous(IntPtr buf, ref Py_buffer src, IntPtr len, char order) => Delegates.PyBuffer_ToContiguous(buf, ref src, len, order); + + + internal static void PyBuffer_FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, char order) => Delegates.PyBuffer_FillContiguousStrides(ndims, shape, strides, itemsize, order); + + + internal static int PyBuffer_FillInfo(ref Py_buffer view, BorrowedReference exporter, IntPtr buf, IntPtr len, int _readonly, int flags) => Delegates.PyBuffer_FillInfo(ref view, exporter, buf, len, _readonly, flags); + + //==================================================================== + // Python number API + //==================================================================== + + + internal static NewReference PyNumber_Long(BorrowedReference ob) => Delegates.PyNumber_Long(ob); + + + internal static NewReference PyNumber_Float(BorrowedReference ob) => Delegates.PyNumber_Float(ob); + + + internal static bool PyNumber_Check(BorrowedReference ob) => Delegates.PyNumber_Check(ob); + + internal static bool PyInt_Check(BorrowedReference ob) + => PyObject_TypeCheck(ob, PyLongType); + + internal static bool PyInt_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyLongType); + + internal static bool PyBool_Check(BorrowedReference ob) + => PyObject_TypeCheck(ob, PyBoolType); + internal static bool PyBool_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyBoolType); + + internal static NewReference PyInt_FromInt32(int value) => PyLong_FromLongLong(value); + + internal static NewReference PyInt_FromInt64(long value) => PyLong_FromLongLong(value); + + internal static NewReference PyLong_FromLongLong(long value) => Delegates.PyLong_FromLongLong(value); + + + internal static NewReference PyLong_FromUnsignedLongLong(ulong value) => Delegates.PyLong_FromUnsignedLongLong(value); + + + internal static NewReference PyLong_FromString(string value, int radix) + { + using var valPtr = new StrPtr(value, Encoding.UTF8); + return Delegates.PyLong_FromString(valPtr, IntPtr.Zero, radix); + } + + + + internal static nuint PyLong_AsUnsignedSize_t(BorrowedReference value) => Delegates.PyLong_AsUnsignedSize_t(value); + + internal static nint PyLong_AsSignedSize_t(BorrowedReference value) => Delegates.PyLong_AsSignedSize_t(value); + + internal static long? PyLong_AsLongLong(BorrowedReference value) + { + long result = Delegates.PyLong_AsLongLong(value); + if (result == -1 && Exceptions.ErrorOccurred()) + { + return null; + } + return result; + } + + internal static ulong? PyLong_AsUnsignedLongLong(BorrowedReference value) + { + ulong result = Delegates.PyLong_AsUnsignedLongLong(value); + if (result == unchecked((ulong)-1) && Exceptions.ErrorOccurred()) + { + return null; + } + return result; + } + + internal static bool PyFloat_Check(BorrowedReference ob) + => PyObject_TypeCheck(ob, PyFloatType); + internal static bool PyFloat_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyFloatType); + + /// + /// Return value: New reference. + /// Create a Python integer from the pointer p. The pointer value can be retrieved from the resulting value using PyLong_AsVoidPtr(). + /// + internal static NewReference PyLong_FromVoidPtr(IntPtr p) => Delegates.PyLong_FromVoidPtr(p); + + /// + /// Convert a Python integer pylong to a C void pointer. If pylong cannot be converted, an OverflowError will be raised. This is only assured to produce a usable void pointer for values created with PyLong_FromVoidPtr(). + /// + + internal static IntPtr PyLong_AsVoidPtr(BorrowedReference ob) => Delegates.PyLong_AsVoidPtr(ob); + + + internal static NewReference PyFloat_FromDouble(double value) => Delegates.PyFloat_FromDouble(value); + + + internal static NewReference PyFloat_FromString(BorrowedReference value) => Delegates.PyFloat_FromString(value); + + + internal static double PyFloat_AsDouble(BorrowedReference ob) => Delegates.PyFloat_AsDouble(ob); + + + internal static NewReference PyNumber_Add(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Add(o1, o2); + + + internal static NewReference PyNumber_Subtract(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Subtract(o1, o2); + + + internal static NewReference PyNumber_Multiply(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Multiply(o1, o2); + + + internal static NewReference PyNumber_TrueDivide(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_TrueDivide(o1, o2); + + + internal static NewReference PyNumber_And(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_And(o1, o2); + + + internal static NewReference PyNumber_Xor(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Xor(o1, o2); + + + internal static NewReference PyNumber_Or(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Or(o1, o2); + + + internal static NewReference PyNumber_Lshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Lshift(o1, o2); + + + internal static NewReference PyNumber_Rshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Rshift(o1, o2); + + + internal static NewReference PyNumber_Power(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Power(o1, o2); + + + internal static NewReference PyNumber_Remainder(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Remainder(o1, o2); + + + internal static NewReference PyNumber_InPlaceAdd(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceAdd(o1, o2); + + + internal static NewReference PyNumber_InPlaceSubtract(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceSubtract(o1, o2); + + + internal static NewReference PyNumber_InPlaceMultiply(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceMultiply(o1, o2); + + + internal static NewReference PyNumber_InPlaceTrueDivide(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceTrueDivide(o1, o2); + + + internal static NewReference PyNumber_InPlaceAnd(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceAnd(o1, o2); + + + internal static NewReference PyNumber_InPlaceXor(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceXor(o1, o2); + + + internal static NewReference PyNumber_InPlaceOr(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceOr(o1, o2); + + + internal static NewReference PyNumber_InPlaceLshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceLshift(o1, o2); + + + internal static NewReference PyNumber_InPlaceRshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceRshift(o1, o2); + + + internal static NewReference PyNumber_InPlacePower(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlacePower(o1, o2); + + + internal static NewReference PyNumber_InPlaceRemainder(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceRemainder(o1, o2); + + + internal static NewReference PyNumber_Negative(BorrowedReference o1) => Delegates.PyNumber_Negative(o1); + + + internal static NewReference PyNumber_Positive(BorrowedReference o1) => Delegates.PyNumber_Positive(o1); + + + internal static NewReference PyNumber_Invert(BorrowedReference o1) => Delegates.PyNumber_Invert(o1); + + + //==================================================================== + // Python sequence API + //==================================================================== + + + internal static bool PySequence_Check(BorrowedReference pointer) => Delegates.PySequence_Check(pointer); + + internal static NewReference PySequence_GetItem(BorrowedReference pointer, nint index) => Delegates.PySequence_GetItem(pointer, index); + internal static int PySequence_SetItem(BorrowedReference pointer, nint index, BorrowedReference value) => Delegates.PySequence_SetItem(pointer, index, value); + + internal static int PySequence_DelItem(BorrowedReference pointer, nint index) => Delegates.PySequence_DelItem(pointer, index); + + internal static NewReference PySequence_GetSlice(BorrowedReference pointer, nint i1, nint i2) => Delegates.PySequence_GetSlice(pointer, i1, i2); + + internal static int PySequence_SetSlice(BorrowedReference pointer, nint i1, nint i2, BorrowedReference v) => Delegates.PySequence_SetSlice(pointer, i1, i2, v); + + internal static int PySequence_DelSlice(BorrowedReference pointer, nint i1, nint i2) => Delegates.PySequence_DelSlice(pointer, i1, i2); + + internal static nint PySequence_Size(BorrowedReference pointer) => Delegates.PySequence_Size(pointer); + + internal static int PySequence_Contains(BorrowedReference pointer, BorrowedReference item) => Delegates.PySequence_Contains(pointer, item); + + + internal static NewReference PySequence_Concat(BorrowedReference pointer, BorrowedReference other) => Delegates.PySequence_Concat(pointer, other); + + internal static NewReference PySequence_Repeat(BorrowedReference pointer, nint count) => Delegates.PySequence_Repeat(pointer, count); + + + internal static nint PySequence_Index(BorrowedReference pointer, BorrowedReference item) => Delegates.PySequence_Index(pointer, item); + + private static nint PySequence_Count(BorrowedReference pointer, BorrowedReference value) => Delegates.PySequence_Count(pointer, value); + + + internal static NewReference PySequence_Tuple(BorrowedReference pointer) => Delegates.PySequence_Tuple(pointer); + + + internal static NewReference PySequence_List(BorrowedReference pointer) => Delegates.PySequence_List(pointer); + + + //==================================================================== + // Python string API + //==================================================================== + internal static bool PyString_Check(BorrowedReference ob) + => PyObject_TypeCheck(ob, PyStringType); + internal static bool PyString_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyStringType); + + internal static NewReference PyString_FromString(string value) + { + fixed(char* ptr = value) + return Delegates.PyUnicode_DecodeUTF16( + (IntPtr)ptr, + value.Length * sizeof(Char), + IntPtr.Zero, + IntPtr.Zero + ); + } + + + internal static NewReference EmptyPyBytes() + { + byte* bytes = stackalloc byte[1]; + bytes[0] = 0; + return Delegates.PyBytes_FromString((IntPtr)bytes); + } + + internal static NewReference PyByteArray_FromStringAndSize(IntPtr strPtr, nint len) => Delegates.PyByteArray_FromStringAndSize(strPtr, len); + internal static NewReference PyByteArray_FromStringAndSize(string s) + { + using var ptr = new StrPtr(s, Encoding.UTF8); + return PyByteArray_FromStringAndSize(ptr.RawPointer, checked((nint)ptr.ByteCount)); + } + + internal static IntPtr PyBytes_AsString(BorrowedReference ob) + { + Debug.Assert(ob != null); + return Delegates.PyBytes_AsString(ob); + } + + internal static nint PyBytes_Size(BorrowedReference op) => Delegates.PyBytes_Size(op); + + internal static IntPtr PyUnicode_AsUTF8(BorrowedReference unicode) => Delegates.PyUnicode_AsUTF8(unicode); + + /// Length in code points + internal static nint PyUnicode_GetLength(BorrowedReference ob) => Delegates.PyUnicode_GetLength(ob); + + + internal static IntPtr PyUnicode_AsUnicode(BorrowedReference ob) => Delegates.PyUnicode_AsUnicode(ob); + internal static NewReference PyUnicode_AsUTF16String(BorrowedReference ob) => Delegates.PyUnicode_AsUTF16String(ob); + + + + internal static NewReference PyUnicode_FromOrdinal(int c) => Delegates.PyUnicode_FromOrdinal(c); + + internal static NewReference PyUnicode_InternFromString(string s) + { + using var ptr = new StrPtr(s, Encoding.UTF8); + return Delegates.PyUnicode_InternFromString(ptr); + } + + internal static int PyUnicode_Compare(BorrowedReference left, BorrowedReference right) => Delegates.PyUnicode_Compare(left, right); + + internal static string ToString(BorrowedReference op) + { + using var strval = PyObject_Str(op); + return GetManagedStringFromUnicodeObject(strval.BorrowOrThrow())!; + } + + /// + /// Function to access the internal PyUnicode/PyString object and + /// convert it to a managed string with the correct encoding. + /// + /// + /// We can't easily do this through through the CustomMarshaler's on + /// the returns because will have access to the IntPtr but not size. + /// + /// For PyUnicodeType, we can't convert with Marshal.PtrToStringUni + /// since it only works for UCS2. + /// + /// PyStringType or PyUnicodeType object to convert + /// Managed String + internal static string? GetManagedString(in BorrowedReference op) + { + var type = PyObject_TYPE(op); + + if (type == PyUnicodeType) + { + return GetManagedStringFromUnicodeObject(op); + } + + return null; + } + + static string GetManagedStringFromUnicodeObject(BorrowedReference op) + { +#if DEBUG + var type = PyObject_TYPE(op); + Debug.Assert(type == PyUnicodeType); +#endif + using var bytes = PyUnicode_AsUTF16String(op); + if (bytes.IsNull()) + { + throw PythonException.ThrowLastAsClrException(); + } + int bytesLength = checked((int)PyBytes_Size(bytes.Borrow())); + char* codePoints = (char*)PyBytes_AsString(bytes.Borrow()); + return new string(codePoints, + startIndex: 1, // skip BOM + length: bytesLength / 2 - 1); // utf16 - BOM + } + + + //==================================================================== + // Python dictionary API + //==================================================================== + + internal static bool PyDict_Check(BorrowedReference ob) + { + return PyObject_TYPE(ob) == PyDictType; + } + + + internal static NewReference PyDict_New() => Delegates.PyDict_New(); + + /// + /// Return NULL if the key is not present, but without setting an exception. + /// + internal static BorrowedReference PyDict_GetItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItem(pointer, key); + + internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key) + { + using var keyStr = new StrPtr(key, Encoding.UTF8); + return Delegates.PyDict_GetItemString(pointer, keyStr); + } + + internal static BorrowedReference PyDict_GetItemWithError(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItemWithError(pointer, key); + + /// + /// Return 0 on success or -1 on failure. + /// + internal static int PyDict_SetItem(BorrowedReference dict, BorrowedReference key, BorrowedReference value) => Delegates.PyDict_SetItem(dict, key, value); + + /// + /// Return 0 on success or -1 on failure. + /// + internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value) + { + using var keyPtr = new StrPtr(key, Encoding.UTF8); + return Delegates.PyDict_SetItemString(dict, keyPtr, value); + } + + internal static int PyDict_DelItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_DelItem(pointer, key); + + + internal static int PyDict_DelItemString(BorrowedReference pointer, string key) + { + using var keyPtr = new StrPtr(key, Encoding.UTF8); + return Delegates.PyDict_DelItemString(pointer, keyPtr); + } + + internal static int PyMapping_HasKey(BorrowedReference pointer, BorrowedReference key) => Delegates.PyMapping_HasKey(pointer, key); + + + internal static NewReference PyDict_Keys(BorrowedReference pointer) => Delegates.PyDict_Keys(pointer); + + internal static NewReference PyDict_Values(BorrowedReference pointer) => Delegates.PyDict_Values(pointer); + + internal static NewReference PyDict_Items(BorrowedReference pointer) => Delegates.PyDict_Items(pointer); + + + internal static NewReference PyDict_Copy(BorrowedReference pointer) => Delegates.PyDict_Copy(pointer); + + + internal static int PyDict_Update(BorrowedReference pointer, BorrowedReference other) => Delegates.PyDict_Update(pointer, other); + + + internal static void PyDict_Clear(BorrowedReference pointer) => Delegates.PyDict_Clear(pointer); + + internal static nint PyDict_Size(BorrowedReference pointer) => Delegates.PyDict_Size(pointer); + + + internal static NewReference PySet_New(BorrowedReference iterable) => Delegates.PySet_New(iterable); + + + internal static int PySet_Add(BorrowedReference set, BorrowedReference key) => Delegates.PySet_Add(set, key); + + /// + /// Return 1 if found, 0 if not found, and -1 if an error is encountered. + /// + + internal static int PySet_Contains(BorrowedReference anyset, BorrowedReference key) => Delegates.PySet_Contains(anyset, key); + + //==================================================================== + // Python list API + //==================================================================== + + internal static bool PyList_Check(BorrowedReference ob) + { + return PyObject_TYPE(ob) == PyListType; + } + + internal static NewReference PyList_New(nint size) => Delegates.PyList_New(size); + + internal static BorrowedReference PyList_GetItem(BorrowedReference pointer, nint index) => Delegates.PyList_GetItem(pointer, index); + + internal static int PyList_SetItem(BorrowedReference pointer, nint index, StolenReference value) => Delegates.PyList_SetItem(pointer, index, value); + + internal static int PyList_Insert(BorrowedReference pointer, nint index, BorrowedReference value) => Delegates.PyList_Insert(pointer, index, value); + + + internal static int PyList_Append(BorrowedReference pointer, BorrowedReference value) => Delegates.PyList_Append(pointer, value); + + + internal static int PyList_Reverse(BorrowedReference pointer) => Delegates.PyList_Reverse(pointer); + + + internal static int PyList_Sort(BorrowedReference pointer) => Delegates.PyList_Sort(pointer); + + private static NewReference PyList_GetSlice(BorrowedReference pointer, nint start, nint end) => Delegates.PyList_GetSlice(pointer, start, end); + + private static int PyList_SetSlice(BorrowedReference pointer, nint start, nint end, BorrowedReference value) => Delegates.PyList_SetSlice(pointer, start, end, value); + + + internal static nint PyList_Size(BorrowedReference pointer) => Delegates.PyList_Size(pointer); + + //==================================================================== + // Python tuple API + //==================================================================== + + internal static bool PyTuple_Check(BorrowedReference ob) + { + return PyObject_TYPE(ob) == PyTupleType; + } + internal static NewReference PyTuple_New(nint size) => Delegates.PyTuple_New(size); + + internal static BorrowedReference PyTuple_GetItem(BorrowedReference pointer, nint index) => Delegates.PyTuple_GetItem(pointer, index); + + internal static int PyTuple_SetItem(BorrowedReference pointer, nint index, BorrowedReference value) + { + var newRef = new NewReference(value); + return PyTuple_SetItem(pointer, index, newRef.Steal()); + } + + internal static int PyTuple_SetItem(BorrowedReference pointer, nint index, StolenReference value) => Delegates.PyTuple_SetItem(pointer, index, value); + + internal static NewReference PyTuple_GetSlice(BorrowedReference pointer, nint start, nint end) => Delegates.PyTuple_GetSlice(pointer, start, end); + + internal static nint PyTuple_Size(BorrowedReference pointer) => Delegates.PyTuple_Size(pointer); + + + //==================================================================== + // Python iterator API + //==================================================================== + internal static bool PyIter_Check(BorrowedReference ob) + { + if (Delegates.PyIter_Check != null) + return Delegates.PyIter_Check(ob) != 0; + var ob_type = PyObject_TYPE(ob); + var tp_iternext = (NativeFunc*)Util.ReadIntPtr(ob_type, TypeOffset.tp_iternext); + return tp_iternext != (NativeFunc*)0 && tp_iternext != _PyObject_NextNotImplemented; + } + internal static NewReference PyIter_Next(BorrowedReference pointer) => Delegates.PyIter_Next(pointer); + + + //==================================================================== + // Python module API + //==================================================================== + + + internal static NewReference PyModule_New(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyModule_New(namePtr); + } + + internal static BorrowedReference PyModule_GetDict(BorrowedReference module) => Delegates.PyModule_GetDict(module); + + internal static NewReference PyImport_Import(BorrowedReference name) => Delegates.PyImport_Import(name); + + /// The module to add the object to. + /// The key that will refer to the object. + /// The object to add to the module. + /// Return -1 on error, 0 on success. + internal static int PyModule_AddObject(BorrowedReference module, string name, StolenReference value) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + IntPtr valueAddr = value.DangerousGetAddressOrNull(); + int res = Delegates.PyModule_AddObject(module, namePtr, valueAddr); + // We can't just exit here because the reference is stolen only on success. + if (res != 0) + { + XDecref(StolenReference.TakeNullable(ref valueAddr)); + } + return res; + + } + + /// + /// Return value: New reference. + /// + + internal static NewReference PyImport_ImportModule(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyImport_ImportModule(namePtr); + } + + internal static NewReference PyImport_ReloadModule(BorrowedReference module) => Delegates.PyImport_ReloadModule(module); + + + internal static BorrowedReference PyImport_AddModule(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PyImport_AddModule(namePtr); + } + + internal static BorrowedReference PyImport_GetModuleDict() => Delegates.PyImport_GetModuleDict(); + + + internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath) + { + var marshaler = StrArrayMarshaler.GetInstance(null); + var argvPtr = marshaler.MarshalManagedToNative(argv); + try + { + Delegates.PySys_SetArgvEx(argc, argvPtr, updatepath); + } + finally + { + marshaler.CleanUpNativeData(argvPtr); + } + } + + /// + /// Return value: Borrowed reference. + /// Return the object name from the sys module or NULL if it does not exist, without setting an exception. + /// + + internal static BorrowedReference PySys_GetObject(string name) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PySys_GetObject(namePtr); + } + + internal static int PySys_SetObject(string name, BorrowedReference ob) + { + using var namePtr = new StrPtr(name, Encoding.UTF8); + return Delegates.PySys_SetObject(namePtr, ob); + } + + + //==================================================================== + // Python type object API + //==================================================================== + internal static bool PyType_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyTypeType); + + + internal static void PyType_Modified(BorrowedReference type) => Delegates.PyType_Modified(type); + internal static bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2) + { + Debug.Assert(t1 != null && t2 != null); + return Delegates.PyType_IsSubtype(t1, t2); + } + + internal static bool PyObject_TypeCheckExact(BorrowedReference ob, BorrowedReference tp) + => PyObject_TYPE(ob) == tp; + internal static bool PyObject_TypeCheck(BorrowedReference ob, BorrowedReference tp) + { + BorrowedReference t = PyObject_TYPE(ob); + return (t == tp) || PyType_IsSubtype(t, tp); + } + + internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedReference ofType) + { + return (type == ofType) || PyType_IsSubtype(type, ofType); + } + + + internal static NewReference PyType_GenericNew(BorrowedReference type, BorrowedReference args, BorrowedReference kw) => Delegates.PyType_GenericNew(type, args, kw); + + internal static NewReference PyType_GenericAlloc(BorrowedReference type, nint n) => Delegates.PyType_GenericAlloc(type, n); + + internal static IntPtr PyType_GetSlot(BorrowedReference type, TypeSlotID slot) => Delegates.PyType_GetSlot(type, slot); + internal static NewReference PyType_FromSpecWithBases(in NativeTypeSpec spec, BorrowedReference bases) => Delegates.PyType_FromSpecWithBases(in spec, bases); + + /// + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type�s base class. Return 0 on success, or return -1 and sets an exception on error. + /// + + internal static int PyType_Ready(BorrowedReference type) => Delegates.PyType_Ready(type); + + + internal static BorrowedReference _PyType_Lookup(BorrowedReference type, BorrowedReference name) => Delegates._PyType_Lookup(type, name); + + + internal static NewReference PyObject_GenericGetAttr(BorrowedReference obj, BorrowedReference name) => Delegates.PyObject_GenericGetAttr(obj, name); + + + internal static int PyObject_GenericSetAttr(BorrowedReference obj, BorrowedReference name, BorrowedReference value) => Delegates.PyObject_GenericSetAttr(obj, name, value); + + internal static NewReference PyObject_GenericGetDict(BorrowedReference o) => PyObject_GenericGetDict(o, IntPtr.Zero); + internal static NewReference PyObject_GenericGetDict(BorrowedReference o, IntPtr context) => Delegates.PyObject_GenericGetDict(o, context); + + internal static void PyObject_GC_Del(StolenReference ob) => Delegates.PyObject_GC_Del(ob); + + + internal static bool PyObject_GC_IsTracked(BorrowedReference ob) + { + if (PyVersion >= new Version(3, 9)) + return Delegates.PyObject_GC_IsTracked(ob) != 0; + + throw new NotSupportedException("Requires Python 3.9"); + } + + internal static void PyObject_GC_Track(BorrowedReference ob) => Delegates.PyObject_GC_Track(ob); + + internal static void PyObject_GC_UnTrack(BorrowedReference ob) => Delegates.PyObject_GC_UnTrack(ob); + + internal static void _PyObject_Dump(BorrowedReference ob) => Delegates._PyObject_Dump(ob); + + //==================================================================== + // Python memory API + //==================================================================== + + internal static IntPtr PyMem_Malloc(long size) + { + return PyMem_Malloc(new IntPtr(size)); + } + + + private static IntPtr PyMem_Malloc(nint size) => Delegates.PyMem_Malloc(size); + + private static IntPtr PyMem_Realloc(IntPtr ptr, nint size) => Delegates.PyMem_Realloc(ptr, size); + + + internal static void PyMem_Free(IntPtr ptr) => Delegates.PyMem_Free(ptr); + + + //==================================================================== + // Python exception API + //==================================================================== + + + internal static void PyErr_SetString(BorrowedReference ob, string message) + { + using var msgPtr = new StrPtr(message, Encoding.UTF8); + Delegates.PyErr_SetString(ob, msgPtr); + } + + internal static void PyErr_SetObject(BorrowedReference type, BorrowedReference exceptionObject) => Delegates.PyErr_SetObject(type, exceptionObject); + + internal static int PyErr_ExceptionMatches(BorrowedReference exception) => Delegates.PyErr_ExceptionMatches(exception); + + + internal static int PyErr_GivenExceptionMatches(BorrowedReference given, BorrowedReference typeOrTypes) => Delegates.PyErr_GivenExceptionMatches(given, typeOrTypes); + + + internal static void PyErr_NormalizeException(ref NewReference type, ref NewReference val, ref NewReference tb) => Delegates.PyErr_NormalizeException(ref type, ref val, ref tb); + + + internal static BorrowedReference PyErr_Occurred() => Delegates.PyErr_Occurred(); + + + internal static void PyErr_Fetch(out NewReference type, out NewReference val, out NewReference tb) => Delegates.PyErr_Fetch(out type, out val, out tb); + + + internal static void PyErr_Restore(StolenReference type, StolenReference val, StolenReference tb) => Delegates.PyErr_Restore(type, val, tb); + + + internal static void PyErr_Clear() => Delegates.PyErr_Clear(); + + + internal static void PyErr_Print() => Delegates.PyErr_Print(); + + + internal static NewReference PyException_GetCause(BorrowedReference ex) + => Delegates.PyException_GetCause(ex); + internal static NewReference PyException_GetTraceback(BorrowedReference ex) + => Delegates.PyException_GetTraceback(ex); + + /// + /// Set the cause associated with the exception to cause. Use NULL to clear it. There is no type check to make sure that cause is either an exception instance or None. This steals a reference to cause. + /// + internal static void PyException_SetCause(BorrowedReference ex, StolenReference cause) + => Delegates.PyException_SetCause(ex, cause); + internal static int PyException_SetTraceback(BorrowedReference ex, BorrowedReference tb) + => Delegates.PyException_SetTraceback(ex, tb); + + //==================================================================== + // Cell API + //==================================================================== + + + internal static NewReference PyCell_Get(BorrowedReference cell) => Delegates.PyCell_Get(cell); + + + internal static int PyCell_Set(BorrowedReference cell, BorrowedReference value) => Delegates.PyCell_Set(cell, value); + + internal static nint PyGC_Collect() => Delegates.PyGC_Collect(); + internal static void Py_CLEAR(BorrowedReference ob, int offset) => ReplaceReference(ob, offset, default); + internal static void Py_CLEAR(ref T? ob) + where T: PyObject + { + ob?.Dispose(); + ob = null; + } + + internal static void ReplaceReference(BorrowedReference ob, int offset, StolenReference newValue) + { + IntPtr raw = Util.ReadIntPtr(ob, offset); + Util.WriteNullableRef(ob, offset, newValue); + XDecref(StolenReference.TakeNullable(ref raw)); + } + + //==================================================================== + // Python Capsules API + //==================================================================== + + + internal static NewReference PyCapsule_New(IntPtr pointer, IntPtr name, IntPtr destructor) + => Delegates.PyCapsule_New(pointer, name, destructor); + + internal static IntPtr PyCapsule_GetPointer(BorrowedReference capsule, IntPtr name) + { + return Delegates.PyCapsule_GetPointer(capsule, name); + } + + internal static int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer) => Delegates.PyCapsule_SetPointer(capsule, pointer); + + //==================================================================== + // Miscellaneous + //==================================================================== + + + internal static int PyThreadState_SetAsyncExcLLP64(uint id, BorrowedReference exc) => Delegates.PyThreadState_SetAsyncExcLLP64(id, exc); + + internal static int PyThreadState_SetAsyncExcLP64(ulong id, BorrowedReference exc) => Delegates.PyThreadState_SetAsyncExcLP64(id, exc); + + + internal static void SetNoSiteFlag() + { + TryUsingDll(() => + { + *Delegates.Py_NoSiteFlag = 1; + return *Delegates.Py_NoSiteFlag; + }); + } + } + + internal class BadPythonDllException : MissingMethodException + { + public BadPythonDllException(string message, Exception innerException) + : base(message, innerException) { } + } +} +using System; +using System.Diagnostics; +using System.Diagnostics.Contracts; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; +using System.Collections.Generic; +using Python.Runtime.Native; +using System.Linq; +using static System.FormattableString; + +namespace Python.Runtime +{ + /// + /// Encapsulates the low-level Python C API. Note that it is + /// the responsibility of the caller to have acquired the GIL + /// before calling any of these methods. + /// + public unsafe partial class Runtime + { + public static string? PythonDLL + { + get => _PythonDll; + set + { + if (_isInitialized) + throw new InvalidOperationException("This property must be set before runtime is initialized"); + _PythonDll = value; + } + } + + static string? _PythonDll = GetDefaultDllName(); + private static string? GetDefaultDllName() + { + string dll = Environment.GetEnvironmentVariable("PYTHONNET_PYDLL"); + if (dll is not null) return dll; + + string verString = Environment.GetEnvironmentVariable("PYTHONNET_PYVER"); + if (!Version.TryParse(verString, out var version)) return null; + + return GetDefaultDllName(version); + } + + private static string GetDefaultDllName(Version version) + { + string prefix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "" : "lib"; + string suffix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Invariant($"{version.Major}{version.Minor}") + : Invariant($"{version.Major}.{version.Minor}"); + string ext = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".dll" + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ".dylib" + : ".so"; + return prefix + "python" + suffix + ext; + } + + private static bool _isInitialized = false; + internal static bool IsInitialized => _isInitialized; + private static bool _typesInitialized = false; + internal static bool TypeManagerInitialized => _typesInitialized; + internal static readonly bool Is32Bit = IntPtr.Size == 4; + + // Available in newer .NET Core versions (>= 5) as IntPtr.MaxValue etc. + internal static readonly long IntPtrMaxValue = Is32Bit ? Int32.MaxValue : Int64.MaxValue; + internal static readonly long IntPtrMinValue = Is32Bit ? Int32.MinValue : Int64.MinValue; + internal static readonly ulong UIntPtrMaxValue = Is32Bit ? UInt32.MaxValue : UInt64.MaxValue; + + // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; + + internal static Version InteropVersion { get; } + = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version; + + public static int MainManagedThreadId { get; private set; } + + private static readonly List _pyRefs = new (); + + internal static Version PyVersion + { + get + { + var versionTuple = PySys_GetObject("version_info"); + var major = Converter.ToInt32(PyTuple_GetItem(versionTuple, 0)); + var minor = Converter.ToInt32(PyTuple_GetItem(versionTuple, 1)); + var micro = Converter.ToInt32(PyTuple_GetItem(versionTuple, 2)); + return new Version(major, minor, micro); + } + } + + const string RunSysPropName = "__pythonnet_run__"; + static int run = 0; + + internal static int GetRun() + { + int runNumber = run; + Debug.Assert(runNumber > 0, "This must only be called after Runtime is initialized at least once"); + return runNumber; + } + + internal static bool HostedInPython; + internal static bool ProcessIsTerminating; + + /// + /// Initialize the runtime... + /// + /// Always call this method from the Main thread. After the + /// first call to this method, the main thread has acquired the GIL. + internal static void Initialize(bool initSigs = false) + { + if (_isInitialized) + { + return; + } + _isInitialized = true; + + bool interpreterAlreadyInitialized = TryUsingDll( + () => Py_IsInitialized() != 0 + ); + if (!interpreterAlreadyInitialized) + { + Py_InitializeEx(initSigs ? 1 : 0); + + NewRun(); + + if (PyEval_ThreadsInitialized() == 0) + { + PyEval_InitThreads(); + } + RuntimeState.Save(); + } + else + { + if (!HostedInPython) + { + PyGILState_Ensure(); + } + + BorrowedReference pyRun = PySys_GetObject(RunSysPropName); + if (pyRun != null) + { + run = checked((int)PyLong_AsSignedSize_t(pyRun)); + } + else + { + NewRun(); + } + } + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; + + Finalizer.Initialize(); + + InitPyMembers(); + + ABI.Initialize(PyVersion); + + InternString.Initialize(); + + GenericUtil.Reset(); + ClassManager.Reset(); + ClassDerivedObject.Reset(); + TypeManager.Initialize(); + CLRObject.creationBlocked = false; + _typesInitialized = true; + + // Initialize modules that depend on the runtime class. + AssemblyManager.Initialize(); + OperatorMethod.Initialize(); + if (RuntimeData.HasStashData()) + { + RuntimeData.RestoreRuntimeData(); + } + else + { + PyCLRMetaType = MetaType.Initialize(); + ImportHook.Initialize(); + } + Exceptions.Initialize(); + + // Need to add the runtime directory to sys.path so that we + // can find built-in assemblies like System.Data, et. al. + string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); + BorrowedReference path = PySys_GetObject("path"); + using var item = PyString_FromString(rtdir); + if (PySequence_Contains(path, item.Borrow()) == 0) + { + PyList_Append(path, item.Borrow()); + } + AssemblyManager.UpdatePath(); + + clrInterop = GetModuleLazy("clr.interop"); + inspect = GetModuleLazy("inspect"); + hexCallable = new(() => new PyString("%x").GetAttr("__mod__")); + } + + static void NewRun() + { + run++; + using var pyRun = PyLong_FromLongLong(run); + PySys_SetObject(RunSysPropName, pyRun.BorrowOrThrow()); + } + + private static void InitPyMembers() + { + using (var builtinsOwned = PyImport_ImportModule("builtins")) + { + var builtins = builtinsOwned.Borrow(); + SetPyMember(out PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented").StealNullable()); + + SetPyMember(out PyBaseObjectType, PyObject_GetAttrString(builtins, "object").StealNullable()); + + SetPyMember(out _PyNone, PyObject_GetAttrString(builtins, "None").StealNullable()); + SetPyMember(out _PyTrue, PyObject_GetAttrString(builtins, "True").StealNullable()); + SetPyMember(out _PyFalse, PyObject_GetAttrString(builtins, "False").StealNullable()); + + SetPyMemberTypeOf(out PyBoolType, _PyTrue!); + SetPyMemberTypeOf(out PyNoneType, _PyNone!); + + SetPyMemberTypeOf(out PyMethodType, PyObject_GetAttrString(builtins, "len").StealNullable()); + + // For some arcane reason, builtins.__dict__.__setitem__ is *not* + // a wrapper_descriptor, even though dict.__setitem__ is. + // + // object.__init__ seems safe, though. + SetPyMemberTypeOf(out PyWrapperDescriptorType, PyObject_GetAttrString(PyBaseObjectType, "__init__").StealNullable()); + + SetPyMember(out PySuper_Type, PyObject_GetAttrString(builtins, "super").StealNullable()); + } + + SetPyMemberTypeOf(out PyStringType, PyString_FromString("string").StealNullable()); + + SetPyMemberTypeOf(out PyUnicodeType, PyString_FromString("unicode").StealNullable()); + + SetPyMemberTypeOf(out PyBytesType, EmptyPyBytes().StealNullable()); + + SetPyMemberTypeOf(out PyTupleType, PyTuple_New(0).StealNullable()); + + SetPyMemberTypeOf(out PyListType, PyList_New(0).StealNullable()); + + SetPyMemberTypeOf(out PyDictType, PyDict_New().StealNullable()); + + SetPyMemberTypeOf(out PyLongType, PyInt_FromInt32(0).StealNullable()); + + SetPyMemberTypeOf(out PyFloatType, PyFloat_FromDouble(0).StealNullable()); + + _PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented(); + { + using var sys = PyImport_ImportModule("sys"); + SetPyMemberTypeOf(out PyModuleType, sys.StealNullable()); + } + } + + private static NativeFunc* Get_PyObject_NextNotImplemented() + { + using var pyType = SlotHelper.CreateObjectType(); + return Util.ReadPtr(pyType.Borrow(), TypeOffset.tp_iternext); + } + + internal static void Shutdown() + { + if (Py_IsInitialized() == 0 || !_isInitialized) + { + return; + } + _isInitialized = false; + + var state = PyGILState_Ensure(); + + if (!HostedInPython && !ProcessIsTerminating) + { + // avoid saving dead objects + TryCollectingGarbage(runs: 3); + + RuntimeData.Stash(); + } + + AssemblyManager.Shutdown(); + OperatorMethod.Shutdown(); + ImportHook.Shutdown(); + + ClearClrModules(); + RemoveClrRootModule(); + + TryCollectingGarbage(MaxCollectRetriesOnShutdown, forceBreakLoops: true, + obj: true, derived: false, buffer: false); + CLRObject.creationBlocked = true; + + NullGCHandles(ExtensionType.loadedExtensions); + ClassManager.RemoveClasses(); + TypeManager.RemoveTypes(); + _typesInitialized = false; + + MetaType.Release(); + PyCLRMetaType.Dispose(); + PyCLRMetaType = null!; + + Exceptions.Shutdown(); + PythonEngine.InteropConfiguration.Dispose(); + DisposeLazyObject(clrInterop); + DisposeLazyObject(inspect); + DisposeLazyObject(hexCallable); + PyObjectConversions.Reset(); + + PyGC_Collect(); + bool everythingSeemsCollected = TryCollectingGarbage(MaxCollectRetriesOnShutdown); + Debug.Assert(everythingSeemsCollected); + + Finalizer.Shutdown(); + InternString.Shutdown(); + + ResetPyMembers(); + + if (!HostedInPython) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + PyGILState_Release(state); + // Then release the GIL for good, if there is somehting to release + // Use the unchecked version as the checked version calls `abort()` + // if the current state is NULL. + if (_PyThreadState_UncheckedGet() != (PyThreadState*)0) + { + PyEval_SaveThread(); + } + + ExtensionType.loadedExtensions.Clear(); + CLRObject.reflectedObjects.Clear(); + } + else + { + PyGILState_Release(state); + } + } + + const int MaxCollectRetriesOnShutdown = 20; + internal static int _collected; + static bool TryCollectingGarbage(int runs, bool forceBreakLoops, + bool obj = true, bool derived = true, bool buffer = true) + { + if (runs <= 0) throw new ArgumentOutOfRangeException(nameof(runs)); + + for (int attempt = 0; attempt < runs; attempt++) + { + Interlocked.Exchange(ref _collected, 0); + nint pyCollected = 0; + for (int i = 0; i < 2; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + pyCollected += PyGC_Collect(); + pyCollected += Finalizer.Instance.DisposeAll(disposeObj: obj, + disposeDerived: derived, + disposeBuffer: buffer); + } + if (Volatile.Read(ref _collected) == 0 && pyCollected == 0) + { + if (attempt + 1 == runs) return true; + } + else if (forceBreakLoops) + { + NullGCHandles(CLRObject.reflectedObjects); + CLRObject.reflectedObjects.Clear(); + } + } + return false; + } + /// + /// Alternates .NET and Python GC runs in an attempt to collect all garbage + /// + /// Total number of GC loops to run + /// true if a steady state was reached upon the requested number of tries (e.g. on the last try no objects were collected). + [ForbidPythonThreads] + public static bool TryCollectingGarbage(int runs) + => TryCollectingGarbage(runs, forceBreakLoops: false); + + static void DisposeLazyObject(Lazy pyObject) + { + if (pyObject.IsValueCreated) + { + pyObject.Value.Dispose(); + } + } + + private static Lazy GetModuleLazy(string moduleName) + => moduleName is null + ? throw new ArgumentNullException(nameof(moduleName)) + : new Lazy(() => PyModule.Import(moduleName), isThreadSafe: false); + + private static void SetPyMember(out PyObject obj, StolenReference value) + { + // XXX: For current usages, value should not be null. + if (value == null) + { + throw PythonException.ThrowLastAsClrException(); + } + obj = new PyObject(value); + _pyRefs.Add(obj); + } + + private static void SetPyMemberTypeOf(out PyType obj, PyObject value) + { + var type = PyObject_Type(value); + obj = new PyType(type.StealOrThrow(), prevalidated: true); + _pyRefs.Add(obj); + } + + private static void SetPyMemberTypeOf(out PyObject obj, StolenReference value) + { + if (value == null) + { + throw PythonException.ThrowLastAsClrException(); + } + var @ref = new BorrowedReference(value.Pointer); + var type = PyObject_Type(@ref); + XDecref(value.AnalyzerWorkaround()); + SetPyMember(out obj, type.StealNullable()); + } + + private static void ResetPyMembers() + { + foreach (var pyObj in _pyRefs) + pyObj.Dispose(); + _pyRefs.Clear(); + } + + private static void ClearClrModules() + { + var modules = PyImport_GetModuleDict(); + using var items = PyDict_Items(modules); + nint length = PyList_Size(items.BorrowOrThrow()); + if (length < 0) throw PythonException.ThrowLastAsClrException(); + for (nint i = 0; i < length; i++) + { + var item = PyList_GetItem(items.Borrow(), i); + var name = PyTuple_GetItem(item, 0); + var module = PyTuple_GetItem(item, 1); + if (ManagedType.IsInstanceOfManagedType(module)) + { + PyDict_DelItem(modules, name); + } + } + } + + private static void RemoveClrRootModule() + { + var modules = PyImport_GetModuleDict(); + PyDictTryDelItem(modules, "clr"); + PyDictTryDelItem(modules, "clr._extra"); + } + + private static void PyDictTryDelItem(BorrowedReference dict, string key) + { + if (PyDict_DelItemString(dict, key) == 0) + { + return; + } + if (!PythonException.CurrentMatches(Exceptions.KeyError)) + { + throw PythonException.ThrowLastAsClrException(); + } + PyErr_Clear(); + } + + private static void NullGCHandles(IEnumerable objects) + { + foreach (IntPtr objWithGcHandle in objects.ToArray()) + { + var @ref = new BorrowedReference(objWithGcHandle); + ManagedType.TryFreeGCHandle(@ref); + } + } + +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + // these objects are initialized in Initialize rather than in constructor + internal static PyObject PyBaseObjectType; + internal static PyObject PyModuleType; + internal static PyObject PySuper_Type; + internal static PyType PyCLRMetaType; + internal static PyObject PyMethodType; + internal static PyObject PyWrapperDescriptorType; + + internal static PyObject PyUnicodeType; + internal static PyObject PyStringType; + internal static PyObject PyTupleType; + internal static PyObject PyListType; + internal static PyObject PyDictType; + internal static PyObject PyLongType; + internal static PyObject PyFloatType; + internal static PyType PyBoolType; + internal static PyType PyNoneType; + internal static BorrowedReference PyTypeType => new(Delegates.PyType_Type); + + internal static PyObject PyBytesType; + internal static NativeFunc* _PyObject_NextNotImplemented; + + internal static PyObject PyNotImplemented; + internal const int Py_LT = 0; + internal const int Py_LE = 1; + internal const int Py_EQ = 2; + internal const int Py_NE = 3; + internal const int Py_GT = 4; + internal const int Py_GE = 5; + + internal static BorrowedReference PyTrue => _PyTrue; + static PyObject _PyTrue; + internal static BorrowedReference PyFalse => _PyFalse; + static PyObject _PyFalse; + internal static BorrowedReference PyNone => _PyNone; + private static PyObject _PyNone; + + private static Lazy inspect; + internal static PyObject InspectModule => inspect.Value; + + private static Lazy clrInterop; + internal static PyObject InteropModule => clrInterop.Value; + + private static Lazy hexCallable; + internal static PyObject HexCallable => hexCallable.Value; +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + + internal static BorrowedReference CLRMetaType => PyCLRMetaType; + + public static PyObject None => new(_PyNone); + + /// + /// Check if any Python Exceptions occurred. + /// If any exist throw new PythonException. + /// + /// + /// Can be used instead of `obj == IntPtr.Zero` for example. + /// + internal static void CheckExceptionOccurred() + { + if (PyErr_Occurred() != null) + { + throw PythonException.ThrowLastAsClrException(); + } + } + + internal static Type[]? PythonArgsToTypeArray(BorrowedReference arg) + { + return PythonArgsToTypeArray(arg, false); + } + + internal static Type[]? PythonArgsToTypeArray(BorrowedReference arg, bool mangleObjects) + { + // Given a PyObject * that is either a single type object or a + // tuple of (managed or unmanaged) type objects, return a Type[] + // containing the CLR Type objects that map to those types. + BorrowedReference args = arg; + NewReference newArgs = default; + + if (!PyTuple_Check(arg)) + { + newArgs = PyTuple_New(1); + args = newArgs.Borrow(); + PyTuple_SetItem(args, 0, arg); + } + + var n = PyTuple_Size(args); + var types = new Type[n]; + Type? t = null; + + for (var i = 0; i < n; i++) + { + BorrowedReference op = PyTuple_GetItem(args, i); + if (mangleObjects && (!PyType_Check(op))) + { + op = PyObject_TYPE(op); + } + ManagedType? mt = ManagedType.GetManagedObject(op); + + if (mt is ClassBase b) + { + var _type = b.type; + t = _type.Valid ? _type.Value : null; + } + else if (mt is CLRObject ob) + { + var inst = ob.inst; + if (inst is Type ty) + { + t = ty; + } + } + else + { + t = Converter.GetTypeByAlias(op); + } + + if (t == null) + { + types = null; + break; + } + types[i] = t; + } + newArgs.Dispose(); + return types; + } + + /// + /// Managed exports of the Python C API. Where appropriate, we do + /// some optimization to avoid managed <--> unmanaged transitions + /// (mostly for heavily used methods). + /// + [Obsolete("Use NewReference or PyObject constructor instead")] + internal static unsafe void XIncref(BorrowedReference op) + { + Py_IncRef(op); + return; + } + + internal static unsafe void XDecref(StolenReference op) + { +#if DEBUG + Debug.Assert(op == null || Refcount(new BorrowedReference(op.Pointer)) > 0); + Debug.Assert(_isInitialized || Py_IsInitialized() != 0 || _Py_IsFinalizing() != false); +#endif + if (op == null) return; + Py_DecRef(op.AnalyzerWorkaround()); + return; + } + + [Pure] + internal static unsafe nint Refcount(BorrowedReference op) + { + if (op == null) + { + return 0; + } + var p = (nint*)(op.DangerousGetAddress() + ABI.RefCountOffset); + return *p; + } + [Pure] + internal static int Refcount32(BorrowedReference op) => checked((int)Refcount(op)); + + internal static void TryUsingDll(Action op) => + TryUsingDll(() => { op(); return 0; }); + + /// + /// Call specified function, and handle PythonDLL-related failures. + /// + internal static T TryUsingDll(Func op) + { + try + { + return op(); + } + catch (TypeInitializationException loadFailure) + { + var delegatesLoadFailure = loadFailure; + // failure to load Delegates type might have been the cause + // of failure to load some higher-level type + while (delegatesLoadFailure.InnerException is TypeInitializationException nested) + { + delegatesLoadFailure = nested; + } + + if (delegatesLoadFailure.InnerException is BadPythonDllException badDll) + { + throw badDll; + } + + throw; + } + } + + /// + /// Export of Macro Py_XIncRef. Use XIncref instead. + /// Limit this function usage for Testing and Py_Debug builds + /// + /// PyObject Ptr + + internal static void Py_IncRef(BorrowedReference ob) => Delegates.Py_IncRef(ob); + + /// + /// Export of Macro Py_XDecRef. Use XDecref instead. + /// Limit this function usage for Testing and Py_Debug builds + /// + /// PyObject Ptr + + internal static void Py_DecRef(StolenReference ob) => Delegates.Py_DecRef(ob); + + + internal static void Py_Initialize() => Delegates.Py_Initialize(); + + + internal static void Py_InitializeEx(int initsigs) => Delegates.Py_InitializeEx(initsigs); + + + internal static int Py_IsInitialized() => Delegates.Py_IsInitialized(); + + + internal static void Py_Finalize() => Delegates.Py_Finalize(); + + + internal static PyThreadState* Py_NewInterpreter() => Delegates.Py_NewInterpreter(); + + + internal static void Py_EndInterpreter(PyThreadState* threadState) => Delegates.Py_EndInterpreter(threadState); + + + internal static PyThreadState* PyThreadState_New(PyInterpreterState* istate) => Delegates.PyThreadState_New(istate); + + + internal static PyThreadState* PyThreadState_Get() => Delegates.PyThreadState_Get(); + + + internal static PyThreadState* _PyThreadState_UncheckedGet() => Delegates._PyThreadState_UncheckedGet(); + + + internal static int PyGILState_Check() => Delegates.PyGILState_Check(); + internal static PyGILState PyGILState_Ensure() => Delegates.PyGILState_Ensure(); + + + internal static void PyGILState_Release(PyGILState gs) => Delegates.PyGILState_Release(gs); + + + + internal static PyThreadState* PyGILState_GetThisThreadState() => Delegates.PyGILState_GetThisThreadState(); + + + internal static void PyEval_InitThreads() => Delegates.PyEval_InitThreads(); + + + internal static int PyEval_ThreadsInitialized() => Delegates.PyEval_ThreadsInitialized(); + + + internal static void PyEval_AcquireLock() => Delegates.PyEval_AcquireLock(); + + + internal static void PyEval_ReleaseLock() => Delegates.PyEval_ReleaseLock(); + + + internal static void PyEval_AcquireThread(PyThreadState* tstate) => Delegates.PyEval_AcquireThread(tstate); + + + internal static void PyEval_ReleaseThread(PyThreadState* tstate) => Delegates.PyEval_ReleaseThread(tstate); + + + internal static PyThreadState* PyEval_SaveThread() => Delegates.PyEval_SaveThread(); + + + internal static void PyEval_RestoreThread(PyThreadState* tstate) => Delegates.PyEval_RestoreThread(tstate); + + + internal static BorrowedReference PyEval_GetBuiltins() => Delegates.PyEval_GetBuiltins(); + + + internal static BorrowedReference PyEval_GetGlobals() => Delegates.PyEval_GetGlobals(); + + + internal static BorrowedReference PyEval_GetLocals() => Delegates.PyEval_GetLocals(); + + + internal static IntPtr Py_GetProgramName() => Delegates.Py_GetProgramName(); + + + internal static void Py_SetProgramName(IntPtr name) => Delegates.Py_SetProgramName(name); + + + internal static IntPtr Py_GetPythonHome() => Delegates.Py_GetPythonHome(); + + + internal static void Py_SetPythonHome(IntPtr home) => Delegates.Py_SetPythonHome(home); + + + internal static IntPtr Py_GetPath() => Delegates.Py_GetPath(); + + + internal static void Py_SetPath(IntPtr home) => Delegates.Py_SetPath(home); + + + internal static IntPtr Py_GetVersion() => Delegates.Py_GetVersion(); + + + internal static IntPtr Py_GetPlatform() => Delegates.Py_GetPlatform(); + + + internal static IntPtr Py_GetCopyright() => Delegates.Py_GetCopyright(); + + + internal static IntPtr Py_GetCompiler() => Delegates.Py_GetCompiler(); + + + internal static IntPtr Py_GetBuildInfo() => Delegates.Py_GetBuildInfo(); + + const PyCompilerFlags Utf8String = PyCompilerFlags.IGNORE_COOKIE | PyCompilerFlags.SOURCE_IS_UTF8; + + internal static int PyRun_SimpleString(string code) + { + using var codePtr = new StrPtr(code); + return Delegates.PyRun_SimpleStringFlags(codePtr, Utf8String); + } + + internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals) + { + using var codePtr = new StrPtr(code); + return Delegates.PyRun_StringFlags(codePtr, st, globals, locals, Utf8String); + } + + internal static NewReference PyEval_EvalCode(BorrowedReference co, BorrowedReference globals, BorrowedReference locals) => Delegates.PyEval_EvalCode(co, globals, locals); + + /// + /// Return value: New reference. + /// This is a simplified interface to Py_CompileStringFlags() below, leaving flags set to NULL. + /// + internal static NewReference Py_CompileString(string str, string file, int start) + { + using var strPtr = new StrPtr(str); + + using var fileObj = new PyString(file); + return Delegates.Py_CompileStringObject(strPtr, fileObj, start, Utf8String, -1); + } + + internal static NewReference PyImport_ExecCodeModule(string name, BorrowedReference code) + { + using var namePtr = new StrPtr(name); + return Delegates.PyImport_ExecCodeModule(namePtr, code); + } + + //==================================================================== + // Python abstract object API + //==================================================================== + + /// + /// A macro-like method to get the type of a Python object. This is + /// designed to be lean and mean in IL & avoid managed <-> unmanaged + /// transitions. Note that this does not incref the type object. + /// + internal static unsafe BorrowedReference PyObject_TYPE(BorrowedReference op) + { + IntPtr address = op.DangerousGetAddressOrNull(); + if (address == IntPtr.Zero) + { + return BorrowedReference.Null; + } + Debug.Assert(TypeOffset.ob_type > 0); + BorrowedReference* typePtr = (BorrowedReference*)(address + TypeOffset.ob_type); + return *typePtr; + } + internal static NewReference PyObject_Type(BorrowedReference o) + => Delegates.PyObject_Type(o); + + internal static string PyObject_GetTypeName(BorrowedReference op) + { + Debug.Assert(TypeOffset.tp_name > 0); + Debug.Assert(op != null); + BorrowedReference pyType = PyObject_TYPE(op); + IntPtr ppName = Util.ReadIntPtr(pyType, TypeOffset.tp_name); + return Marshal.PtrToStringAnsi(ppName); + } + + /// + /// Test whether the Python object is an iterable. + /// + internal static bool PyObject_IsIterable(BorrowedReference ob) + { + var ob_type = PyObject_TYPE(ob); + return Util.ReadIntPtr(ob_type, TypeOffset.tp_iter) != IntPtr.Zero; + } + + internal static int PyObject_HasAttrString(BorrowedReference pointer, string name) + { + using var namePtr = new StrPtr(name); + return Delegates.PyObject_HasAttrString(pointer, namePtr); + } + + internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, string name) + { + using var namePtr = new StrPtr(name); + return Delegates.PyObject_GetAttrString(pointer, namePtr); + } + + internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, StrPtr name) + => Delegates.PyObject_GetAttrString(pointer, name); + + + internal static int PyObject_DelAttr(BorrowedReference @object, BorrowedReference name) => Delegates.PyObject_SetAttr(@object, name, null); + internal static int PyObject_DelAttrString(BorrowedReference @object, string name) + { + using var namePtr = new StrPtr(name); + return Delegates.PyObject_SetAttrString(@object, namePtr, null); + } + internal static int PyObject_SetAttrString(BorrowedReference @object, string name, BorrowedReference value) + { + using var namePtr = new StrPtr(name); + return Delegates.PyObject_SetAttrString(@object, namePtr, value); + } + + internal static int PyObject_HasAttr(BorrowedReference pointer, BorrowedReference name) => Delegates.PyObject_HasAttr(pointer, name); + + + internal static NewReference PyObject_GetAttr(BorrowedReference pointer, IntPtr name) + => Delegates.PyObject_GetAttr(pointer, new BorrowedReference(name)); + internal static NewReference PyObject_GetAttr(BorrowedReference o, BorrowedReference name) => Delegates.PyObject_GetAttr(o, name); + + + internal static int PyObject_SetAttr(BorrowedReference o, BorrowedReference name, BorrowedReference value) => Delegates.PyObject_SetAttr(o, name, value); + + + internal static NewReference PyObject_GetItem(BorrowedReference o, BorrowedReference key) => Delegates.PyObject_GetItem(o, key); + + + internal static int PyObject_SetItem(BorrowedReference o, BorrowedReference key, BorrowedReference value) => Delegates.PyObject_SetItem(o, key, value); + + + internal static int PyObject_DelItem(BorrowedReference o, BorrowedReference key) => Delegates.PyObject_DelItem(o, key); + + + internal static NewReference PyObject_GetIter(BorrowedReference op) => Delegates.PyObject_GetIter(op); + + + internal static NewReference PyObject_Call(BorrowedReference pointer, BorrowedReference args, BorrowedReference kw) => Delegates.PyObject_Call(pointer, args, kw); + + internal static NewReference PyObject_CallObject(BorrowedReference callable, BorrowedReference args) => Delegates.PyObject_CallObject(callable, args); + internal static IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args) + => Delegates.PyObject_CallObject(new BorrowedReference(pointer), new BorrowedReference(args)) + .DangerousMoveToPointerOrNull(); + + + internal static int PyObject_RichCompareBool(BorrowedReference value1, BorrowedReference value2, int opid) => Delegates.PyObject_RichCompareBool(value1, value2, opid); + + + internal static int PyObject_IsInstance(BorrowedReference ob, BorrowedReference type) => Delegates.PyObject_IsInstance(ob, type); + + + internal static int PyObject_IsSubclass(BorrowedReference ob, BorrowedReference type) => Delegates.PyObject_IsSubclass(ob, type); + + internal static void PyObject_ClearWeakRefs(BorrowedReference ob) => Delegates.PyObject_ClearWeakRefs(ob); + + internal static BorrowedReference PyObject_GetWeakRefList(BorrowedReference ob) + { + Debug.Assert(ob != null); + var type = PyObject_TYPE(ob); + int offset = Util.ReadInt32(type, TypeOffset.tp_weaklistoffset); + if (offset <= 0) return BorrowedReference.Null; + Debug.Assert(offset > 0); + return Util.ReadRef(ob, offset); + } + + + internal static int PyCallable_Check(BorrowedReference o) => Delegates.PyCallable_Check(o); + + + internal static int PyObject_IsTrue(IntPtr pointer) => PyObject_IsTrue(new BorrowedReference(pointer)); + internal static int PyObject_IsTrue(BorrowedReference pointer) => Delegates.PyObject_IsTrue(pointer); + + + internal static int PyObject_Not(BorrowedReference o) => Delegates.PyObject_Not(o); + + internal static nint PyObject_Size(BorrowedReference pointer) => Delegates.PyObject_Size(pointer); + + + internal static nint PyObject_Hash(BorrowedReference op) => Delegates.PyObject_Hash(op); + + + internal static NewReference PyObject_Repr(BorrowedReference pointer) + { + AssertNoErorSet(); + + return Delegates.PyObject_Repr(pointer); + } + + + internal static NewReference PyObject_Str(BorrowedReference pointer) + { + AssertNoErorSet(); + + return Delegates.PyObject_Str(pointer); + } + + [Conditional("DEBUG")] + internal static void AssertNoErorSet() + { + if (Exceptions.ErrorOccurred()) + throw new InvalidOperationException( + "Can't call with exception set", + PythonException.FetchCurrent()); + } + + + internal static NewReference PyObject_Dir(BorrowedReference pointer) => Delegates.PyObject_Dir(pointer); + + internal static void _Py_NewReference(BorrowedReference ob) + { + if (Delegates._Py_NewReference != null) + Delegates._Py_NewReference(ob); + } + + internal static bool? _Py_IsFinalizing() + { + if (Delegates._Py_IsFinalizing != null) + return Delegates._Py_IsFinalizing() != 0; + else + return null; ; + } + + //==================================================================== + // Python buffer API + //==================================================================== + + + internal static int PyObject_GetBuffer(BorrowedReference exporter, out Py_buffer view, int flags) => Delegates.PyObject_GetBuffer(exporter, out view, flags); + + + internal static void PyBuffer_Release(ref Py_buffer view) => Delegates.PyBuffer_Release(ref view); + + + internal static nint PyBuffer_SizeFromFormat(string format) + { + using var formatPtr = new StrPtr(format, Encoding.ASCII); + return Delegates.PyBuffer_SizeFromFormat(formatPtr); + } + + internal static int PyBuffer_IsContiguous(ref Py_buffer view, char order) => Delegates.PyBuffer_IsContiguous(ref view, order); + + + internal static IntPtr PyBuffer_GetPointer(ref Py_buffer view, nint[] indices) => Delegates.PyBuffer_GetPointer(ref view, indices); + + + internal static int PyBuffer_FromContiguous(ref Py_buffer view, IntPtr buf, IntPtr len, char fort) => Delegates.PyBuffer_FromContiguous(ref view, buf, len, fort); + + + internal static int PyBuffer_ToContiguous(IntPtr buf, ref Py_buffer src, IntPtr len, char order) => Delegates.PyBuffer_ToContiguous(buf, ref src, len, order); + + + internal static void PyBuffer_FillContiguousStrides(int ndims, IntPtr shape, IntPtr strides, int itemsize, char order) => Delegates.PyBuffer_FillContiguousStrides(ndims, shape, strides, itemsize, order); + + + internal static int PyBuffer_FillInfo(ref Py_buffer view, BorrowedReference exporter, IntPtr buf, IntPtr len, int _readonly, int flags) => Delegates.PyBuffer_FillInfo(ref view, exporter, buf, len, _readonly, flags); + + //==================================================================== + // Python number API + //==================================================================== + + + internal static NewReference PyNumber_Long(BorrowedReference ob) => Delegates.PyNumber_Long(ob); + + + internal static NewReference PyNumber_Float(BorrowedReference ob) => Delegates.PyNumber_Float(ob); + + + internal static bool PyNumber_Check(BorrowedReference ob) => Delegates.PyNumber_Check(ob); + + internal static bool PyInt_Check(BorrowedReference ob) + => PyObject_TypeCheck(ob, PyLongType); + + internal static bool PyInt_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyLongType); + + internal static bool PyBool_Check(BorrowedReference ob) + => PyObject_TypeCheck(ob, PyBoolType); + internal static bool PyBool_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyBoolType); + + internal static NewReference PyInt_FromInt32(int value) => PyLong_FromLongLong(value); + + internal static NewReference PyInt_FromInt64(long value) => PyLong_FromLongLong(value); + + internal static NewReference PyLong_FromLongLong(long value) => Delegates.PyLong_FromLongLong(value); + + + internal static NewReference PyLong_FromUnsignedLongLong(ulong value) => Delegates.PyLong_FromUnsignedLongLong(value); + + + internal static NewReference PyLong_FromString(string value, int radix) + { + using var valPtr = new StrPtr(value); + return Delegates.PyLong_FromString(valPtr, IntPtr.Zero, radix); + } + + + + internal static nuint PyLong_AsUnsignedSize_t(BorrowedReference value) => Delegates.PyLong_AsUnsignedSize_t(value); + + internal static nint PyLong_AsSignedSize_t(BorrowedReference value) => Delegates.PyLong_AsSignedSize_t(value); + + internal static long? PyLong_AsLongLong(BorrowedReference value) + { + long result = Delegates.PyLong_AsLongLong(value); + if (result == -1 && Exceptions.ErrorOccurred()) + { + return null; + } + return result; + } + + internal static ulong? PyLong_AsUnsignedLongLong(BorrowedReference value) + { + ulong result = Delegates.PyLong_AsUnsignedLongLong(value); + if (result == unchecked((ulong)-1) && Exceptions.ErrorOccurred()) + { + return null; + } + return result; + } + + internal static bool PyFloat_Check(BorrowedReference ob) + => PyObject_TypeCheck(ob, PyFloatType); + internal static bool PyFloat_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyFloatType); + + /// + /// Return value: New reference. + /// Create a Python integer from the pointer p. The pointer value can be retrieved from the resulting value using PyLong_AsVoidPtr(). + /// + internal static NewReference PyLong_FromVoidPtr(IntPtr p) => Delegates.PyLong_FromVoidPtr(p); + + /// + /// Convert a Python integer pylong to a C void pointer. If pylong cannot be converted, an OverflowError will be raised. This is only assured to produce a usable void pointer for values created with PyLong_FromVoidPtr(). + /// + + internal static IntPtr PyLong_AsVoidPtr(BorrowedReference ob) => Delegates.PyLong_AsVoidPtr(ob); + + + internal static NewReference PyFloat_FromDouble(double value) => Delegates.PyFloat_FromDouble(value); + + + internal static NewReference PyFloat_FromString(BorrowedReference value) => Delegates.PyFloat_FromString(value); + + + internal static double PyFloat_AsDouble(BorrowedReference ob) => Delegates.PyFloat_AsDouble(ob); + + + internal static NewReference PyNumber_Add(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Add(o1, o2); + + + internal static NewReference PyNumber_Subtract(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Subtract(o1, o2); + + + internal static NewReference PyNumber_Multiply(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Multiply(o1, o2); + + + internal static NewReference PyNumber_TrueDivide(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_TrueDivide(o1, o2); + + + internal static NewReference PyNumber_And(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_And(o1, o2); + + + internal static NewReference PyNumber_Xor(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Xor(o1, o2); + + + internal static NewReference PyNumber_Or(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Or(o1, o2); + + + internal static NewReference PyNumber_Lshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Lshift(o1, o2); + + + internal static NewReference PyNumber_Rshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Rshift(o1, o2); + + + internal static NewReference PyNumber_Power(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Power(o1, o2); + + + internal static NewReference PyNumber_Remainder(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_Remainder(o1, o2); + + + internal static NewReference PyNumber_InPlaceAdd(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceAdd(o1, o2); + + + internal static NewReference PyNumber_InPlaceSubtract(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceSubtract(o1, o2); + + + internal static NewReference PyNumber_InPlaceMultiply(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceMultiply(o1, o2); + + + internal static NewReference PyNumber_InPlaceTrueDivide(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceTrueDivide(o1, o2); + + + internal static NewReference PyNumber_InPlaceAnd(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceAnd(o1, o2); + + + internal static NewReference PyNumber_InPlaceXor(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceXor(o1, o2); + + + internal static NewReference PyNumber_InPlaceOr(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceOr(o1, o2); + + + internal static NewReference PyNumber_InPlaceLshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceLshift(o1, o2); + + + internal static NewReference PyNumber_InPlaceRshift(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceRshift(o1, o2); + + + internal static NewReference PyNumber_InPlacePower(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlacePower(o1, o2); + + + internal static NewReference PyNumber_InPlaceRemainder(BorrowedReference o1, BorrowedReference o2) => Delegates.PyNumber_InPlaceRemainder(o1, o2); + + + internal static NewReference PyNumber_Negative(BorrowedReference o1) => Delegates.PyNumber_Negative(o1); + + + internal static NewReference PyNumber_Positive(BorrowedReference o1) => Delegates.PyNumber_Positive(o1); + + + internal static NewReference PyNumber_Invert(BorrowedReference o1) => Delegates.PyNumber_Invert(o1); + + + //==================================================================== + // Python sequence API + //==================================================================== + + + internal static bool PySequence_Check(BorrowedReference pointer) => Delegates.PySequence_Check(pointer); + + internal static NewReference PySequence_GetItem(BorrowedReference pointer, nint index) => Delegates.PySequence_GetItem(pointer, index); + internal static int PySequence_SetItem(BorrowedReference pointer, nint index, BorrowedReference value) => Delegates.PySequence_SetItem(pointer, index, value); + + internal static int PySequence_DelItem(BorrowedReference pointer, nint index) => Delegates.PySequence_DelItem(pointer, index); + + internal static NewReference PySequence_GetSlice(BorrowedReference pointer, nint i1, nint i2) => Delegates.PySequence_GetSlice(pointer, i1, i2); + + internal static int PySequence_SetSlice(BorrowedReference pointer, nint i1, nint i2, BorrowedReference v) => Delegates.PySequence_SetSlice(pointer, i1, i2, v); + + internal static int PySequence_DelSlice(BorrowedReference pointer, nint i1, nint i2) => Delegates.PySequence_DelSlice(pointer, i1, i2); + + internal static nint PySequence_Size(BorrowedReference pointer) => Delegates.PySequence_Size(pointer); + + internal static int PySequence_Contains(BorrowedReference pointer, BorrowedReference item) => Delegates.PySequence_Contains(pointer, item); + + + internal static NewReference PySequence_Concat(BorrowedReference pointer, BorrowedReference other) => Delegates.PySequence_Concat(pointer, other); + + internal static NewReference PySequence_Repeat(BorrowedReference pointer, nint count) => Delegates.PySequence_Repeat(pointer, count); + + + internal static nint PySequence_Index(BorrowedReference pointer, BorrowedReference item) => Delegates.PySequence_Index(pointer, item); + + private static nint PySequence_Count(BorrowedReference pointer, BorrowedReference value) => Delegates.PySequence_Count(pointer, value); + + + internal static NewReference PySequence_Tuple(BorrowedReference pointer) => Delegates.PySequence_Tuple(pointer); + + + internal static NewReference PySequence_List(BorrowedReference pointer) => Delegates.PySequence_List(pointer); + + + //==================================================================== + // Python string API + //==================================================================== + internal static bool PyString_Check(BorrowedReference ob) + => PyObject_TypeCheck(ob, PyStringType); + internal static bool PyString_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyStringType); + + internal static NewReference PyString_FromString(string value) + { + int byteorder = BitConverter.IsLittleEndian ? -1 : 1; + int* byteorderPtr = &byteorder; + fixed(char* ptr = value) + return Delegates.PyUnicode_DecodeUTF16( + (IntPtr)ptr, + value.Length * sizeof(Char), + IntPtr.Zero, + (IntPtr)byteorderPtr + ); + } + + + internal static NewReference EmptyPyBytes() + { + byte* bytes = stackalloc byte[1]; + bytes[0] = 0; + return Delegates.PyBytes_FromString((IntPtr)bytes); + } + + internal static NewReference PyByteArray_FromStringAndSize(IntPtr strPtr, nint len) => Delegates.PyByteArray_FromStringAndSize(strPtr, len); + internal static NewReference PyByteArray_FromStringAndSize(string s) + { + using var ptr = new StrPtr(s); + return PyByteArray_FromStringAndSize(ptr.RawPointer, checked((nint)ptr.ByteCount)); + } + + internal static IntPtr PyBytes_AsString(BorrowedReference ob) + { + Debug.Assert(ob != null); + return Delegates.PyBytes_AsString(ob); + } + + internal static nint PyBytes_Size(BorrowedReference op) => Delegates.PyBytes_Size(op); + + internal static IntPtr PyUnicode_AsUTF8(BorrowedReference unicode) => Delegates.PyUnicode_AsUTF8(unicode); + + /// Length in code points + internal static nint PyUnicode_GetLength(BorrowedReference ob) => Delegates.PyUnicode_GetLength(ob); + + + internal static NewReference PyUnicode_AsUTF16String(BorrowedReference ob) => Delegates.PyUnicode_AsUTF16String(ob); + + internal static int PyUnicode_ReadChar(BorrowedReference ob, nint index) => Delegates.PyUnicode_ReadChar(ob, index); + + + + internal static NewReference PyUnicode_FromOrdinal(int c) => Delegates.PyUnicode_FromOrdinal(c); + + internal static NewReference PyUnicode_InternFromString(string s) + { + using var ptr = new StrPtr(s); + return Delegates.PyUnicode_InternFromString(ptr); + } + + internal static int PyUnicode_Compare(BorrowedReference left, BorrowedReference right) => Delegates.PyUnicode_Compare(left, right); + + internal static string ToString(BorrowedReference op) + { + using var strval = PyObject_Str(op); + return GetManagedStringFromUnicodeObject(strval.BorrowOrThrow())!; + } + + /// + /// Function to access the internal PyUnicode/PyString object and + /// convert it to a managed string with the correct encoding. + /// + /// + /// We can't easily do this through through the CustomMarshaler's on + /// the returns because will have access to the IntPtr but not size. + /// + /// For PyUnicodeType, we can't convert with Marshal.PtrToStringUni + /// since it only works for UCS2. + /// + /// PyStringType or PyUnicodeType object to convert + /// Managed String + internal static string? GetManagedString(in BorrowedReference op) + { + var type = PyObject_TYPE(op); + + if (type == PyUnicodeType) + { + return GetManagedStringFromUnicodeObject(op); + } + + return null; + } + + static string GetManagedStringFromUnicodeObject(BorrowedReference op) + { +#if DEBUG + var type = PyObject_TYPE(op); + Debug.Assert(type == PyUnicodeType); +#endif + using var bytes = PyUnicode_AsUTF16String(op); + if (bytes.IsNull()) + { + throw PythonException.ThrowLastAsClrException(); + } + int bytesLength = checked((int)PyBytes_Size(bytes.Borrow())); + char* codePoints = (char*)PyBytes_AsString(bytes.Borrow()); + return new string(codePoints, + startIndex: 1, // skip BOM + length: bytesLength / 2 - 1); // utf16 - BOM + } + + + //==================================================================== + // Python dictionary API + //==================================================================== + + internal static bool PyDict_Check(BorrowedReference ob) + { + return PyObject_TYPE(ob) == PyDictType; + } + + + internal static NewReference PyDict_New() => Delegates.PyDict_New(); + + /// + /// Return NULL if the key is not present, but without setting an exception. + /// + internal static BorrowedReference PyDict_GetItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItem(pointer, key); + + internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key) + { + using var keyStr = new StrPtr(key); + return Delegates.PyDict_GetItemString(pointer, keyStr); + } + + internal static BorrowedReference PyDict_GetItemWithError(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_GetItemWithError(pointer, key); + + /// + /// Return 0 on success or -1 on failure. + /// + internal static int PyDict_SetItem(BorrowedReference dict, BorrowedReference key, BorrowedReference value) => Delegates.PyDict_SetItem(dict, key, value); + + /// + /// Return 0 on success or -1 on failure. + /// + internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value) + { + using var keyPtr = new StrPtr(key); + return Delegates.PyDict_SetItemString(dict, keyPtr, value); + } + + internal static int PyDict_DelItem(BorrowedReference pointer, BorrowedReference key) => Delegates.PyDict_DelItem(pointer, key); + + + internal static int PyDict_DelItemString(BorrowedReference pointer, string key) + { + using var keyPtr = new StrPtr(key); + return Delegates.PyDict_DelItemString(pointer, keyPtr); + } + + internal static int PyMapping_HasKey(BorrowedReference pointer, BorrowedReference key) => Delegates.PyMapping_HasKey(pointer, key); + + + internal static NewReference PyDict_Keys(BorrowedReference pointer) => Delegates.PyDict_Keys(pointer); + + internal static NewReference PyDict_Values(BorrowedReference pointer) => Delegates.PyDict_Values(pointer); + + internal static NewReference PyDict_Items(BorrowedReference pointer) => Delegates.PyDict_Items(pointer); + + + internal static NewReference PyDict_Copy(BorrowedReference pointer) => Delegates.PyDict_Copy(pointer); + + + internal static int PyDict_Update(BorrowedReference pointer, BorrowedReference other) => Delegates.PyDict_Update(pointer, other); + + + internal static void PyDict_Clear(BorrowedReference pointer) => Delegates.PyDict_Clear(pointer); + + internal static nint PyDict_Size(BorrowedReference pointer) => Delegates.PyDict_Size(pointer); + + + internal static NewReference PySet_New(BorrowedReference iterable) => Delegates.PySet_New(iterable); + + + internal static int PySet_Add(BorrowedReference set, BorrowedReference key) => Delegates.PySet_Add(set, key); + + /// + /// Return 1 if found, 0 if not found, and -1 if an error is encountered. + /// + + internal static int PySet_Contains(BorrowedReference anyset, BorrowedReference key) => Delegates.PySet_Contains(anyset, key); + + //==================================================================== + // Python list API + //==================================================================== + + internal static bool PyList_Check(BorrowedReference ob) + { + return PyObject_TYPE(ob) == PyListType; + } + + internal static NewReference PyList_New(nint size) => Delegates.PyList_New(size); + + internal static BorrowedReference PyList_GetItem(BorrowedReference pointer, nint index) => Delegates.PyList_GetItem(pointer, index); + + internal static int PyList_SetItem(BorrowedReference pointer, nint index, StolenReference value) => Delegates.PyList_SetItem(pointer, index, value); + + internal static int PyList_Insert(BorrowedReference pointer, nint index, BorrowedReference value) => Delegates.PyList_Insert(pointer, index, value); + + + internal static int PyList_Append(BorrowedReference pointer, BorrowedReference value) => Delegates.PyList_Append(pointer, value); + + + internal static int PyList_Reverse(BorrowedReference pointer) => Delegates.PyList_Reverse(pointer); + + + internal static int PyList_Sort(BorrowedReference pointer) => Delegates.PyList_Sort(pointer); + + private static NewReference PyList_GetSlice(BorrowedReference pointer, nint start, nint end) => Delegates.PyList_GetSlice(pointer, start, end); + + private static int PyList_SetSlice(BorrowedReference pointer, nint start, nint end, BorrowedReference value) => Delegates.PyList_SetSlice(pointer, start, end, value); + + + internal static nint PyList_Size(BorrowedReference pointer) => Delegates.PyList_Size(pointer); + + //==================================================================== + // Python tuple API + //==================================================================== + + internal static bool PyTuple_Check(BorrowedReference ob) + { + return PyObject_TYPE(ob) == PyTupleType; + } + internal static NewReference PyTuple_New(nint size) => Delegates.PyTuple_New(size); + + internal static BorrowedReference PyTuple_GetItem(BorrowedReference pointer, nint index) => Delegates.PyTuple_GetItem(pointer, index); + + internal static int PyTuple_SetItem(BorrowedReference pointer, nint index, BorrowedReference value) + { + var newRef = new NewReference(value); + return PyTuple_SetItem(pointer, index, newRef.Steal()); + } + + internal static int PyTuple_SetItem(BorrowedReference pointer, nint index, StolenReference value) => Delegates.PyTuple_SetItem(pointer, index, value); + + internal static NewReference PyTuple_GetSlice(BorrowedReference pointer, nint start, nint end) => Delegates.PyTuple_GetSlice(pointer, start, end); + + internal static nint PyTuple_Size(BorrowedReference pointer) => Delegates.PyTuple_Size(pointer); + + + //==================================================================== + // Python iterator API + //==================================================================== + internal static bool PyIter_Check(BorrowedReference ob) + { + if (Delegates.PyIter_Check != null) + return Delegates.PyIter_Check(ob) != 0; + var ob_type = PyObject_TYPE(ob); + var tp_iternext = (NativeFunc*)Util.ReadIntPtr(ob_type, TypeOffset.tp_iternext); + return tp_iternext != (NativeFunc*)0 && tp_iternext != _PyObject_NextNotImplemented; + } + internal static NewReference PyIter_Next(BorrowedReference pointer) => Delegates.PyIter_Next(pointer); + + + //==================================================================== + // Python module API + //==================================================================== + + + internal static NewReference PyModule_New(string name) + { + using var namePtr = new StrPtr(name); + return Delegates.PyModule_New(namePtr); + } + + internal static BorrowedReference PyModule_GetDict(BorrowedReference module) => Delegates.PyModule_GetDict(module); + + internal static NewReference PyImport_Import(BorrowedReference name) => Delegates.PyImport_Import(name); + + /// The module to add the object to. + /// The key that will refer to the object. + /// The object to add to the module. + /// Return -1 on error, 0 on success. + internal static int PyModule_AddObject(BorrowedReference module, string name, StolenReference value) + { + using var namePtr = new StrPtr(name); + IntPtr valueAddr = value.DangerousGetAddressOrNull(); + int res = Delegates.PyModule_AddObject(module, namePtr, valueAddr); + // We can't just exit here because the reference is stolen only on success. + if (res != 0) + { + XDecref(StolenReference.TakeNullable(ref valueAddr)); + } + return res; + + } + + /// + /// Return value: New reference. + /// + + internal static NewReference PyImport_ImportModule(string name) + { + using var namePtr = new StrPtr(name); + return Delegates.PyImport_ImportModule(namePtr); + } + + internal static NewReference PyImport_ReloadModule(BorrowedReference module) => Delegates.PyImport_ReloadModule(module); + + + internal static BorrowedReference PyImport_AddModule(string name) + { + using var namePtr = new StrPtr(name); + return Delegates.PyImport_AddModule(namePtr); + } + + internal static BorrowedReference PyImport_GetModuleDict() => Delegates.PyImport_GetModuleDict(); + + + internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath) + { + var marshaler = StrArrayMarshaler.GetInstance(null); + var argvPtr = marshaler.MarshalManagedToNative(argv); + try + { + Delegates.PySys_SetArgvEx(argc, argvPtr, updatepath); + } + finally + { + marshaler.CleanUpNativeData(argvPtr); + } + } + + /// + /// Return value: Borrowed reference. + /// Return the object name from the sys module or NULL if it does not exist, without setting an exception. + /// + + internal static BorrowedReference PySys_GetObject(string name) + { + using var namePtr = new StrPtr(name); + return Delegates.PySys_GetObject(namePtr); + } + + internal static int PySys_SetObject(string name, BorrowedReference ob) + { + using var namePtr = new StrPtr(name); + return Delegates.PySys_SetObject(namePtr, ob); + } + + + //==================================================================== + // Python type object API + //==================================================================== + internal static bool PyType_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyTypeType); + + + internal static void PyType_Modified(BorrowedReference type) => Delegates.PyType_Modified(type); + internal static bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2) + { + Debug.Assert(t1 != null && t2 != null); + return Delegates.PyType_IsSubtype(t1, t2); + } + + internal static bool PyObject_TypeCheckExact(BorrowedReference ob, BorrowedReference tp) + => PyObject_TYPE(ob) == tp; + internal static bool PyObject_TypeCheck(BorrowedReference ob, BorrowedReference tp) + { + BorrowedReference t = PyObject_TYPE(ob); + return (t == tp) || PyType_IsSubtype(t, tp); + } + + internal static bool PyType_IsSameAsOrSubtype(BorrowedReference type, BorrowedReference ofType) + { + return (type == ofType) || PyType_IsSubtype(type, ofType); + } + + + internal static NewReference PyType_GenericNew(BorrowedReference type, BorrowedReference args, BorrowedReference kw) => Delegates.PyType_GenericNew(type, args, kw); + + internal static NewReference PyType_GenericAlloc(BorrowedReference type, nint n) => Delegates.PyType_GenericAlloc(type, n); + + internal static IntPtr PyType_GetSlot(BorrowedReference type, TypeSlotID slot) => Delegates.PyType_GetSlot(type, slot); + internal static NewReference PyType_FromSpecWithBases(in NativeTypeSpec spec, BorrowedReference bases) => Delegates.PyType_FromSpecWithBases(in spec, bases); + + /// + /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type�s base class. Return 0 on success, or return -1 and sets an exception on error. + /// + + internal static int PyType_Ready(BorrowedReference type) => Delegates.PyType_Ready(type); + + + internal static BorrowedReference _PyType_Lookup(BorrowedReference type, BorrowedReference name) => Delegates._PyType_Lookup(type, name); + + + internal static NewReference PyObject_GenericGetAttr(BorrowedReference obj, BorrowedReference name) => Delegates.PyObject_GenericGetAttr(obj, name); + + + internal static int PyObject_GenericSetAttr(BorrowedReference obj, BorrowedReference name, BorrowedReference value) => Delegates.PyObject_GenericSetAttr(obj, name, value); + + internal static NewReference PyObject_GenericGetDict(BorrowedReference o) => PyObject_GenericGetDict(o, IntPtr.Zero); + internal static NewReference PyObject_GenericGetDict(BorrowedReference o, IntPtr context) => Delegates.PyObject_GenericGetDict(o, context); + + internal static void PyObject_GC_Del(StolenReference ob) => Delegates.PyObject_GC_Del(ob); + + + internal static bool PyObject_GC_IsTracked(BorrowedReference ob) + { + if (PyVersion >= new Version(3, 9)) + return Delegates.PyObject_GC_IsTracked(ob) != 0; + + throw new NotSupportedException("Requires Python 3.9"); + } + + internal static void PyObject_GC_Track(BorrowedReference ob) => Delegates.PyObject_GC_Track(ob); + + internal static void PyObject_GC_UnTrack(BorrowedReference ob) => Delegates.PyObject_GC_UnTrack(ob); + + internal static void _PyObject_Dump(BorrowedReference ob) => Delegates._PyObject_Dump(ob); + + //==================================================================== + // Python memory API + //==================================================================== + + internal static IntPtr PyMem_Malloc(long size) + { + return PyMem_Malloc(new IntPtr(size)); + } + + + private static IntPtr PyMem_Malloc(nint size) => Delegates.PyMem_Malloc(size); + + private static IntPtr PyMem_Realloc(IntPtr ptr, nint size) => Delegates.PyMem_Realloc(ptr, size); + + + internal static void PyMem_Free(IntPtr ptr) => Delegates.PyMem_Free(ptr); + + + //==================================================================== + // Python exception API + //==================================================================== + + + internal static void PyErr_SetString(BorrowedReference ob, string message) + { + using var msgPtr = new StrPtr(message); + Delegates.PyErr_SetString(ob, msgPtr); + } + + internal static void PyErr_SetObject(BorrowedReference type, BorrowedReference exceptionObject) => Delegates.PyErr_SetObject(type, exceptionObject); + + internal static int PyErr_ExceptionMatches(BorrowedReference exception) => Delegates.PyErr_ExceptionMatches(exception); + + + internal static int PyErr_GivenExceptionMatches(BorrowedReference given, BorrowedReference typeOrTypes) => Delegates.PyErr_GivenExceptionMatches(given, typeOrTypes); + + + internal static void PyErr_NormalizeException(ref NewReference type, ref NewReference val, ref NewReference tb) => Delegates.PyErr_NormalizeException(ref type, ref val, ref tb); + + + internal static BorrowedReference PyErr_Occurred() => Delegates.PyErr_Occurred(); + + + internal static void PyErr_Fetch(out NewReference type, out NewReference val, out NewReference tb) => Delegates.PyErr_Fetch(out type, out val, out tb); + + + internal static void PyErr_Restore(StolenReference type, StolenReference val, StolenReference tb) => Delegates.PyErr_Restore(type, val, tb); + + + internal static void PyErr_Clear() => Delegates.PyErr_Clear(); + + + internal static void PyErr_Print() => Delegates.PyErr_Print(); + + + internal static NewReference PyException_GetCause(BorrowedReference ex) + => Delegates.PyException_GetCause(ex); + internal static NewReference PyException_GetTraceback(BorrowedReference ex) + => Delegates.PyException_GetTraceback(ex); + + /// + /// Set the cause associated with the exception to cause. Use NULL to clear it. There is no type check to make sure that cause is either an exception instance or None. This steals a reference to cause. + /// + internal static void PyException_SetCause(BorrowedReference ex, StolenReference cause) + => Delegates.PyException_SetCause(ex, cause); + internal static int PyException_SetTraceback(BorrowedReference ex, BorrowedReference tb) + => Delegates.PyException_SetTraceback(ex, tb); + + //==================================================================== + // Cell API + //==================================================================== + + + internal static NewReference PyCell_Get(BorrowedReference cell) => Delegates.PyCell_Get(cell); + + + internal static int PyCell_Set(BorrowedReference cell, BorrowedReference value) => Delegates.PyCell_Set(cell, value); + + internal static nint PyGC_Collect() => Delegates.PyGC_Collect(); + internal static void Py_CLEAR(BorrowedReference ob, int offset) => ReplaceReference(ob, offset, default); + internal static void Py_CLEAR(ref T? ob) + where T: PyObject + { + ob?.Dispose(); + ob = null; + } + + internal static void ReplaceReference(BorrowedReference ob, int offset, StolenReference newValue) + { + IntPtr raw = Util.ReadIntPtr(ob, offset); + Util.WriteNullableRef(ob, offset, newValue); + XDecref(StolenReference.TakeNullable(ref raw)); + } + + //==================================================================== + // Python Capsules API + //==================================================================== + + + internal static NewReference PyCapsule_New(IntPtr pointer, IntPtr name, IntPtr destructor) + => Delegates.PyCapsule_New(pointer, name, destructor); + + internal static IntPtr PyCapsule_GetPointer(BorrowedReference capsule, IntPtr name) + { + return Delegates.PyCapsule_GetPointer(capsule, name); + } + + internal static int PyCapsule_SetPointer(BorrowedReference capsule, IntPtr pointer) => Delegates.PyCapsule_SetPointer(capsule, pointer); + + //==================================================================== + // Miscellaneous + //==================================================================== + + + internal static int PyThreadState_SetAsyncExcLLP64(uint id, BorrowedReference exc) => Delegates.PyThreadState_SetAsyncExcLLP64(id, exc); + + internal static int PyThreadState_SetAsyncExcLP64(ulong id, BorrowedReference exc) => Delegates.PyThreadState_SetAsyncExcLP64(id, exc); + + + internal static void SetNoSiteFlag() + { + TryUsingDll(() => + { + *Delegates.Py_NoSiteFlag = 1; + return *Delegates.Py_NoSiteFlag; + }); + } + } + + internal class BadPythonDllException : MissingMethodException + { + public BadPythonDllException(string message, Exception innerException) + : base(message, innerException) { } + } +} diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 559d5148e..9abfad23d 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -1,922 +1,1851 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Runtime.InteropServices; -using System.Diagnostics; -using Python.Runtime.Native; -using Python.Runtime.StateSerialization; - - -namespace Python.Runtime -{ - - /// - /// The TypeManager class is responsible for building binary-compatible - /// Python type objects that are implemented in managed code. - /// - internal class TypeManager - { - internal static IntPtr subtype_traverse; - internal static IntPtr subtype_clear; -#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. - /// initialized in rather than in constructor - internal static IPythonBaseTypeProvider pythonBaseTypeProvider; -#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. - - - private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; - private static readonly Dictionary cache = new(); - - static readonly Dictionary _slotsHolders = new(PythonReferenceComparer.Instance); - - // Slots which must be set - private static readonly string[] _requiredSlots = new string[] - { - "tp_traverse", - "tp_clear", - }; - - internal static void Initialize() - { - Debug.Assert(cache.Count == 0, "Cache should be empty", - "Some errors may occurred on last shutdown"); - using (var plainType = SlotHelper.CreateObjectType()) - { - subtype_traverse = Util.ReadIntPtr(plainType.Borrow(), TypeOffset.tp_traverse); - subtype_clear = Util.ReadIntPtr(plainType.Borrow(), TypeOffset.tp_clear); - } - pythonBaseTypeProvider = PythonEngine.InteropConfiguration.pythonBaseTypeProviders; - } - - internal static void RemoveTypes() - { - if (Runtime.HostedInPython) - { - foreach (var holder in _slotsHolders) - { - // If refcount > 1, it needs to reset the managed slot, - // otherwise it can dealloc without any trick. - if (holder.Key.Refcount > 1) - { - holder.Value.ResetSlots(); - } - } - } - - foreach (var type in cache.Values) - { - type.Dispose(); - } - cache.Clear(); - _slotsHolders.Clear(); - } - - internal static TypeManagerState SaveRuntimeData() - => new() - { - Cache = cache, - }; - - internal static void RestoreRuntimeData(TypeManagerState storage) - { - Debug.Assert(cache == null || cache.Count == 0); - var typeCache = storage.Cache; - foreach (var entry in typeCache) - { - var type = entry.Key.Value; - cache![type] = entry.Value; - SlotsHolder holder = CreateSlotsHolder(entry.Value); - InitializeSlots(entry.Value, type, holder); - Runtime.PyType_Modified(entry.Value); - } - } - - internal static PyType GetType(Type type) - { - // Note that these types are cached with a refcount of 1, so they - // effectively exist until the CPython runtime is finalized. - if (!cache.TryGetValue(type, out var pyType)) - { - pyType = CreateType(type); - cache[type] = pyType; - } - return pyType; - } - /// - /// Given a managed Type derived from ExtensionType, get the handle to - /// a Python type object that delegates its implementation to the Type - /// object. These Python type instances are used to implement internal - /// descriptor and utility types like ModuleObject, PropertyObject, etc. - /// - internal static BorrowedReference GetTypeReference(Type type) => GetType(type).Reference; - - /// - /// The following CreateType implementations do the necessary work to - /// create Python types to represent managed extension types, reflected - /// types, subclasses of reflected types and the managed metatype. The - /// dance is slightly different for each kind of type due to different - /// behavior needed and the desire to have the existing Python runtime - /// do as much of the allocation and initialization work as possible. - /// - internal static unsafe PyType CreateType(Type impl) - { - // TODO: use PyType(TypeSpec) constructor - PyType type = AllocateTypeObject(impl.Name, metatype: Runtime.PyCLRMetaType); - - BorrowedReference base_ = impl == typeof(CLRModule) - ? Runtime.PyModuleType - : Runtime.PyBaseObjectType; - - type.BaseReference = base_; - - int newFieldOffset = InheritOrAllocateStandardFields(type, base_); - - int tp_clr_inst_offset = newFieldOffset; - newFieldOffset += IntPtr.Size; - - int ob_size = newFieldOffset; - // Set tp_basicsize to the size of our managed instance objects. - Util.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, tp_clr_inst_offset); - Util.WriteIntPtr(type, TypeOffset.tp_new, (IntPtr)Runtime.Delegates.PyType_GenericNew); - - SlotsHolder slotsHolder = CreateSlotsHolder(type); - InitializeSlots(type, impl, slotsHolder); - - type.Flags = TypeFlags.Default | TypeFlags.HasClrInstance | - TypeFlags.HeapType | TypeFlags.HaveGC; - - if (Runtime.PyType_Ready(type) != 0) - { - throw PythonException.ThrowLastAsClrException(); - } - - - using (var dict = Runtime.PyObject_GenericGetDict(type.Reference)) - using (var mod = Runtime.PyString_FromString("CLR")) - { - Runtime.PyDict_SetItem(dict.Borrow(), PyIdentifier.__module__, mod.Borrow()); - } - - // The type has been modified after PyType_Ready has been called - // Refresh the type - Runtime.PyType_Modified(type.Reference); - return type; - } - - - internal static void InitializeClassCore(Type clrType, PyType pyType, ClassBase impl) - { - if (pyType.BaseReference != null) - { - return; - } - - // Hide the gchandle of the implementation in a magic type slot. - GCHandle gc = GCHandle.Alloc(impl); - ManagedType.InitGCHandle(pyType, Runtime.CLRMetaType, gc); - - using var baseTuple = GetBaseTypeTuple(clrType); - - InitializeBases(pyType, baseTuple); - // core fields must be initialized in partially constructed classes, - // otherwise it would be impossible to manipulate GCHandle and check type size - InitializeCoreFields(pyType); - } - - internal static string GetPythonTypeName(Type clrType) - { - var result = new System.Text.StringBuilder(); - GetPythonTypeName(clrType, target: result); - return result.ToString(); - } - - static void GetPythonTypeName(Type clrType, System.Text.StringBuilder target) - { - if (clrType.IsGenericType) - { - string fullName = clrType.GetGenericTypeDefinition().FullName; - int argCountIndex = fullName.LastIndexOf('`'); - if (argCountIndex >= 0) - { - string nonGenericFullName = fullName.Substring(0, argCountIndex); - string nonGenericName = CleanupFullName(nonGenericFullName); - target.Append(nonGenericName); - - var arguments = clrType.GetGenericArguments(); - target.Append('['); - for (int argIndex = 0; argIndex < arguments.Length; argIndex++) - { - if (argIndex != 0) - { - target.Append(','); - } - - GetPythonTypeName(arguments[argIndex], target); - } - - target.Append(']'); - - int nestedStart = fullName.IndexOf('+'); - while (nestedStart >= 0) - { - target.Append('.'); - int nextNested = fullName.IndexOf('+', nestedStart + 1); - if (nextNested < 0) - { - target.Append(fullName.Substring(nestedStart + 1)); - } - else - { - target.Append(fullName.Substring(nestedStart + 1, length: nextNested - nestedStart - 1)); - } - nestedStart = nextNested; - } - - return; - } - } - - string name = CleanupFullName(clrType.FullName); - target.Append(name); - } - - static string CleanupFullName(string fullTypeName) - { - // Cleanup the type name to get rid of funny nested type names. - string name = "clr." + fullTypeName; - int i = name.LastIndexOf('+'); - if (i > -1) - { - name = name.Substring(i + 1); - } - - i = name.LastIndexOf('.'); - if (i > -1) - { - name = name.Substring(i + 1); - } - - return name; - } - - static BorrowedReference InitializeBases(PyType pyType, PyTuple baseTuple) - { - Debug.Assert(baseTuple.Length() > 0); - var primaryBase = baseTuple[0].Reference; - pyType.BaseReference = primaryBase; - - if (baseTuple.Length() > 1) - { - Util.WriteIntPtr(pyType, TypeOffset.tp_bases, baseTuple.NewReferenceOrNull().DangerousMoveToPointer()); - } - return primaryBase; - } - - static void InitializeCoreFields(PyType type) - { - int newFieldOffset = InheritOrAllocateStandardFields(type); - - if (ManagedType.IsManagedType(type.BaseReference)) - { - int baseClrInstOffset = Util.ReadInt32(type.BaseReference, ManagedType.Offsets.tp_clr_inst_offset); - Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, baseClrInstOffset); - } - else - { - Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, newFieldOffset); - newFieldOffset += IntPtr.Size; - } - - int ob_size = newFieldOffset; - - Util.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - Util.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); - } - - internal static void InitializeClass(PyType type, ClassBase impl, Type clrType) - { - // we want to do this after the slot stuff above in case the class itself implements a slot method - SlotsHolder slotsHolder = CreateSlotsHolder(type); - InitializeSlots(type, impl.GetType(), slotsHolder); - - impl.InitializeSlots(type, slotsHolder); - - OperatorMethod.FixupSlots(type, clrType); - // Leverage followup initialization from the Python runtime. Note - // that the type of the new type must PyType_Type at the time we - // call this, else PyType_Ready will skip some slot initialization. - - if (!type.IsReady && Runtime.PyType_Ready(type) != 0) - { - throw PythonException.ThrowLastAsClrException(); - } - - var dict = Util.ReadRef(type, TypeOffset.tp_dict); - string mn = clrType.Namespace ?? ""; - using (var mod = Runtime.PyString_FromString(mn)) - Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod.Borrow()); - - Runtime.PyType_Modified(type.Reference); - - //DebugUtil.DumpType(type); - } - - static int InheritOrAllocateStandardFields(BorrowedReference type) - { - var @base = Util.ReadRef(type, TypeOffset.tp_base); - return InheritOrAllocateStandardFields(type, @base); - } - static int InheritOrAllocateStandardFields(BorrowedReference typeRef, BorrowedReference @base) - { - IntPtr baseAddress = @base.DangerousGetAddress(); - IntPtr type = typeRef.DangerousGetAddress(); - int baseSize = Util.ReadInt32(@base, TypeOffset.tp_basicsize); - int newFieldOffset = baseSize; - - void InheritOrAllocate(int typeField) - { - int value = Marshal.ReadInt32(baseAddress, typeField); - if (value == 0) - { - Marshal.WriteIntPtr(type, typeField, new IntPtr(newFieldOffset)); - newFieldOffset += IntPtr.Size; - } - else - { - Marshal.WriteIntPtr(type, typeField, new IntPtr(value)); - } - } - - InheritOrAllocate(TypeOffset.tp_dictoffset); - InheritOrAllocate(TypeOffset.tp_weaklistoffset); - - return newFieldOffset; - } - - static PyTuple GetBaseTypeTuple(Type clrType) - { - var bases = pythonBaseTypeProvider - .GetBaseTypes(clrType, new PyType[0]) - ?.ToArray(); - if (bases is null || bases.Length == 0) - { - throw new InvalidOperationException("At least one base type must be specified"); - } - var nonBases = bases.Where(@base => !@base.Flags.HasFlag(TypeFlags.BaseType)).ToList(); - if (nonBases.Count > 0) - { - throw new InvalidProgramException("The specified Python type(s) can not be inherited from: " - + string.Join(", ", nonBases)); - } - - return new PyTuple(bases); - } - - internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase py_base_type, IList interfaces, BorrowedReference dictRef) - { - // Utility to create a subtype of a managed type with the ability for the - // a python subtype able to override the managed implementation - string? name = Runtime.GetManagedString(py_name); - if (name is null) - { - Exceptions.SetError(Exceptions.ValueError, "Class name must not be None"); - return default; - } - - // the derived class can have class attributes __assembly__ and __module__ which - // control the name of the assembly and module the new type is created in. - object? assembly = null; - object? namespaceStr = null; - - using (var assemblyKey = new PyString("__assembly__")) - { - var assemblyPtr = Runtime.PyDict_GetItemWithError(dictRef, assemblyKey.Reference); - if (assemblyPtr.IsNull) - { - if (Exceptions.ErrorOccurred()) return default; - } - else if (!Converter.ToManagedValue(assemblyPtr, typeof(string), out assembly, true)) - { - return Exceptions.RaiseTypeError("Couldn't convert __assembly__ value to string"); - } - - using var namespaceKey = new PyString("__namespace__"); - var pyNamespace = Runtime.PyDict_GetItemWithError(dictRef, namespaceKey.Reference); - if (pyNamespace.IsNull) - { - if (Exceptions.ErrorOccurred()) return default; - } - else if (!Converter.ToManagedValue(pyNamespace, typeof(string), out namespaceStr, true)) - { - return Exceptions.RaiseTypeError("Couldn't convert __namespace__ value to string"); - } - } - - // create the new managed type subclassing the base managed type - return ReflectedClrType.CreateSubclass(py_base_type, interfaces, name, - ns: (string?)namespaceStr, - assembly: (string?)assembly, - dict: dictRef); - } - - internal static IntPtr WriteMethodDef(IntPtr mdef, IntPtr name, IntPtr func, PyMethodFlags flags, IntPtr doc) - { - Marshal.WriteIntPtr(mdef, name); - Marshal.WriteIntPtr(mdef, 1 * IntPtr.Size, func); - Marshal.WriteInt32(mdef, 2 * IntPtr.Size, (int)flags); - Marshal.WriteIntPtr(mdef, 3 * IntPtr.Size, doc); - return mdef + 4 * IntPtr.Size; - } - - internal static IntPtr WriteMethodDef(IntPtr mdef, string name, IntPtr func, PyMethodFlags flags = PyMethodFlags.VarArgs, - string? doc = null) - { - IntPtr namePtr = Marshal.StringToHGlobalAnsi(name); - IntPtr docPtr = doc != null ? Marshal.StringToHGlobalAnsi(doc) : IntPtr.Zero; - - return WriteMethodDef(mdef, namePtr, func, flags, docPtr); - } - - internal static IntPtr WriteMethodDefSentinel(IntPtr mdef) - { - return WriteMethodDef(mdef, IntPtr.Zero, IntPtr.Zero, 0, IntPtr.Zero); - } - - internal static void FreeMethodDef(IntPtr mdef) - { - unsafe - { - var def = (PyMethodDef*)mdef; - if (def->ml_name != IntPtr.Zero) - { - Marshal.FreeHGlobal(def->ml_name); - def->ml_name = IntPtr.Zero; - } - if (def->ml_doc != IntPtr.Zero) - { - Marshal.FreeHGlobal(def->ml_doc); - def->ml_doc = IntPtr.Zero; - } - } - } - - internal static PyType CreateMetatypeWithGCHandleOffset() - { - var py_type = new PyType(Runtime.PyTypeType, prevalidated: true); - int size = Util.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) - + IntPtr.Size // tp_clr_inst_offset - ; - - var slots = new[] { - new TypeSpec.Slot(TypeSlotID.tp_traverse, subtype_traverse), - new TypeSpec.Slot(TypeSlotID.tp_clear, subtype_clear) - }; - var result = new PyType( - new TypeSpec( - "clr._internal.GCOffsetBase", - basicSize: size, - slots: slots, - TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC - ), - bases: new PyTuple(new[] { py_type }) - ); - - return result; - } - - internal static PyType CreateMetaType(Type impl, out SlotsHolder slotsHolder) - { - // The managed metatype is functionally little different than the - // standard Python metatype (PyType_Type). It overrides certain of - // the standard type slots, and has to subclass PyType_Type for - // certain functions in the C runtime to work correctly with it. - - PyType gcOffsetBase = CreateMetatypeWithGCHandleOffset(); - - PyType type = AllocateTypeObject("CLRMetatype", metatype: gcOffsetBase); - - Util.WriteRef(type, TypeOffset.tp_base, new NewReference(gcOffsetBase).Steal()); - - nint size = Util.ReadInt32(gcOffsetBase, TypeOffset.tp_basicsize) - + IntPtr.Size // tp_clr_inst - ; - Util.WriteIntPtr(type, TypeOffset.tp_basicsize, size); - Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, ManagedType.Offsets.tp_clr_inst); - - const TypeFlags flags = TypeFlags.Default - | TypeFlags.HeapType - | TypeFlags.HaveGC - | TypeFlags.HasClrInstance; - Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); - - // Slots will inherit from TypeType, it's not neccesary for setting them. - // Inheried slots: - // tp_basicsize, tp_itemsize, - // tp_dictoffset, tp_weaklistoffset, - // tp_traverse, tp_clear, tp_is_gc, etc. - slotsHolder = SetupMetaSlots(impl, type); - - if (Runtime.PyType_Ready(type) != 0) - { - throw PythonException.ThrowLastAsClrException(); - } - - BorrowedReference dict = Util.ReadRef(type, TypeOffset.tp_dict); - using (var mod = Runtime.PyString_FromString("clr._internal")) - Runtime.PyDict_SetItemString(dict, "__module__", mod.Borrow()); - - // The type has been modified after PyType_Ready has been called - // Refresh the type - Runtime.PyType_Modified(type); - //DebugUtil.DumpType(type); - - return type; - } - - internal static SlotsHolder SetupMetaSlots(Type impl, PyType type) - { - // Override type slots with those of the managed implementation. - var slotsHolder = new SlotsHolder(type); - InitializeSlots(type, impl, slotsHolder); - - // We need space for 3 PyMethodDef structs. - int mdefSize = (MetaType.CustomMethods.Length + 1) * Marshal.SizeOf(typeof(PyMethodDef)); - IntPtr mdef = Runtime.PyMem_Malloc(mdefSize); - IntPtr mdefStart = mdef; - foreach (var methodName in MetaType.CustomMethods) - { - mdef = AddCustomMetaMethod(methodName, type, mdef, slotsHolder); - } - mdef = WriteMethodDefSentinel(mdef); - Debug.Assert((long)(mdefStart + mdefSize) <= (long)mdef); - - Util.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); - - // XXX: Hard code with mode check. - if (Runtime.HostedInPython) - { - slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => - { - var p = Util.ReadIntPtr(t, offset); - Runtime.PyMem_Free(p); - Util.WriteIntPtr(t, offset, IntPtr.Zero); - }); - } - return slotsHolder; - } - - private static IntPtr AddCustomMetaMethod(string name, PyType type, IntPtr mdef, SlotsHolder slotsHolder) - { - MethodInfo mi = typeof(MetaType).GetMethod(name); - ThunkInfo thunkInfo = Interop.GetThunk(mi); - slotsHolder.KeeapAlive(thunkInfo); - - // XXX: Hard code with mode check. - if (Runtime.HostedInPython) - { - IntPtr mdefAddr = mdef; - slotsHolder.AddDealloctor(() => - { - var tp_dict = Util.ReadRef(type, TypeOffset.tp_dict); - if (Runtime.PyDict_DelItemString(tp_dict, name) != 0) - { - Runtime.PyErr_Print(); - Debug.Fail($"Cannot remove {name} from metatype"); - } - FreeMethodDef(mdefAddr); - }); - } - mdef = WriteMethodDef(mdef, name, thunkInfo.Address); - return mdef; - } - - /// - /// Utility method to allocate a type object & do basic initialization. - /// - internal static PyType AllocateTypeObject(string name, PyType metatype) - { - var newType = Runtime.PyType_GenericAlloc(metatype, 0); - var type = new PyType(newType.StealOrThrow()); - // Clr type would not use __slots__, - // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), - // thus set the ob_size to 0 for avoiding slots iterations. - Util.WriteIntPtr(type, TypeOffset.ob_size, IntPtr.Zero); - - // Cheat a little: we'll set tp_name to the internal char * of - // the Python version of the type name - otherwise we'd have to - // allocate the tp_name and would have no way to free it. - using var temp = Runtime.PyString_FromString(name); - IntPtr raw = Runtime.PyUnicode_AsUTF8(temp.BorrowOrThrow()); - Util.WriteIntPtr(type, TypeOffset.tp_name, raw); - Util.WriteRef(type, TypeOffset.name, new NewReference(temp).Steal()); - Util.WriteRef(type, TypeOffset.qualname, temp.Steal()); - - // Ensure that tp_traverse and tp_clear are always set, since their - // existence is enforced in newer Python versions in PyType_Ready - Util.WriteIntPtr(type, TypeOffset.tp_traverse, subtype_traverse); - Util.WriteIntPtr(type, TypeOffset.tp_clear, subtype_clear); - - InheritSubstructs(type.Reference.DangerousGetAddress()); - - return type; - } - - /// - /// Inherit substructs, that are not inherited by default: - /// https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_as_number - /// - static void InheritSubstructs(IntPtr type) - { - IntPtr substructAddress = type + TypeOffset.nb_add; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, substructAddress); - - substructAddress = type + TypeOffset.sq_length; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, substructAddress); - - substructAddress = type + TypeOffset.mp_length; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, substructAddress); - - substructAddress = type + TypeOffset.bf_getbuffer; - Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, substructAddress); - } - - /// - /// Given a newly allocated Python type object and a managed Type that - /// provides the implementation for the type, connect the type slots of - /// the Python object to the managed methods of the implementing Type. - /// - internal static void InitializeSlots(PyType type, Type impl, SlotsHolder? slotsHolder = null) - { - // We work from the most-derived class up; make sure to get - // the most-derived slot and not to override it with a base - // class's slot. - var seen = new HashSet(); - - while (impl != null) - { - MethodInfo[] methods = impl.GetMethods(tbFlags); - foreach (MethodInfo method in methods) - { - string name = method.Name; - if (!name.StartsWith("tp_") && !TypeOffset.IsSupportedSlotName(name)) - { - Debug.Assert(!name.Contains("_") || name.StartsWith("_") || method.IsSpecialName); - continue; - } - - if (seen.Contains(name)) - { - continue; - } - - InitializeSlot(type, Interop.GetThunk(method), name, slotsHolder); - - seen.Add(name); - } - - var initSlot = impl.GetMethod("InitializeSlots", BindingFlags.Static | BindingFlags.Public); - initSlot?.Invoke(null, parameters: new object?[] { type, seen, slotsHolder }); - - impl = impl.BaseType; - } - - SetRequiredSlots(type, seen); - } - - private static void SetRequiredSlots(PyType type, HashSet seen) - { - foreach (string slot in _requiredSlots) - { - if (seen.Contains(slot)) - { - continue; - } - var offset = TypeOffset.GetSlotOffset(slot); - Util.WriteIntPtr(type, offset, SlotsHolder.GetDefaultSlot(offset)); - } - } - - static void InitializeSlot(BorrowedReference type, ThunkInfo thunk, string name, SlotsHolder? slotsHolder) - { - if (!Enum.TryParse(name, out _)) - { - throw new NotSupportedException("Bad slot name " + name); - } - int offset = TypeOffset.GetSlotOffset(name); - InitializeSlot(type, offset, thunk, slotsHolder); - } - - static void InitializeSlot(BorrowedReference type, int slotOffset, MethodInfo method, SlotsHolder slotsHolder) - { - var thunk = Interop.GetThunk(method); - InitializeSlot(type, slotOffset, thunk, slotsHolder); - } - - internal static void InitializeSlot(BorrowedReference type, int slotOffset, Delegate impl, SlotsHolder slotsHolder) - { - var thunk = Interop.GetThunk(impl); - InitializeSlot(type, slotOffset, thunk, slotsHolder); - } - - internal static void InitializeSlotIfEmpty(BorrowedReference type, int slotOffset, Delegate impl, SlotsHolder slotsHolder) - { - if (slotsHolder.IsHolding(slotOffset)) return; - InitializeSlot(type, slotOffset, impl, slotsHolder); - } - - static void InitializeSlot(BorrowedReference type, int slotOffset, ThunkInfo thunk, SlotsHolder? slotsHolder) - { - Util.WriteIntPtr(type, slotOffset, thunk.Address); - if (slotsHolder != null) - { - slotsHolder.Set(slotOffset, thunk); - } - } - - /// - /// Utility method to copy slots from a given type to another type. - /// - internal static void CopySlot(BorrowedReference from, BorrowedReference to, int offset) - { - IntPtr fp = Util.ReadIntPtr(from, offset); - Util.WriteIntPtr(to, offset, fp); - } - - internal static SlotsHolder CreateSlotsHolder(PyType type) - { - type = new PyType(type); - var holder = new SlotsHolder(type); - _slotsHolders.Add(type, holder); - return holder; - } - } - - - class SlotsHolder - { - public delegate void Resetor(PyType type, int offset); - - private readonly Dictionary _slots = new(); - private readonly List _keepalive = new(); - private readonly Dictionary _customResetors = new(); - private readonly List _deallocators = new(); - private bool _alreadyReset = false; - - private readonly PyType Type; - - public string?[] Holds => _slots.Keys.Select(TypeOffset.GetSlotName).ToArray(); - - /// - /// Create slots holder for holding the delegate of slots and be able to reset them. - /// - /// Steals a reference to target type - public SlotsHolder(PyType type) - { - this.Type = type; - } - - public bool IsHolding(int offset) => _slots.ContainsKey(offset); - - public ICollection Slots => _slots.Keys; - - public void Set(int offset, ThunkInfo thunk) - { - _slots[offset] = thunk; - } - - public void Set(int offset, Resetor resetor) - { - _customResetors[offset] = resetor; - } - - public void AddDealloctor(Action deallocate) - { - _deallocators.Add(deallocate); - } - - public void KeeapAlive(ThunkInfo thunk) - { - _keepalive.Add(thunk); - } - - public static void ResetSlots(BorrowedReference type, IEnumerable slots) - { - foreach (int offset in slots) - { - IntPtr ptr = GetDefaultSlot(offset); -#if DEBUG - //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); -#endif - Util.WriteIntPtr(type, offset, ptr); - } - } - - public void ResetSlots() - { - if (_alreadyReset) - { - return; - } - _alreadyReset = true; -#if DEBUG - IntPtr tp_name = Util.ReadIntPtr(Type, TypeOffset.tp_name); - string typeName = Marshal.PtrToStringAnsi(tp_name); -#endif - ResetSlots(Type, _slots.Keys); - - foreach (var action in _deallocators) - { - action(); - } - - foreach (var pair in _customResetors) - { - int offset = pair.Key; - var resetor = pair.Value; - resetor?.Invoke(Type, offset); - } - - _customResetors.Clear(); - _slots.Clear(); - _keepalive.Clear(); - _deallocators.Clear(); - - // Custom reset - if (Type != Runtime.CLRMetaType) - { - var metatype = Runtime.PyObject_TYPE(Type); - ManagedType.TryFreeGCHandle(Type, metatype); - } - Runtime.PyType_Modified(Type); - } - - public static IntPtr GetDefaultSlot(int offset) - { - if (offset == TypeOffset.tp_clear) - { - return TypeManager.subtype_clear; - } - else if (offset == TypeOffset.tp_traverse) - { - return TypeManager.subtype_traverse; - } - else if (offset == TypeOffset.tp_dealloc) - { - // tp_free of PyTypeType is point to PyObejct_GC_Del. - return Util.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); - } - else if (offset == TypeOffset.tp_free) - { - // PyObject_GC_Del - return Util.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); - } - else if (offset == TypeOffset.tp_call) - { - return IntPtr.Zero; - } - else if (offset == TypeOffset.tp_new) - { - // PyType_GenericNew - return Util.ReadIntPtr(Runtime.PySuper_Type, TypeOffset.tp_new); - } - else if (offset == TypeOffset.tp_getattro) - { - // PyObject_GenericGetAttr - return Util.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_getattro); - } - else if (offset == TypeOffset.tp_setattro) - { - // PyObject_GenericSetAttr - return Util.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_setattro); - } - - return Util.ReadIntPtr(Runtime.PyTypeType, offset); - } - } - - - static class SlotHelper - { - public static NewReference CreateObjectType() - { - using var globals = Runtime.PyDict_New(); - if (Runtime.PyDict_SetItemString(globals.Borrow(), "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) - { - globals.Dispose(); - throw PythonException.ThrowLastAsClrException(); - } - const string code = "class A(object): pass"; - using var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals.Borrow(), globals.Borrow()); - if (resRef.IsNull()) - { - globals.Dispose(); - throw PythonException.ThrowLastAsClrException(); - } - resRef.Dispose(); - BorrowedReference A = Runtime.PyDict_GetItemString(globals.Borrow(), "A"); - return new NewReference(A); - } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Diagnostics; +using Python.Runtime.Native; +using Python.Runtime.StateSerialization; + + +namespace Python.Runtime +{ + + /// + /// The TypeManager class is responsible for building binary-compatible + /// Python type objects that are implemented in managed code. + /// + internal class TypeManager + { + internal static IntPtr subtype_traverse; + internal static IntPtr subtype_clear; +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + /// initialized in rather than in constructor + internal static IPythonBaseTypeProvider pythonBaseTypeProvider; +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + + + private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; + private static readonly Dictionary cache = new(); + + static readonly Dictionary _slotsHolders = new(PythonReferenceComparer.Instance); + + // Slots which must be set + private static readonly string[] _requiredSlots = new string[] + { + "tp_traverse", + "tp_clear", + }; + + internal static void Initialize() + { + Debug.Assert(cache.Count == 0, "Cache should be empty", + "Some errors may occurred on last shutdown"); + using (var plainType = SlotHelper.CreateObjectType()) + { + subtype_traverse = Util.ReadIntPtr(plainType.Borrow(), TypeOffset.tp_traverse); + subtype_clear = Util.ReadIntPtr(plainType.Borrow(), TypeOffset.tp_clear); + } + pythonBaseTypeProvider = PythonEngine.InteropConfiguration.pythonBaseTypeProviders; + } + + internal static void RemoveTypes() + { + if (Runtime.HostedInPython) + { + foreach (var holder in _slotsHolders) + { + // If refcount > 1, it needs to reset the managed slot, + // otherwise it can dealloc without any trick. + if (holder.Key.Refcount > 1) + { + holder.Value.ResetSlots(); + } + } + } + + foreach (var type in cache.Values) + { + type.Dispose(); + } + cache.Clear(); + _slotsHolders.Clear(); + } + + internal static TypeManagerState SaveRuntimeData() + => new() + { + Cache = cache, + }; + + internal static void RestoreRuntimeData(TypeManagerState storage) + { + Debug.Assert(cache == null || cache.Count == 0); + var typeCache = storage.Cache; + foreach (var entry in typeCache) + { + var type = entry.Key.Value; + cache![type] = entry.Value; + SlotsHolder holder = CreateSlotsHolder(entry.Value); + InitializeSlots(entry.Value, type, holder); + Runtime.PyType_Modified(entry.Value); + } + } + + internal static PyType GetType(Type type) + { + // Note that these types are cached with a refcount of 1, so they + // effectively exist until the CPython runtime is finalized. + if (!cache.TryGetValue(type, out var pyType)) + { + pyType = CreateType(type); + cache[type] = pyType; + } + return pyType; + } + /// + /// Given a managed Type derived from ExtensionType, get the handle to + /// a Python type object that delegates its implementation to the Type + /// object. These Python type instances are used to implement internal + /// descriptor and utility types like ModuleObject, PropertyObject, etc. + /// + internal static BorrowedReference GetTypeReference(Type type) => GetType(type).Reference; + + /// + /// The following CreateType implementations do the necessary work to + /// create Python types to represent managed extension types, reflected + /// types, subclasses of reflected types and the managed metatype. The + /// dance is slightly different for each kind of type due to different + /// behavior needed and the desire to have the existing Python runtime + /// do as much of the allocation and initialization work as possible. + /// + internal static unsafe PyType CreateType(Type impl) + { + // TODO: use PyType(TypeSpec) constructor + PyType type = AllocateTypeObject(impl.Name, metatype: Runtime.PyCLRMetaType); + + BorrowedReference base_ = impl == typeof(CLRModule) + ? Runtime.PyModuleType + : Runtime.PyBaseObjectType; + + type.BaseReference = base_; + + int newFieldOffset = InheritOrAllocateStandardFields(type, base_); + + int tp_clr_inst_offset = newFieldOffset; + newFieldOffset += IntPtr.Size; + + int ob_size = newFieldOffset; + // Set tp_basicsize to the size of our managed instance objects. + Util.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); + Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, tp_clr_inst_offset); + Util.WriteIntPtr(type, TypeOffset.tp_new, (IntPtr)Runtime.Delegates.PyType_GenericNew); + + SlotsHolder slotsHolder = CreateSlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); + + type.Flags = TypeFlags.Default | TypeFlags.HasClrInstance | + TypeFlags.HeapType | TypeFlags.HaveGC; + + if (Runtime.PyType_Ready(type) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } + + + using (var dict = Runtime.PyObject_GenericGetDict(type.Reference)) + using (var mod = Runtime.PyString_FromString("CLR")) + { + Runtime.PyDict_SetItem(dict.Borrow(), PyIdentifier.__module__, mod.Borrow()); + } + + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type.Reference); + return type; + } + + + internal static void InitializeClassCore(Type clrType, PyType pyType, ClassBase impl) + { + if (pyType.BaseReference != null) + { + return; + } + + // Hide the gchandle of the implementation in a magic type slot. + GCHandle gc = GCHandle.Alloc(impl); + ManagedType.InitGCHandle(pyType, Runtime.CLRMetaType, gc); + + using var baseTuple = GetBaseTypeTuple(clrType); + + InitializeBases(pyType, baseTuple); + // core fields must be initialized in partially constructed classes, + // otherwise it would be impossible to manipulate GCHandle and check type size + InitializeCoreFields(pyType); + } + + internal static string GetPythonTypeName(Type clrType) + { + var result = new System.Text.StringBuilder(); + GetPythonTypeName(clrType, target: result); + return result.ToString(); + } + + static void GetPythonTypeName(Type clrType, System.Text.StringBuilder target) + { + if (clrType.IsGenericType) + { + string fullName = clrType.GetGenericTypeDefinition().FullName; + int argCountIndex = fullName.LastIndexOf('`'); + if (argCountIndex >= 0) + { + string nonGenericFullName = fullName.Substring(0, argCountIndex); + string nonGenericName = CleanupFullName(nonGenericFullName); + target.Append(nonGenericName); + + var arguments = clrType.GetGenericArguments(); + target.Append('['); + for (int argIndex = 0; argIndex < arguments.Length; argIndex++) + { + if (argIndex != 0) + { + target.Append(','); + } + + GetPythonTypeName(arguments[argIndex], target); + } + + target.Append(']'); + + int nestedStart = fullName.IndexOf('+'); + while (nestedStart >= 0) + { + target.Append('.'); + int nextNested = fullName.IndexOf('+', nestedStart + 1); + if (nextNested < 0) + { + target.Append(fullName.Substring(nestedStart + 1)); + } + else + { + target.Append(fullName.Substring(nestedStart + 1, length: nextNested - nestedStart - 1)); + } + nestedStart = nextNested; + } + + return; + } + } + + string name = CleanupFullName(clrType.FullName); + target.Append(name); + } + + static string CleanupFullName(string fullTypeName) + { + // Cleanup the type name to get rid of funny nested type names. + string name = "clr." + fullTypeName; + int i = name.LastIndexOf('+'); + if (i > -1) + { + name = name.Substring(i + 1); + } + + i = name.LastIndexOf('.'); + if (i > -1) + { + name = name.Substring(i + 1); + } + + return name; + } + + static BorrowedReference InitializeBases(PyType pyType, PyTuple baseTuple) + { + Debug.Assert(baseTuple.Length() > 0); + var primaryBase = baseTuple[0].Reference; + pyType.BaseReference = primaryBase; + + if (baseTuple.Length() > 1) + { + Util.WriteIntPtr(pyType, TypeOffset.tp_bases, baseTuple.NewReferenceOrNull().DangerousMoveToPointer()); + } + return primaryBase; + } + + static void InitializeCoreFields(PyType type) + { + int newFieldOffset = InheritOrAllocateStandardFields(type); + + if (ManagedType.IsManagedType(type.BaseReference)) + { + int baseClrInstOffset = Util.ReadInt32(type.BaseReference, ManagedType.Offsets.tp_clr_inst_offset); + Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, baseClrInstOffset); + } + else + { + Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, newFieldOffset); + newFieldOffset += IntPtr.Size; + } + + int ob_size = newFieldOffset; + + Util.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); + Util.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); + } + + internal static void InitializeClass(PyType type, ClassBase impl, Type clrType) + { + // we want to do this after the slot stuff above in case the class itself implements a slot method + SlotsHolder slotsHolder = CreateSlotsHolder(type); + InitializeSlots(type, impl.GetType(), slotsHolder); + + impl.InitializeSlots(type, slotsHolder); + + OperatorMethod.FixupSlots(type, clrType); + // Leverage followup initialization from the Python runtime. Note + // that the type of the new type must PyType_Type at the time we + // call this, else PyType_Ready will skip some slot initialization. + + if (!type.IsReady && Runtime.PyType_Ready(type) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } + + var dict = Util.ReadRef(type, TypeOffset.tp_dict); + string mn = clrType.Namespace ?? ""; + using (var mod = Runtime.PyString_FromString(mn)) + Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod.Borrow()); + + Runtime.PyType_Modified(type.Reference); + + //DebugUtil.DumpType(type); + } + + static int InheritOrAllocateStandardFields(BorrowedReference type) + { + var @base = Util.ReadRef(type, TypeOffset.tp_base); + return InheritOrAllocateStandardFields(type, @base); + } + static int InheritOrAllocateStandardFields(BorrowedReference typeRef, BorrowedReference @base) + { + IntPtr baseAddress = @base.DangerousGetAddress(); + IntPtr type = typeRef.DangerousGetAddress(); + int baseSize = Util.ReadInt32(@base, TypeOffset.tp_basicsize); + int newFieldOffset = baseSize; + + void InheritOrAllocate(int typeField) + { + int value = Marshal.ReadInt32(baseAddress, typeField); + if (value == 0) + { + Marshal.WriteIntPtr(type, typeField, new IntPtr(newFieldOffset)); + newFieldOffset += IntPtr.Size; + } + else + { + Marshal.WriteIntPtr(type, typeField, new IntPtr(value)); + } + } + + InheritOrAllocate(TypeOffset.tp_dictoffset); + InheritOrAllocate(TypeOffset.tp_weaklistoffset); + + return newFieldOffset; + } + + static PyTuple GetBaseTypeTuple(Type clrType) + { + var bases = pythonBaseTypeProvider + .GetBaseTypes(clrType, new PyType[0]) + ?.ToArray(); + if (bases is null || bases.Length == 0) + { + throw new InvalidOperationException("At least one base type must be specified"); + } + var nonBases = bases.Where(@base => !@base.Flags.HasFlag(TypeFlags.BaseType)).ToList(); + if (nonBases.Count > 0) + { + throw new InvalidProgramException("The specified Python type(s) can not be inherited from: " + + string.Join(", ", nonBases)); + } + + return new PyTuple(bases); + } + + internal static NewReference CreateSubType(BorrowedReference py_name, BorrowedReference py_base_type, BorrowedReference dictRef) + { + // Utility to create a subtype of a managed type with the ability for the + // a python subtype able to override the managed implementation + string? name = Runtime.GetManagedString(py_name); + if (name is null) + { + Exceptions.SetError(Exceptions.ValueError, "Class name must not be None"); + return default; + } + + // the derived class can have class attributes __assembly__ and __module__ which + // control the name of the assembly and module the new type is created in. + object? assembly = null; + object? namespaceStr = null; + + using (var assemblyKey = new PyString("__assembly__")) + { + var assemblyPtr = Runtime.PyDict_GetItemWithError(dictRef, assemblyKey.Reference); + if (assemblyPtr.IsNull) + { + if (Exceptions.ErrorOccurred()) return default; + } + else if (!Converter.ToManagedValue(assemblyPtr, typeof(string), out assembly, true)) + { + return Exceptions.RaiseTypeError("Couldn't convert __assembly__ value to string"); + } + + using var namespaceKey = new PyString("__namespace__"); + var pyNamespace = Runtime.PyDict_GetItemWithError(dictRef, namespaceKey.Reference); + if (pyNamespace.IsNull) + { + if (Exceptions.ErrorOccurred()) return default; + } + else if (!Converter.ToManagedValue(pyNamespace, typeof(string), out namespaceStr, true)) + { + return Exceptions.RaiseTypeError("Couldn't convert __namespace__ value to string"); + } + } + + // create the new managed type subclassing the base managed type + if (ManagedType.GetManagedObject(py_base_type) is ClassBase baseClass) + { + return ReflectedClrType.CreateSubclass(baseClass, name, + ns: (string?)namespaceStr, + assembly: (string?)assembly, + dict: dictRef); + } + else + { + return Exceptions.RaiseTypeError("invalid base class, expected CLR class type"); + } + } + + internal static IntPtr WriteMethodDef(IntPtr mdef, IntPtr name, IntPtr func, PyMethodFlags flags, IntPtr doc) + { + Marshal.WriteIntPtr(mdef, name); + Marshal.WriteIntPtr(mdef, 1 * IntPtr.Size, func); + Marshal.WriteInt32(mdef, 2 * IntPtr.Size, (int)flags); + Marshal.WriteIntPtr(mdef, 3 * IntPtr.Size, doc); + return mdef + 4 * IntPtr.Size; + } + + internal static IntPtr WriteMethodDef(IntPtr mdef, string name, IntPtr func, PyMethodFlags flags = PyMethodFlags.VarArgs, + string? doc = null) + { + IntPtr namePtr = Marshal.StringToHGlobalAnsi(name); + IntPtr docPtr = doc != null ? Marshal.StringToHGlobalAnsi(doc) : IntPtr.Zero; + + return WriteMethodDef(mdef, namePtr, func, flags, docPtr); + } + + internal static IntPtr WriteMethodDefSentinel(IntPtr mdef) + { + return WriteMethodDef(mdef, IntPtr.Zero, IntPtr.Zero, 0, IntPtr.Zero); + } + + internal static void FreeMethodDef(IntPtr mdef) + { + unsafe + { + var def = (PyMethodDef*)mdef; + if (def->ml_name != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_name); + def->ml_name = IntPtr.Zero; + } + if (def->ml_doc != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_doc); + def->ml_doc = IntPtr.Zero; + } + } + } + + internal static PyType CreateMetatypeWithGCHandleOffset() + { + var py_type = new PyType(Runtime.PyTypeType, prevalidated: true); + int size = Util.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) + + IntPtr.Size // tp_clr_inst_offset + ; + + var slots = new[] { + new TypeSpec.Slot(TypeSlotID.tp_traverse, subtype_traverse), + new TypeSpec.Slot(TypeSlotID.tp_clear, subtype_clear) + }; + var result = new PyType( + new TypeSpec( + "clr._internal.GCOffsetBase", + basicSize: size, + slots: slots, + TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC + ), + bases: new PyTuple(new[] { py_type }) + ); + + return result; + } + + internal static PyType CreateMetaType(Type impl, out SlotsHolder slotsHolder) + { + // The managed metatype is functionally little different than the + // standard Python metatype (PyType_Type). It overrides certain of + // the standard type slots, and has to subclass PyType_Type for + // certain functions in the C runtime to work correctly with it. + + PyType gcOffsetBase = CreateMetatypeWithGCHandleOffset(); + + PyType type = AllocateTypeObject("CLRMetatype", metatype: gcOffsetBase); + + Util.WriteRef(type, TypeOffset.tp_base, new NewReference(gcOffsetBase).Steal()); + + nint size = Util.ReadInt32(gcOffsetBase, TypeOffset.tp_basicsize) + + IntPtr.Size // tp_clr_inst + ; + Util.WriteIntPtr(type, TypeOffset.tp_basicsize, size); + Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, ManagedType.Offsets.tp_clr_inst); + + const TypeFlags flags = TypeFlags.Default + | TypeFlags.HeapType + | TypeFlags.HaveGC + | TypeFlags.HasClrInstance; + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); + + // Slots will inherit from TypeType, it's not neccesary for setting them. + // Inheried slots: + // tp_basicsize, tp_itemsize, + // tp_dictoffset, tp_weaklistoffset, + // tp_traverse, tp_clear, tp_is_gc, etc. + slotsHolder = SetupMetaSlots(impl, type); + + if (Runtime.PyType_Ready(type) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } + + BorrowedReference dict = Util.ReadRef(type, TypeOffset.tp_dict); + using (var mod = Runtime.PyString_FromString("clr._internal")) + Runtime.PyDict_SetItemString(dict, "__module__", mod.Borrow()); + + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); + //DebugUtil.DumpType(type); + + return type; + } + + internal static SlotsHolder SetupMetaSlots(Type impl, PyType type) + { + // Override type slots with those of the managed implementation. + var slotsHolder = new SlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); + + // We need space for 3 PyMethodDef structs. + int mdefSize = (MetaType.CustomMethods.Length + 1) * Marshal.SizeOf(typeof(PyMethodDef)); + IntPtr mdef = Runtime.PyMem_Malloc(mdefSize); + IntPtr mdefStart = mdef; + foreach (var methodName in MetaType.CustomMethods) + { + mdef = AddCustomMetaMethod(methodName, type, mdef, slotsHolder); + } + mdef = WriteMethodDefSentinel(mdef); + Debug.Assert((long)(mdefStart + mdefSize) <= (long)mdef); + + Util.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); + + // XXX: Hard code with mode check. + if (Runtime.HostedInPython) + { + slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => + { + var p = Util.ReadIntPtr(t, offset); + Runtime.PyMem_Free(p); + Util.WriteIntPtr(t, offset, IntPtr.Zero); + }); + } + return slotsHolder; + } + + private static IntPtr AddCustomMetaMethod(string name, PyType type, IntPtr mdef, SlotsHolder slotsHolder) + { + MethodInfo mi = typeof(MetaType).GetMethod(name); + ThunkInfo thunkInfo = Interop.GetThunk(mi); + slotsHolder.KeeapAlive(thunkInfo); + + // XXX: Hard code with mode check. + if (Runtime.HostedInPython) + { + IntPtr mdefAddr = mdef; + slotsHolder.AddDealloctor(() => + { + var tp_dict = Util.ReadRef(type, TypeOffset.tp_dict); + if (Runtime.PyDict_DelItemString(tp_dict, name) != 0) + { + Runtime.PyErr_Print(); + Debug.Fail($"Cannot remove {name} from metatype"); + } + FreeMethodDef(mdefAddr); + }); + } + mdef = WriteMethodDef(mdef, name, thunkInfo.Address); + return mdef; + } + + /// + /// Utility method to allocate a type object & do basic initialization. + /// + internal static PyType AllocateTypeObject(string name, PyType metatype) + { + var newType = Runtime.PyType_GenericAlloc(metatype, 0); + var type = new PyType(newType.StealOrThrow()); + // Clr type would not use __slots__, + // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), + // thus set the ob_size to 0 for avoiding slots iterations. + Util.WriteIntPtr(type, TypeOffset.ob_size, IntPtr.Zero); + + // Cheat a little: we'll set tp_name to the internal char * of + // the Python version of the type name - otherwise we'd have to + // allocate the tp_name and would have no way to free it. + using var temp = Runtime.PyString_FromString(name); + IntPtr raw = Runtime.PyUnicode_AsUTF8(temp.BorrowOrThrow()); + Util.WriteIntPtr(type, TypeOffset.tp_name, raw); + Util.WriteRef(type, TypeOffset.name, new NewReference(temp).Steal()); + Util.WriteRef(type, TypeOffset.qualname, temp.Steal()); + + // Ensure that tp_traverse and tp_clear are always set, since their + // existence is enforced in newer Python versions in PyType_Ready + Util.WriteIntPtr(type, TypeOffset.tp_traverse, subtype_traverse); + Util.WriteIntPtr(type, TypeOffset.tp_clear, subtype_clear); + + InheritSubstructs(type.Reference.DangerousGetAddress()); + + return type; + } + + /// + /// Inherit substructs, that are not inherited by default: + /// https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_as_number + /// + static void InheritSubstructs(IntPtr type) + { + IntPtr substructAddress = type + TypeOffset.nb_add; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, substructAddress); + + substructAddress = type + TypeOffset.sq_length; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, substructAddress); + + substructAddress = type + TypeOffset.mp_length; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, substructAddress); + + substructAddress = type + TypeOffset.bf_getbuffer; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, substructAddress); + } + + /// + /// Given a newly allocated Python type object and a managed Type that + /// provides the implementation for the type, connect the type slots of + /// the Python object to the managed methods of the implementing Type. + /// + internal static void InitializeSlots(PyType type, Type impl, SlotsHolder? slotsHolder = null) + { + // We work from the most-derived class up; make sure to get + // the most-derived slot and not to override it with a base + // class's slot. + var seen = new HashSet(); + + while (impl != null) + { + MethodInfo[] methods = impl.GetMethods(tbFlags); + foreach (MethodInfo method in methods) + { + string name = method.Name; + if (!name.StartsWith("tp_") && !TypeOffset.IsSupportedSlotName(name)) + { + Debug.Assert(!name.Contains("_") || name.StartsWith("_") || method.IsSpecialName); + continue; + } + + if (seen.Contains(name)) + { + continue; + } + + InitializeSlot(type, Interop.GetThunk(method), name, slotsHolder); + + seen.Add(name); + } + + var initSlot = impl.GetMethod("InitializeSlots", BindingFlags.Static | BindingFlags.Public); + initSlot?.Invoke(null, parameters: new object?[] { type, seen, slotsHolder }); + + impl = impl.BaseType; + } + + SetRequiredSlots(type, seen); + } + + private static void SetRequiredSlots(PyType type, HashSet seen) + { + foreach (string slot in _requiredSlots) + { + if (seen.Contains(slot)) + { + continue; + } + var offset = TypeOffset.GetSlotOffset(slot); + Util.WriteIntPtr(type, offset, SlotsHolder.GetDefaultSlot(offset)); + } + } + + static void InitializeSlot(BorrowedReference type, ThunkInfo thunk, string name, SlotsHolder? slotsHolder) + { + if (!Enum.TryParse(name, out _)) + { + throw new NotSupportedException("Bad slot name " + name); + } + int offset = TypeOffset.GetSlotOffset(name); + InitializeSlot(type, offset, thunk, slotsHolder); + } + + static void InitializeSlot(BorrowedReference type, int slotOffset, MethodInfo method, SlotsHolder slotsHolder) + { + var thunk = Interop.GetThunk(method); + InitializeSlot(type, slotOffset, thunk, slotsHolder); + } + + internal static void InitializeSlot(BorrowedReference type, int slotOffset, Delegate impl, SlotsHolder slotsHolder) + { + var thunk = Interop.GetThunk(impl); + InitializeSlot(type, slotOffset, thunk, slotsHolder); + } + + internal static void InitializeSlotIfEmpty(BorrowedReference type, int slotOffset, Delegate impl, SlotsHolder slotsHolder) + { + if (slotsHolder.IsHolding(slotOffset)) return; + InitializeSlot(type, slotOffset, impl, slotsHolder); + } + + static void InitializeSlot(BorrowedReference type, int slotOffset, ThunkInfo thunk, SlotsHolder? slotsHolder) + { + Util.WriteIntPtr(type, slotOffset, thunk.Address); + if (slotsHolder != null) + { + slotsHolder.Set(slotOffset, thunk); + } + } + + /// + /// Utility method to copy slots from a given type to another type. + /// + internal static void CopySlot(BorrowedReference from, BorrowedReference to, int offset) + { + IntPtr fp = Util.ReadIntPtr(from, offset); + Util.WriteIntPtr(to, offset, fp); + } + + internal static SlotsHolder CreateSlotsHolder(PyType type) + { + type = new PyType(type); + var holder = new SlotsHolder(type); + _slotsHolders.Add(type, holder); + return holder; + } + } + + + class SlotsHolder + { + public delegate void Resetor(PyType type, int offset); + + private readonly Dictionary _slots = new(); + private readonly List _keepalive = new(); + private readonly Dictionary _customResetors = new(); + private readonly List _deallocators = new(); + private bool _alreadyReset = false; + + private readonly PyType Type; + + public string?[] Holds => _slots.Keys.Select(TypeOffset.GetSlotName).ToArray(); + + /// + /// Create slots holder for holding the delegate of slots and be able to reset them. + /// + /// Steals a reference to target type + public SlotsHolder(PyType type) + { + this.Type = type; + } + + public bool IsHolding(int offset) => _slots.ContainsKey(offset); + + public ICollection Slots => _slots.Keys; + + public void Set(int offset, ThunkInfo thunk) + { + _slots[offset] = thunk; + } + + public void Set(int offset, Resetor resetor) + { + _customResetors[offset] = resetor; + } + + public void AddDealloctor(Action deallocate) + { + _deallocators.Add(deallocate); + } + + public void KeeapAlive(ThunkInfo thunk) + { + _keepalive.Add(thunk); + } + + public static void ResetSlots(BorrowedReference type, IEnumerable slots) + { + foreach (int offset in slots) + { + IntPtr ptr = GetDefaultSlot(offset); +#if DEBUG + //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); +#endif + Util.WriteIntPtr(type, offset, ptr); + } + } + + public void ResetSlots() + { + if (_alreadyReset) + { + return; + } + _alreadyReset = true; +#if DEBUG + IntPtr tp_name = Util.ReadIntPtr(Type, TypeOffset.tp_name); + string typeName = Marshal.PtrToStringAnsi(tp_name); +#endif + ResetSlots(Type, _slots.Keys); + + foreach (var action in _deallocators) + { + action(); + } + + foreach (var pair in _customResetors) + { + int offset = pair.Key; + var resetor = pair.Value; + resetor?.Invoke(Type, offset); + } + + _customResetors.Clear(); + _slots.Clear(); + _keepalive.Clear(); + _deallocators.Clear(); + + // Custom reset + if (Type != Runtime.CLRMetaType) + { + var metatype = Runtime.PyObject_TYPE(Type); + ManagedType.TryFreeGCHandle(Type, metatype); + } + Runtime.PyType_Modified(Type); + } + + public static IntPtr GetDefaultSlot(int offset) + { + if (offset == TypeOffset.tp_clear) + { + return TypeManager.subtype_clear; + } + else if (offset == TypeOffset.tp_traverse) + { + return TypeManager.subtype_traverse; + } + else if (offset == TypeOffset.tp_dealloc) + { + // tp_free of PyTypeType is point to PyObejct_GC_Del. + return Util.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_free) + { + // PyObject_GC_Del + return Util.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_call) + { + return IntPtr.Zero; + } + else if (offset == TypeOffset.tp_new) + { + // PyType_GenericNew + return Util.ReadIntPtr(Runtime.PySuper_Type, TypeOffset.tp_new); + } + else if (offset == TypeOffset.tp_getattro) + { + // PyObject_GenericGetAttr + return Util.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_getattro); + } + else if (offset == TypeOffset.tp_setattro) + { + // PyObject_GenericSetAttr + return Util.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_setattro); + } + + return Util.ReadIntPtr(Runtime.PyTypeType, offset); + } + } + + + static class SlotHelper + { + public static NewReference CreateObjectType() + { + using var globals = Runtime.PyDict_New(); + if (Runtime.PyDict_SetItemString(globals.Borrow(), "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) + { + globals.Dispose(); + throw PythonException.ThrowLastAsClrException(); + } + const string code = "class A(object): pass"; + using var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals.Borrow(), globals.Borrow()); + if (resRef.IsNull()) + { + globals.Dispose(); + throw PythonException.ThrowLastAsClrException(); + } + resRef.Dispose(); + BorrowedReference A = Runtime.PyDict_GetItemString(globals.Borrow(), "A"); + return new NewReference(A); + } + } +} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Diagnostics; +using Python.Runtime.Native; +using Python.Runtime.StateSerialization; + + +namespace Python.Runtime +{ + + /// + /// The TypeManager class is responsible for building binary-compatible + /// Python type objects that are implemented in managed code. + /// + internal class TypeManager + { + internal static IntPtr subtype_traverse; + internal static IntPtr subtype_clear; +#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + /// initialized in rather than in constructor + internal static IPythonBaseTypeProvider pythonBaseTypeProvider; +#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. + + + private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; + private static readonly Dictionary cache = new(); + + static readonly Dictionary _slotsHolders = new(PythonReferenceComparer.Instance); + + // Slots which must be set + private static readonly string[] _requiredSlots = new string[] + { + "tp_traverse", + "tp_clear", + }; + + internal static void Initialize() + { + Debug.Assert(cache.Count == 0, "Cache should be empty", + "Some errors may occurred on last shutdown"); + using (var plainType = SlotHelper.CreateObjectType()) + { + subtype_traverse = Util.ReadIntPtr(plainType.Borrow(), TypeOffset.tp_traverse); + subtype_clear = Util.ReadIntPtr(plainType.Borrow(), TypeOffset.tp_clear); + } + pythonBaseTypeProvider = PythonEngine.InteropConfiguration.pythonBaseTypeProviders; + } + + internal static void RemoveTypes() + { + if (Runtime.HostedInPython) + { + foreach (var holder in _slotsHolders) + { + // If refcount > 1, it needs to reset the managed slot, + // otherwise it can dealloc without any trick. + if (holder.Key.Refcount > 1) + { + holder.Value.ResetSlots(); + } + } + } + + foreach (var type in cache.Values) + { + type.Dispose(); + } + cache.Clear(); + _slotsHolders.Clear(); + } + + internal static TypeManagerState SaveRuntimeData() + => new() + { + Cache = cache, + }; + + internal static void RestoreRuntimeData(TypeManagerState storage) + { + Debug.Assert(cache == null || cache.Count == 0); + var typeCache = storage.Cache; + foreach (var entry in typeCache) + { + var type = entry.Key.Value; + cache![type] = entry.Value; + SlotsHolder holder = CreateSlotsHolder(entry.Value); + InitializeSlots(entry.Value, type, holder); + Runtime.PyType_Modified(entry.Value); + } + } + + internal static PyType GetType(Type type) + { + // Note that these types are cached with a refcount of 1, so they + // effectively exist until the CPython runtime is finalized. + if (!cache.TryGetValue(type, out var pyType)) + { + pyType = CreateType(type); + cache[type] = pyType; + } + return pyType; + } + /// + /// Given a managed Type derived from ExtensionType, get the handle to + /// a Python type object that delegates its implementation to the Type + /// object. These Python type instances are used to implement internal + /// descriptor and utility types like ModuleObject, PropertyObject, etc. + /// + internal static BorrowedReference GetTypeReference(Type type) => GetType(type).Reference; + + /// + /// The following CreateType implementations do the necessary work to + /// create Python types to represent managed extension types, reflected + /// types, subclasses of reflected types and the managed metatype. The + /// dance is slightly different for each kind of type due to different + /// behavior needed and the desire to have the existing Python runtime + /// do as much of the allocation and initialization work as possible. + /// + internal static unsafe PyType CreateType(Type impl) + { + // TODO: use PyType(TypeSpec) constructor + PyType type = AllocateTypeObject(impl.Name, metatype: Runtime.PyCLRMetaType); + + BorrowedReference base_ = impl == typeof(CLRModule) + ? Runtime.PyModuleType + : Runtime.PyBaseObjectType; + + type.BaseReference = base_; + + int newFieldOffset = InheritOrAllocateStandardFields(type, base_); + + int tp_clr_inst_offset = newFieldOffset; + newFieldOffset += IntPtr.Size; + + int ob_size = newFieldOffset; + // Set tp_basicsize to the size of our managed instance objects. + Util.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); + Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, tp_clr_inst_offset); + Util.WriteIntPtr(type, TypeOffset.tp_new, (IntPtr)Runtime.Delegates.PyType_GenericNew); + + SlotsHolder slotsHolder = CreateSlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); + + type.Flags = TypeFlags.Default | TypeFlags.HasClrInstance | + TypeFlags.HeapType | TypeFlags.HaveGC; + + if (Runtime.PyType_Ready(type) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } + + + using (var dict = Runtime.PyObject_GenericGetDict(type.Reference)) + using (var mod = Runtime.PyString_FromString("CLR")) + { + Runtime.PyDict_SetItem(dict.Borrow(), PyIdentifier.__module__, mod.Borrow()); + } + + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type.Reference); + return type; + } + + + internal static void InitializeClassCore(Type clrType, PyType pyType, ClassBase impl) + { + if (pyType.BaseReference != null) + { + return; + } + + // Hide the gchandle of the implementation in a magic type slot. + GCHandle gc = GCHandle.Alloc(impl); + ManagedType.InitGCHandle(pyType, Runtime.CLRMetaType, gc); + + using var baseTuple = GetBaseTypeTuple(clrType); + + InitializeBases(pyType, baseTuple); + // core fields must be initialized in partially constructed classes, + // otherwise it would be impossible to manipulate GCHandle and check type size + InitializeCoreFields(pyType); + } + + internal static string GetPythonTypeName(Type clrType) + { + var result = new System.Text.StringBuilder(); + GetPythonTypeName(clrType, target: result); + return result.ToString(); + } + + static void GetPythonTypeName(Type clrType, System.Text.StringBuilder target) + { + if (clrType.IsGenericType) + { + string fullName = clrType.GetGenericTypeDefinition().FullName; + int argCountIndex = fullName.LastIndexOf('`'); + if (argCountIndex >= 0) + { + string nonGenericFullName = fullName.Substring(0, argCountIndex); + string nonGenericName = CleanupFullName(nonGenericFullName); + target.Append(nonGenericName); + + var arguments = clrType.GetGenericArguments(); + target.Append('['); + for (int argIndex = 0; argIndex < arguments.Length; argIndex++) + { + if (argIndex != 0) + { + target.Append(','); + } + + GetPythonTypeName(arguments[argIndex], target); + } + + target.Append(']'); + + int nestedStart = fullName.IndexOf('+'); + while (nestedStart >= 0) + { + target.Append('.'); + int nextNested = fullName.IndexOf('+', nestedStart + 1); + if (nextNested < 0) + { + target.Append(fullName.Substring(nestedStart + 1)); + } + else + { + target.Append(fullName.Substring(nestedStart + 1, length: nextNested - nestedStart - 1)); + } + nestedStart = nextNested; + } + + return; + } + } + + string name = CleanupFullName(clrType.FullName); + target.Append(name); + } + + static string CleanupFullName(string fullTypeName) + { + // Cleanup the type name to get rid of funny nested type names. + string name = "clr." + fullTypeName; + int i = name.LastIndexOf('+'); + if (i > -1) + { + name = name.Substring(i + 1); + } + + i = name.LastIndexOf('.'); + if (i > -1) + { + name = name.Substring(i + 1); + } + + return name; + } + + static BorrowedReference InitializeBases(PyType pyType, PyTuple baseTuple) + { + Debug.Assert(baseTuple.Length() > 0); + var primaryBase = baseTuple[0].Reference; + pyType.BaseReference = primaryBase; + + if (baseTuple.Length() > 1) + { + Util.WriteIntPtr(pyType, TypeOffset.tp_bases, baseTuple.NewReferenceOrNull().DangerousMoveToPointer()); + } + return primaryBase; + } + + static void InitializeCoreFields(PyType type) + { + int newFieldOffset = InheritOrAllocateStandardFields(type); + + if (ManagedType.IsManagedType(type.BaseReference)) + { + int baseClrInstOffset = Util.ReadInt32(type.BaseReference, ManagedType.Offsets.tp_clr_inst_offset); + Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, baseClrInstOffset); + } + else + { + Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, newFieldOffset); + newFieldOffset += IntPtr.Size; + } + + int ob_size = newFieldOffset; + + Util.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); + Util.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); + } + + internal static void InitializeClass(PyType type, ClassBase impl, Type clrType) + { + // we want to do this after the slot stuff above in case the class itself implements a slot method + SlotsHolder slotsHolder = CreateSlotsHolder(type); + InitializeSlots(type, impl.GetType(), slotsHolder); + + impl.InitializeSlots(type, slotsHolder); + + OperatorMethod.FixupSlots(type, clrType); + // Leverage followup initialization from the Python runtime. Note + // that the type of the new type must PyType_Type at the time we + // call this, else PyType_Ready will skip some slot initialization. + + if (!type.IsReady && Runtime.PyType_Ready(type) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } + + var dict = Util.ReadRef(type, TypeOffset.tp_dict); + string mn = clrType.Namespace ?? ""; + using (var mod = Runtime.PyString_FromString(mn)) + Runtime.PyDict_SetItem(dict, PyIdentifier.__module__, mod.Borrow()); + + Runtime.PyType_Modified(type.Reference); + + //DebugUtil.DumpType(type); + } + + static int InheritOrAllocateStandardFields(BorrowedReference type) + { + var @base = Util.ReadRef(type, TypeOffset.tp_base); + return InheritOrAllocateStandardFields(type, @base); + } + static int InheritOrAllocateStandardFields(BorrowedReference typeRef, BorrowedReference @base) + { + IntPtr baseAddress = @base.DangerousGetAddress(); + IntPtr type = typeRef.DangerousGetAddress(); + int baseSize = Util.ReadInt32(@base, TypeOffset.tp_basicsize); + int newFieldOffset = baseSize; + + void InheritOrAllocate(int typeField) + { + int value = Marshal.ReadInt32(baseAddress, typeField); + if (value == 0) + { + Marshal.WriteIntPtr(type, typeField, new IntPtr(newFieldOffset)); + newFieldOffset += IntPtr.Size; + } + else + { + Marshal.WriteIntPtr(type, typeField, new IntPtr(value)); + } + } + + InheritOrAllocate(TypeOffset.tp_dictoffset); + InheritOrAllocate(TypeOffset.tp_weaklistoffset); + + return newFieldOffset; + } + + static PyTuple GetBaseTypeTuple(Type clrType) + { + var bases = pythonBaseTypeProvider + .GetBaseTypes(clrType, new PyType[0]) + ?.ToArray(); + if (bases is null || bases.Length == 0) + { + throw new InvalidOperationException("At least one base type must be specified"); + } + var nonBases = bases.Where(@base => !@base.Flags.HasFlag(TypeFlags.BaseType)).ToList(); + if (nonBases.Count > 0) + { + throw new InvalidProgramException("The specified Python type(s) can not be inherited from: " + + string.Join(", ", nonBases)); + } + + return new PyTuple(bases); + } + + internal static NewReference CreateSubType(BorrowedReference py_name, ClassBase py_base_type, IList interfaces, BorrowedReference dictRef) + { + // Utility to create a subtype of a managed type with the ability for the + // a python subtype able to override the managed implementation + string? name = Runtime.GetManagedString(py_name); + if (name is null) + { + Exceptions.SetError(Exceptions.ValueError, "Class name must not be None"); + return default; + } + + // the derived class can have class attributes __assembly__ and __module__ which + // control the name of the assembly and module the new type is created in. + object? assembly = null; + object? namespaceStr = null; + + using (var assemblyKey = new PyString("__assembly__")) + { + var assemblyPtr = Runtime.PyDict_GetItemWithError(dictRef, assemblyKey.Reference); + if (assemblyPtr.IsNull) + { + if (Exceptions.ErrorOccurred()) return default; + } + else if (!Converter.ToManagedValue(assemblyPtr, typeof(string), out assembly, true)) + { + return Exceptions.RaiseTypeError("Couldn't convert __assembly__ value to string"); + } + + using var namespaceKey = new PyString("__namespace__"); + var pyNamespace = Runtime.PyDict_GetItemWithError(dictRef, namespaceKey.Reference); + if (pyNamespace.IsNull) + { + if (Exceptions.ErrorOccurred()) return default; + } + else if (!Converter.ToManagedValue(pyNamespace, typeof(string), out namespaceStr, true)) + { + return Exceptions.RaiseTypeError("Couldn't convert __namespace__ value to string"); + } + } + + // create the new managed type subclassing the base managed type + return ReflectedClrType.CreateSubclass(py_base_type, interfaces, name, + ns: (string?)namespaceStr, + assembly: (string?)assembly, + dict: dictRef); + } + + internal static IntPtr WriteMethodDef(IntPtr mdef, IntPtr name, IntPtr func, PyMethodFlags flags, IntPtr doc) + { + Marshal.WriteIntPtr(mdef, name); + Marshal.WriteIntPtr(mdef, 1 * IntPtr.Size, func); + Marshal.WriteInt32(mdef, 2 * IntPtr.Size, (int)flags); + Marshal.WriteIntPtr(mdef, 3 * IntPtr.Size, doc); + return mdef + 4 * IntPtr.Size; + } + + internal static IntPtr WriteMethodDef(IntPtr mdef, string name, IntPtr func, PyMethodFlags flags = PyMethodFlags.VarArgs, + string? doc = null) + { + IntPtr namePtr = Marshal.StringToHGlobalAnsi(name); + IntPtr docPtr = doc != null ? Marshal.StringToHGlobalAnsi(doc) : IntPtr.Zero; + + return WriteMethodDef(mdef, namePtr, func, flags, docPtr); + } + + internal static IntPtr WriteMethodDefSentinel(IntPtr mdef) + { + return WriteMethodDef(mdef, IntPtr.Zero, IntPtr.Zero, 0, IntPtr.Zero); + } + + internal static void FreeMethodDef(IntPtr mdef) + { + unsafe + { + var def = (PyMethodDef*)mdef; + if (def->ml_name != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_name); + def->ml_name = IntPtr.Zero; + } + if (def->ml_doc != IntPtr.Zero) + { + Marshal.FreeHGlobal(def->ml_doc); + def->ml_doc = IntPtr.Zero; + } + } + } + + internal static PyType CreateMetatypeWithGCHandleOffset() + { + var py_type = new PyType(Runtime.PyTypeType, prevalidated: true); + int size = Util.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) + + IntPtr.Size // tp_clr_inst_offset + ; + + var slots = new[] { + new TypeSpec.Slot(TypeSlotID.tp_traverse, subtype_traverse), + new TypeSpec.Slot(TypeSlotID.tp_clear, subtype_clear) + }; + var result = new PyType( + new TypeSpec( + "clr._internal.GCOffsetBase", + basicSize: size, + slots: slots, + TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC + ), + bases: new PyTuple(new[] { py_type }) + ); + + return result; + } + + internal static PyType CreateMetaType(Type impl, out SlotsHolder slotsHolder) + { + // The managed metatype is functionally little different than the + // standard Python metatype (PyType_Type). It overrides certain of + // the standard type slots, and has to subclass PyType_Type for + // certain functions in the C runtime to work correctly with it. + + PyType gcOffsetBase = CreateMetatypeWithGCHandleOffset(); + + PyType type = AllocateTypeObject("CLRMetatype", metatype: gcOffsetBase); + + Util.WriteRef(type, TypeOffset.tp_base, new NewReference(gcOffsetBase).Steal()); + + nint size = Util.ReadInt32(gcOffsetBase, TypeOffset.tp_basicsize) + + IntPtr.Size // tp_clr_inst + ; + Util.WriteIntPtr(type, TypeOffset.tp_basicsize, size); + Util.WriteInt32(type, ManagedType.Offsets.tp_clr_inst_offset, ManagedType.Offsets.tp_clr_inst); + + const TypeFlags flags = TypeFlags.Default + | TypeFlags.HeapType + | TypeFlags.HaveGC + | TypeFlags.HasClrInstance; + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); + + // Slots will inherit from TypeType, it's not neccesary for setting them. + // Inheried slots: + // tp_basicsize, tp_itemsize, + // tp_dictoffset, tp_weaklistoffset, + // tp_traverse, tp_clear, tp_is_gc, etc. + slotsHolder = SetupMetaSlots(impl, type); + + if (Runtime.PyType_Ready(type) != 0) + { + throw PythonException.ThrowLastAsClrException(); + } + + BorrowedReference dict = Util.ReadRef(type, TypeOffset.tp_dict); + using (var mod = Runtime.PyString_FromString("clr._internal")) + Runtime.PyDict_SetItemString(dict, "__module__", mod.Borrow()); + + // The type has been modified after PyType_Ready has been called + // Refresh the type + Runtime.PyType_Modified(type); + //DebugUtil.DumpType(type); + + return type; + } + + internal static SlotsHolder SetupMetaSlots(Type impl, PyType type) + { + // Override type slots with those of the managed implementation. + var slotsHolder = new SlotsHolder(type); + InitializeSlots(type, impl, slotsHolder); + + // We need space for 3 PyMethodDef structs. + int mdefSize = (MetaType.CustomMethods.Length + 1) * Marshal.SizeOf(typeof(PyMethodDef)); + IntPtr mdef = Runtime.PyMem_Malloc(mdefSize); + IntPtr mdefStart = mdef; + foreach (var methodName in MetaType.CustomMethods) + { + mdef = AddCustomMetaMethod(methodName, type, mdef, slotsHolder); + } + mdef = WriteMethodDefSentinel(mdef); + Debug.Assert((long)(mdefStart + mdefSize) <= (long)mdef); + + Util.WriteIntPtr(type, TypeOffset.tp_methods, mdefStart); + + // XXX: Hard code with mode check. + if (Runtime.HostedInPython) + { + slotsHolder.Set(TypeOffset.tp_methods, (t, offset) => + { + var p = Util.ReadIntPtr(t, offset); + Runtime.PyMem_Free(p); + Util.WriteIntPtr(t, offset, IntPtr.Zero); + }); + } + return slotsHolder; + } + + private static IntPtr AddCustomMetaMethod(string name, PyType type, IntPtr mdef, SlotsHolder slotsHolder) + { + MethodInfo mi = typeof(MetaType).GetMethod(name); + ThunkInfo thunkInfo = Interop.GetThunk(mi); + slotsHolder.KeeapAlive(thunkInfo); + + // XXX: Hard code with mode check. + if (Runtime.HostedInPython) + { + IntPtr mdefAddr = mdef; + slotsHolder.AddDealloctor(() => + { + var tp_dict = Util.ReadRef(type, TypeOffset.tp_dict); + if (Runtime.PyDict_DelItemString(tp_dict, name) != 0) + { + Runtime.PyErr_Print(); + Debug.Fail($"Cannot remove {name} from metatype"); + } + FreeMethodDef(mdefAddr); + }); + } + mdef = WriteMethodDef(mdef, name, thunkInfo.Address); + return mdef; + } + + /// + /// Utility method to allocate a type object & do basic initialization. + /// + internal static PyType AllocateTypeObject(string name, PyType metatype) + { + var newType = Runtime.PyType_GenericAlloc(metatype, 0); + var type = new PyType(newType.StealOrThrow()); + // Clr type would not use __slots__, + // and the PyMemberDef after PyHeapTypeObject will have other uses(e.g. type handle), + // thus set the ob_size to 0 for avoiding slots iterations. + Util.WriteIntPtr(type, TypeOffset.ob_size, IntPtr.Zero); + + // Cheat a little: we'll set tp_name to the internal char * of + // the Python version of the type name - otherwise we'd have to + // allocate the tp_name and would have no way to free it. + using var temp = Runtime.PyString_FromString(name); + IntPtr raw = Runtime.PyUnicode_AsUTF8(temp.BorrowOrThrow()); + Util.WriteIntPtr(type, TypeOffset.tp_name, raw); + Util.WriteRef(type, TypeOffset.name, new NewReference(temp).Steal()); + Util.WriteRef(type, TypeOffset.qualname, temp.Steal()); + + // Ensure that tp_traverse and tp_clear are always set, since their + // existence is enforced in newer Python versions in PyType_Ready + Util.WriteIntPtr(type, TypeOffset.tp_traverse, subtype_traverse); + Util.WriteIntPtr(type, TypeOffset.tp_clear, subtype_clear); + + InheritSubstructs(type.Reference.DangerousGetAddress()); + + return type; + } + + /// + /// Inherit substructs, that are not inherited by default: + /// https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_as_number + /// + static void InheritSubstructs(IntPtr type) + { + IntPtr substructAddress = type + TypeOffset.nb_add; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, substructAddress); + + substructAddress = type + TypeOffset.sq_length; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, substructAddress); + + substructAddress = type + TypeOffset.mp_length; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, substructAddress); + + substructAddress = type + TypeOffset.bf_getbuffer; + Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, substructAddress); + } + + /// + /// Given a newly allocated Python type object and a managed Type that + /// provides the implementation for the type, connect the type slots of + /// the Python object to the managed methods of the implementing Type. + /// + internal static void InitializeSlots(PyType type, Type impl, SlotsHolder? slotsHolder = null) + { + // We work from the most-derived class up; make sure to get + // the most-derived slot and not to override it with a base + // class's slot. + var seen = new HashSet(); + + while (impl != null) + { + MethodInfo[] methods = impl.GetMethods(tbFlags); + foreach (MethodInfo method in methods) + { + string name = method.Name; + if (!name.StartsWith("tp_") && !TypeOffset.IsSupportedSlotName(name)) + { + Debug.Assert(!name.Contains("_") || name.StartsWith("_") || method.IsSpecialName); + continue; + } + + if (seen.Contains(name)) + { + continue; + } + + InitializeSlot(type, Interop.GetThunk(method), name, slotsHolder); + + seen.Add(name); + } + + var initSlot = impl.GetMethod("InitializeSlots", BindingFlags.Static | BindingFlags.Public); + initSlot?.Invoke(null, parameters: new object?[] { type, seen, slotsHolder }); + + impl = impl.BaseType; + } + + SetRequiredSlots(type, seen); + } + + private static void SetRequiredSlots(PyType type, HashSet seen) + { + foreach (string slot in _requiredSlots) + { + if (seen.Contains(slot)) + { + continue; + } + var offset = TypeOffset.GetSlotOffset(slot); + Util.WriteIntPtr(type, offset, SlotsHolder.GetDefaultSlot(offset)); + } + } + + static void InitializeSlot(BorrowedReference type, ThunkInfo thunk, string name, SlotsHolder? slotsHolder) + { + if (!Enum.TryParse(name, out _)) + { + throw new NotSupportedException("Bad slot name " + name); + } + int offset = TypeOffset.GetSlotOffset(name); + InitializeSlot(type, offset, thunk, slotsHolder); + } + + static void InitializeSlot(BorrowedReference type, int slotOffset, MethodInfo method, SlotsHolder slotsHolder) + { + var thunk = Interop.GetThunk(method); + InitializeSlot(type, slotOffset, thunk, slotsHolder); + } + + internal static void InitializeSlot(BorrowedReference type, int slotOffset, Delegate impl, SlotsHolder slotsHolder) + { + var thunk = Interop.GetThunk(impl); + InitializeSlot(type, slotOffset, thunk, slotsHolder); + } + + internal static void InitializeSlotIfEmpty(BorrowedReference type, int slotOffset, Delegate impl, SlotsHolder slotsHolder) + { + if (slotsHolder.IsHolding(slotOffset)) return; + InitializeSlot(type, slotOffset, impl, slotsHolder); + } + + static void InitializeSlot(BorrowedReference type, int slotOffset, ThunkInfo thunk, SlotsHolder? slotsHolder) + { + Util.WriteIntPtr(type, slotOffset, thunk.Address); + if (slotsHolder != null) + { + slotsHolder.Set(slotOffset, thunk); + } + } + + /// + /// Utility method to copy slots from a given type to another type. + /// + internal static void CopySlot(BorrowedReference from, BorrowedReference to, int offset) + { + IntPtr fp = Util.ReadIntPtr(from, offset); + Util.WriteIntPtr(to, offset, fp); + } + + internal static SlotsHolder CreateSlotsHolder(PyType type) + { + type = new PyType(type); + var holder = new SlotsHolder(type); + _slotsHolders.Add(type, holder); + return holder; + } + } + + + class SlotsHolder + { + public delegate void Resetor(PyType type, int offset); + + private readonly Dictionary _slots = new(); + private readonly List _keepalive = new(); + private readonly Dictionary _customResetors = new(); + private readonly List _deallocators = new(); + private bool _alreadyReset = false; + + private readonly PyType Type; + + public string?[] Holds => _slots.Keys.Select(TypeOffset.GetSlotName).ToArray(); + + /// + /// Create slots holder for holding the delegate of slots and be able to reset them. + /// + /// Steals a reference to target type + public SlotsHolder(PyType type) + { + this.Type = type; + } + + public bool IsHolding(int offset) => _slots.ContainsKey(offset); + + public ICollection Slots => _slots.Keys; + + public void Set(int offset, ThunkInfo thunk) + { + _slots[offset] = thunk; + } + + public void Set(int offset, Resetor resetor) + { + _customResetors[offset] = resetor; + } + + public void AddDealloctor(Action deallocate) + { + _deallocators.Add(deallocate); + } + + public void KeeapAlive(ThunkInfo thunk) + { + _keepalive.Add(thunk); + } + + public static void ResetSlots(BorrowedReference type, IEnumerable slots) + { + foreach (int offset in slots) + { + IntPtr ptr = GetDefaultSlot(offset); +#if DEBUG + //DebugUtil.Print($"Set slot<{TypeOffsetHelper.GetSlotNameByOffset(offset)}> to 0x{ptr.ToString("X")} at {typeName}<0x{_type}>"); +#endif + Util.WriteIntPtr(type, offset, ptr); + } + } + + public void ResetSlots() + { + if (_alreadyReset) + { + return; + } + _alreadyReset = true; +#if DEBUG + IntPtr tp_name = Util.ReadIntPtr(Type, TypeOffset.tp_name); + string typeName = Marshal.PtrToStringAnsi(tp_name); +#endif + ResetSlots(Type, _slots.Keys); + + foreach (var action in _deallocators) + { + action(); + } + + foreach (var pair in _customResetors) + { + int offset = pair.Key; + var resetor = pair.Value; + resetor?.Invoke(Type, offset); + } + + _customResetors.Clear(); + _slots.Clear(); + _keepalive.Clear(); + _deallocators.Clear(); + + // Custom reset + if (Type != Runtime.CLRMetaType) + { + var metatype = Runtime.PyObject_TYPE(Type); + ManagedType.TryFreeGCHandle(Type, metatype); + } + Runtime.PyType_Modified(Type); + } + + public static IntPtr GetDefaultSlot(int offset) + { + if (offset == TypeOffset.tp_clear) + { + return TypeManager.subtype_clear; + } + else if (offset == TypeOffset.tp_traverse) + { + return TypeManager.subtype_traverse; + } + else if (offset == TypeOffset.tp_dealloc) + { + // tp_free of PyTypeType is point to PyObejct_GC_Del. + return Util.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_free) + { + // PyObject_GC_Del + return Util.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_free); + } + else if (offset == TypeOffset.tp_call) + { + return IntPtr.Zero; + } + else if (offset == TypeOffset.tp_new) + { + // PyType_GenericNew + return Util.ReadIntPtr(Runtime.PySuper_Type, TypeOffset.tp_new); + } + else if (offset == TypeOffset.tp_getattro) + { + // PyObject_GenericGetAttr + return Util.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_getattro); + } + else if (offset == TypeOffset.tp_setattro) + { + // PyObject_GenericSetAttr + return Util.ReadIntPtr(Runtime.PyBaseObjectType, TypeOffset.tp_setattro); + } + + return Util.ReadIntPtr(Runtime.PyTypeType, offset); + } + } + + + static class SlotHelper + { + public static NewReference CreateObjectType() + { + using var globals = Runtime.PyDict_New(); + if (Runtime.PyDict_SetItemString(globals.Borrow(), "__builtins__", Runtime.PyEval_GetBuiltins()) != 0) + { + globals.Dispose(); + throw PythonException.ThrowLastAsClrException(); + } + const string code = "class A(object): pass"; + using var resRef = Runtime.PyRun_String(code, RunFlagType.File, globals.Borrow(), globals.Borrow()); + if (resRef.IsNull()) + { + globals.Dispose(); + throw PythonException.ThrowLastAsClrException(); + } + resRef.Dispose(); + BorrowedReference A = Runtime.PyDict_GetItemString(globals.Borrow(), "A"); + return new NewReference(A); + } + } +} diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 592eefd55..5dc01c442 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -137,6 +137,973 @@ internal static NewReference ToPython(IPythonDerivedType obj) return result; } + /// + /// Creates a new managed type derived from a base type with any virtual + /// methods overridden to call out to python if the associated python + /// object has overridden the method. + /// + internal static Type CreateDerivedType(string name, + Type baseType, + BorrowedReference py_dict, + string? namespaceStr, + string? assemblyName, + string moduleName = "Python.Runtime.Dynamic.dll") + { + // TODO: clean up + if (null != namespaceStr) + { + name = namespaceStr + "." + name; + } + + if (null == assemblyName) + { + assemblyName = "Python.Runtime.Dynamic"; + } + + ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); + + Type baseClass = baseType; + var interfaces = new List { typeof(IPythonDerivedType) }; + + // if the base type is an interface then use System.Object as the base class + // and add the base type to the list of interfaces this new class will implement. + if (baseType.IsInterface) + { + interfaces.Add(baseType); + baseClass = typeof(object); + } + + TypeBuilder typeBuilder = moduleBuilder.DefineType(name, + TypeAttributes.Public | TypeAttributes.Class, + baseClass, + interfaces.ToArray()); + + // add a field for storing the python object pointer + // FIXME: fb not used + FieldBuilder fb = typeBuilder.DefineField(PyObjName, +#pragma warning disable CS0618 // Type or member is obsolete. OK for internal use. + typeof(UnsafeReferenceWithRun), +#pragma warning restore CS0618 // Type or member is obsolete + FieldAttributes.Private); + + // override any constructors + ConstructorInfo[] constructors = baseClass.GetConstructors(); + foreach (ConstructorInfo ctor in constructors) + { + AddConstructor(ctor, baseType, typeBuilder); + } + + // Override any properties explicitly overridden in python + var pyProperties = new HashSet(); + if (py_dict != null && Runtime.PyDict_Check(py_dict)) + { + using var dict = new PyDict(py_dict); + using var keys = dict.Keys(); + foreach (PyObject pyKey in keys) + { + using var value = dict[pyKey]; + if (value.HasAttr("_clr_property_type_")) + { + string propertyName = pyKey.ToString()!; + pyProperties.Add(propertyName); + + // Add the property to the type + AddPythonProperty(propertyName, value, typeBuilder); + } + pyKey.Dispose(); + } + } + + // override any virtual methods not already overridden by the properties above + MethodInfo[] methods = baseType.GetMethods(); + var virtualMethods = new HashSet(); + foreach (MethodInfo method in methods) + { + if (!method.Attributes.HasFlag(MethodAttributes.Virtual) | + method.Attributes.HasFlag(MethodAttributes.Final)) + { + continue; + } + + // skip if this property has already been overridden + if ((method.Name.StartsWith("get_") || method.Name.StartsWith("set_")) + && pyProperties.Contains(method.Name.Substring(4))) + { + continue; + } + + // keep track of the virtual methods redirected to the python instance + virtualMethods.Add(method.Name); + + // override the virtual method to call out to the python method, if there is one. + AddVirtualMethod(method, baseType, typeBuilder); + } + + // Add any additional methods and properties explicitly exposed from Python. + if (py_dict != null && Runtime.PyDict_Check(py_dict)) + { + using var dict = new PyDict(py_dict); + using var keys = dict.Keys(); + foreach (PyObject pyKey in keys) + { + using var value = dict[pyKey]; + if (value.HasAttr("_clr_return_type_") && value.HasAttr("_clr_arg_types_")) + { + string methodName = pyKey.ToString()!; + + // if this method has already been redirected to the python method skip it + if (virtualMethods.Contains(methodName)) + { + continue; + } + + // Add the method to the type + AddPythonMethod(methodName, value, typeBuilder); + } + pyKey.Dispose(); + } + } + + // add the destructor so the python object created in the constructor gets destroyed + MethodBuilder methodBuilder = typeBuilder.DefineMethod("Finalize", + MethodAttributes.Family | + MethodAttributes.Virtual | + MethodAttributes.HideBySig, + CallingConventions.Standard, + typeof(void), + Type.EmptyTypes); + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); +#pragma warning disable CS0618 // PythonDerivedType is for internal use only + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(PyFinalize))); +#pragma warning restore CS0618 // PythonDerivedType is for internal use only + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Call, baseClass.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance)); + il.Emit(OpCodes.Ret); + + Type type = typeBuilder.CreateType(); + + // scan the assembly so the newly added class can be imported + Assembly assembly = Assembly.GetAssembly(type); + AssemblyManager.ScanAssembly(assembly); + + // FIXME: assemblyBuilder not used + AssemblyBuilder assemblyBuilder = assemblyBuilders[assemblyName]; + + return type; + } + + /// + /// Add a constructor override that calls the python ctor after calling the base type constructor. + /// + /// constructor to be called before calling the python ctor + /// Python callable object + /// TypeBuilder for the new type the ctor is to be added to + private static void AddConstructor(ConstructorInfo ctor, Type baseType, TypeBuilder typeBuilder) + { + ParameterInfo[] parameters = ctor.GetParameters(); + Type[] parameterTypes = (from param in parameters select param.ParameterType).ToArray(); + + // create a method for calling the original constructor + string baseCtorName = "_" + baseType.Name + "__cinit__"; + MethodBuilder methodBuilder = typeBuilder.DefineMethod(baseCtorName, + MethodAttributes.Public | + MethodAttributes.Final | + MethodAttributes.HideBySig, + typeof(void), + parameterTypes); + + // emit the assembly for calling the original method using call instead of callvirt + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + for (var i = 0; i < parameters.Length; ++i) + { + il.Emit(OpCodes.Ldarg, i + 1); + } + il.Emit(OpCodes.Call, ctor); + il.Emit(OpCodes.Ret); + + // override the original method with a new one that dispatches to python + ConstructorBuilder cb = typeBuilder.DefineConstructor(MethodAttributes.Public | + MethodAttributes.ReuseSlot | + MethodAttributes.HideBySig, + ctor.CallingConvention, + parameterTypes); + il = cb.GetILGenerator(); + il.DeclareLocal(typeof(object[])); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, baseCtorName); + il.Emit(OpCodes.Ldc_I4, parameters.Length); + il.Emit(OpCodes.Newarr, typeof(object)); + il.Emit(OpCodes.Stloc_0); + for (var i = 0; i < parameters.Length; ++i) + { + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, i); + il.Emit(OpCodes.Ldarg, i + 1); + if (parameterTypes[i].IsValueType) + { + il.Emit(OpCodes.Box, parameterTypes[i]); + } + il.Emit(OpCodes.Stelem, typeof(object)); + } + il.Emit(OpCodes.Ldloc_0); +#pragma warning disable CS0618 // PythonDerivedType is for internal use only + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(InvokeCtor))); +#pragma warning restore CS0618 // PythonDerivedType is for internal use only + il.Emit(OpCodes.Ret); + } + + /// + /// Add a virtual method override that checks for an override on the python instance + /// and calls it, otherwise fall back to the base class method. + /// + /// virtual method to be overridden + /// Python callable object + /// TypeBuilder for the new type the method is to be added to + private static void AddVirtualMethod(MethodInfo method, Type baseType, TypeBuilder typeBuilder) + { + ParameterInfo[] parameters = method.GetParameters(); + Type[] parameterTypes = (from param in parameters select param.ParameterType).ToArray(); + + // If the method isn't abstract create a method for calling the original method + string? baseMethodName = null; + if (!method.IsAbstract) + { + baseMethodName = "_" + baseType.Name + "__" + method.Name; + MethodBuilder baseMethodBuilder = typeBuilder.DefineMethod(baseMethodName, + MethodAttributes.Public | + MethodAttributes.Final | + MethodAttributes.HideBySig, + method.ReturnType, + parameterTypes); + + // emit the assembly for calling the original method using call instead of callvirt + ILGenerator baseIl = baseMethodBuilder.GetILGenerator(); + baseIl.Emit(OpCodes.Ldarg_0); + for (var i = 0; i < parameters.Length; ++i) + { + baseIl.Emit(OpCodes.Ldarg, i + 1); + } + baseIl.Emit(OpCodes.Call, method); + baseIl.Emit(OpCodes.Ret); + } + + // override the original method with a new one that dispatches to python + MethodBuilder methodBuilder = typeBuilder.DefineMethod(method.Name, + MethodAttributes.Public | + MethodAttributes.ReuseSlot | + MethodAttributes.Virtual | + MethodAttributes.HideBySig, + method.CallingConvention, + method.ReturnType, + parameterTypes); + ILGenerator il = methodBuilder.GetILGenerator(); + il.DeclareLocal(typeof(object[])); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, method.Name); + + // don't fall back to the base type's method if it's abstract + if (null != baseMethodName) + { + il.Emit(OpCodes.Ldstr, baseMethodName); + } + else + { + il.Emit(OpCodes.Ldnull); + } + + il.Emit(OpCodes.Ldc_I4, parameters.Length); + il.Emit(OpCodes.Newarr, typeof(object)); + il.Emit(OpCodes.Stloc_0); + for (var i = 0; i < parameters.Length; ++i) + { + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, i); + il.Emit(OpCodes.Ldarg, i + 1); + var type = parameterTypes[i]; + if (type.IsByRef) + { + type = type.GetElementType(); + il.Emit(OpCodes.Ldobj, type); + } + if (type.IsValueType) + { + il.Emit(OpCodes.Box, type); + } + il.Emit(OpCodes.Stelem, typeof(object)); + } + il.Emit(OpCodes.Ldloc_0); + + il.Emit(OpCodes.Ldtoken, method); + il.Emit(OpCodes.Ldtoken, method.DeclaringType); +#pragma warning disable CS0618 // PythonDerivedType is for internal use only + if (method.ReturnType == typeof(void)) + { + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(InvokeMethodVoid))); + } + else + { + il.Emit(OpCodes.Call, + typeof(PythonDerivedType).GetMethod(nameof(InvokeMethod)).MakeGenericMethod(method.ReturnType)); + } +#pragma warning restore CS0618 // PythonDerivedType is for internal use only + CodeGenerator.GenerateMarshalByRefsBack(il, parameterTypes); + il.Emit(OpCodes.Ret); + } + + /// + /// Python method may have the following function attributes set to control how they're exposed: + /// - _clr_return_type_ - method return type (required) + /// - _clr_arg_types_ - list of method argument types (required) + /// - _clr_method_name_ - method name, if different from the python method name (optional) + /// + /// Method name to add to the type + /// Python callable object + /// TypeBuilder for the new type the method/property is to be added to + private static void AddPythonMethod(string methodName, PyObject func, TypeBuilder typeBuilder) + { + const string methodNameAttribute = "_clr_method_name_"; + if (func.HasAttr(methodNameAttribute)) + { + using PyObject pyMethodName = func.GetAttr(methodNameAttribute); + methodName = pyMethodName.As() ?? throw new ArgumentNullException(methodNameAttribute); + } + + using var pyReturnType = func.GetAttr("_clr_return_type_"); + using var pyArgTypes = func.GetAttr("_clr_arg_types_"); + using var pyArgTypesIter = PyIter.GetIter(pyArgTypes); + var returnType = pyReturnType.AsManagedObject(typeof(Type)) as Type; + if (returnType == null) + { + returnType = typeof(void); + } + + var argTypes = new List(); + foreach (PyObject pyArgType in pyArgTypesIter) + { + var argType = pyArgType.AsManagedObject(typeof(Type)) as Type; + if (argType == null) + { + throw new ArgumentException("_clr_arg_types_ must be a list or tuple of CLR types"); + } + argTypes.Add(argType); + pyArgType.Dispose(); + } + + // add the method to call back into python + MethodAttributes methodAttribs = MethodAttributes.Public | + MethodAttributes.Virtual | + MethodAttributes.ReuseSlot | + MethodAttributes.HideBySig; + + MethodBuilder methodBuilder = typeBuilder.DefineMethod(methodName, + methodAttribs, + returnType, + argTypes.ToArray()); + + ILGenerator il = methodBuilder.GetILGenerator(); + + il.DeclareLocal(typeof(object[])); + il.DeclareLocal(typeof(RuntimeMethodHandle)); + il.DeclareLocal(typeof(RuntimeTypeHandle)); + + // this + il.Emit(OpCodes.Ldarg_0); + + // Python method to call + il.Emit(OpCodes.Ldstr, methodName); + + // original method name + il.Emit(OpCodes.Ldnull); // don't fall back to the base type's method + + // create args array + il.Emit(OpCodes.Ldc_I4, argTypes.Count); + il.Emit(OpCodes.Newarr, typeof(object)); + il.Emit(OpCodes.Stloc_0); + + // fill args array + for (var i = 0; i < argTypes.Count; ++i) + { + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, i); + il.Emit(OpCodes.Ldarg, i + 1); + var type = argTypes[i]; + if (type.IsByRef) + { + type = type.GetElementType(); + il.Emit(OpCodes.Ldobj, type); + } + if (type.IsValueType) + { + il.Emit(OpCodes.Box, type); + } + il.Emit(OpCodes.Stelem, typeof(object)); + } + + // args array + il.Emit(OpCodes.Ldloc_0); + + // method handle for the base method is null + il.Emit(OpCodes.Ldloca_S, 1); + il.Emit(OpCodes.Initobj, typeof(RuntimeMethodHandle)); + il.Emit(OpCodes.Ldloc_1); + + // type handle is also not required + il.Emit(OpCodes.Ldloca_S, 2); + il.Emit(OpCodes.Initobj, typeof(RuntimeTypeHandle)); + il.Emit(OpCodes.Ldloc_2); +#pragma warning disable CS0618 // PythonDerivedType is for internal use only + + // invoke the method + if (returnType == typeof(void)) + { + il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(InvokeMethodVoid))); + } + else + { + il.Emit(OpCodes.Call, + typeof(PythonDerivedType).GetMethod(nameof(InvokeMethod)).MakeGenericMethod(returnType)); + } + + CodeGenerator.GenerateMarshalByRefsBack(il, argTypes); + +#pragma warning restore CS0618 // PythonDerivedType is for internal use only + il.Emit(OpCodes.Ret); + } + + /// + /// Python properties may have the following function attributes set to control how they're exposed: + /// - _clr_property_type_ - property type (required) + /// + /// Property name to add to the type + /// Python property object + /// TypeBuilder for the new type the method/property is to be added to + private static void AddPythonProperty(string propertyName, PyObject func, TypeBuilder typeBuilder) + { + // add the method to call back into python + MethodAttributes methodAttribs = MethodAttributes.Public | + MethodAttributes.Virtual | + MethodAttributes.ReuseSlot | + MethodAttributes.HideBySig | + MethodAttributes.SpecialName; + + using var pyPropertyType = func.GetAttr("_clr_property_type_"); + var propertyType = pyPropertyType.AsManagedObject(typeof(Type)) as Type; + if (propertyType == null) + { + throw new ArgumentException("_clr_property_type must be a CLR type"); + } + + PropertyBuilder propertyBuilder = typeBuilder.DefineProperty(propertyName, + PropertyAttributes.None, + propertyType, + null); + + if (func.HasAttr("fget")) + { + using var pyfget = func.GetAttr("fget"); + if (pyfget.IsTrue()) + { + MethodBuilder methodBuilder = typeBuilder.DefineMethod("get_" + propertyName, + methodAttribs, + propertyType, + null); + + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, propertyName); +#pragma warning disable CS0618 // PythonDerivedType is for internal use only + il.Emit(OpCodes.Call, + typeof(PythonDerivedType).GetMethod("InvokeGetProperty").MakeGenericMethod(propertyType)); +#pragma warning restore CS0618 // PythonDerivedType is for internal use only + il.Emit(OpCodes.Ret); + + propertyBuilder.SetGetMethod(methodBuilder); + } + } + + if (func.HasAttr("fset")) + { + using var pyset = func.GetAttr("fset"); + if (pyset.IsTrue()) + { + MethodBuilder methodBuilder = typeBuilder.DefineMethod("set_" + propertyName, + methodAttribs, + null, + new[] { propertyType }); + + ILGenerator il = methodBuilder.GetILGenerator(); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldstr, propertyName); + il.Emit(OpCodes.Ldarg_1); +#pragma warning disable CS0618 // PythonDerivedType is for internal use only + il.Emit(OpCodes.Call, + typeof(PythonDerivedType).GetMethod("InvokeSetProperty").MakeGenericMethod(propertyType)); +#pragma warning restore CS0618 // PythonDerivedType is for internal use only + il.Emit(OpCodes.Ret); + + propertyBuilder.SetSetMethod(methodBuilder); + } + } + } + + private static ModuleBuilder GetModuleBuilder(string assemblyName, string moduleName) + { + // find or create a dynamic assembly and module + AppDomain domain = AppDomain.CurrentDomain; + ModuleBuilder moduleBuilder; + + if (moduleBuilders.ContainsKey(Tuple.Create(assemblyName, moduleName))) + { + moduleBuilder = moduleBuilders[Tuple.Create(assemblyName, moduleName)]; + } + else + { + AssemblyBuilder assemblyBuilder; + if (assemblyBuilders.ContainsKey(assemblyName)) + { + assemblyBuilder = assemblyBuilders[assemblyName]; + } + else + { + assemblyBuilder = domain.DefineDynamicAssembly(new AssemblyName(assemblyName), + AssemblyBuilderAccess.Run); + assemblyBuilders[assemblyName] = assemblyBuilder; + } + + moduleBuilder = assemblyBuilder.DefineDynamicModule(moduleName); + moduleBuilders[Tuple.Create(assemblyName, moduleName)] = moduleBuilder; + } + + return moduleBuilder; + } + } + + /// + /// PythonDerivedType contains static methods used by the dynamically created + /// derived type that allow it to call back into python from overridden virtual + /// methods, and also handle the construction and destruction of the python + /// object. + /// + /// + /// This has to be public as it's called from methods on dynamically built classes + /// potentially in other assemblies. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete(Util.InternalUseOnly)] + public class PythonDerivedType + { + internal const string PyObjName = "__pyobj__"; + internal const BindingFlags PyObjFlags = BindingFlags.Instance | BindingFlags.NonPublic; + + /// + /// This is the implementation of the overridden methods in the derived + /// type. It looks for a python method with the same name as the method + /// on the managed base class and if it exists and isn't the managed + /// method binding (i.e. it has been overridden in the derived python + /// class) it calls it, otherwise it calls the base method. + /// + public static T? InvokeMethod(IPythonDerivedType obj, string methodName, string origMethodName, + object[] args, RuntimeMethodHandle methodHandle, RuntimeTypeHandle declaringTypeHandle) + { + var self = GetPyObj(obj); + + if (null != self.Ref) + { + var disposeList = new List(); + PyGILState gs = Runtime.PyGILState_Ensure(); + try + { + using var pyself = new PyObject(self.CheckRun()); + using PyObject method = pyself.GetAttr(methodName, Runtime.None); + if (method.Reference != Runtime.PyNone) + { + // if the method hasn't been overridden then it will be a managed object + ManagedType? managedMethod = ManagedType.GetManagedObject(method.Reference); + if (null == managedMethod) + { + var pyargs = new PyObject[args.Length]; + for (var i = 0; i < args.Length; ++i) + { + pyargs[i] = Converter.ToPythonImplicit(args[i]).MoveToPyObject(); + disposeList.Add(pyargs[i]); + } + + PyObject py_result = method.Invoke(pyargs); + var clrMethod = methodHandle != default + ? MethodBase.GetMethodFromHandle(methodHandle, declaringTypeHandle) + : null; + PyTuple? result_tuple = MarshalByRefsBack(args, clrMethod, py_result, outsOffset: 1); + return result_tuple is not null + ? result_tuple[0].As() + : py_result.As(); + } + } + } + finally + { + foreach (PyObject x in disposeList) + { + x?.Dispose(); + } + Runtime.PyGILState_Release(gs); + } + } + + if (origMethodName == null) + { + throw new NotImplementedException("Python object does not have a '" + methodName + "' method"); + } + + return (T)obj.GetType().InvokeMember(origMethodName, + BindingFlags.InvokeMethod, + null, + obj, + args); + } + + public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, string origMethodName, + object?[] args, RuntimeMethodHandle methodHandle, RuntimeTypeHandle declaringTypeHandle) + { + var self = GetPyObj(obj); + if (null != self.Ref) + { + var disposeList = new List(); + PyGILState gs = Runtime.PyGILState_Ensure(); + try + { + using var pyself = new PyObject(self.CheckRun()); + using PyObject method = pyself.GetAttr(methodName, Runtime.None); + if (method.Reference != Runtime.None) + { + // if the method hasn't been overridden then it will be a managed object + ManagedType? managedMethod = ManagedType.GetManagedObject(method); + if (null == managedMethod) + { + var pyargs = new PyObject[args.Length]; + for (var i = 0; i < args.Length; ++i) + { + pyargs[i] = Converter.ToPythonImplicit(args[i]).MoveToPyObject(); + disposeList.Add(pyargs[i]); + } + + PyObject py_result = method.Invoke(pyargs); + var clrMethod = methodHandle != default + ? MethodBase.GetMethodFromHandle(methodHandle, declaringTypeHandle) + : null; + MarshalByRefsBack(args, clrMethod, py_result, outsOffset: 0); + return; + } + } + } + finally + { + foreach (PyObject x in disposeList) + { + x?.Dispose(); + } + Runtime.PyGILState_Release(gs); + } + } + + if (origMethodName == null) + { + throw new NotImplementedException($"Python object does not have a '{methodName}' method"); + } + + obj.GetType().InvokeMember(origMethodName, + BindingFlags.InvokeMethod, + null, + obj, + args); + } + + /// + /// If the method has byref arguments, reinterprets Python return value + /// as a tuple of new values for those arguments, and updates corresponding + /// elements of array. + /// + private static PyTuple? MarshalByRefsBack(object?[] args, MethodBase? method, PyObject pyResult, int outsOffset) + { + if (method is null) return null; + + var parameters = method.GetParameters(); + PyTuple? outs = null; + int byrefIndex = 0; + for (int i = 0; i < parameters.Length; ++i) + { + Type type = parameters[i].ParameterType; + if (!type.IsByRef) + { + continue; + } + + type = type.GetElementType(); + + if (outs is null) + { + outs = new PyTuple(pyResult); + pyResult.Dispose(); + } + + args[i] = outs[byrefIndex + outsOffset].AsManagedObject(type); + byrefIndex++; + } + if (byrefIndex > 0 && outs!.Length() > byrefIndex + outsOffset) + throw new ArgumentException("Too many output parameters"); + + return outs; + } + + public static T? InvokeGetProperty(IPythonDerivedType obj, string propertyName) + { + var self = GetPyObj(obj); + + if (null == self.Ref) + { + throw new NullReferenceException("Instance must be specified when getting a property"); + } + + PyGILState gs = Runtime.PyGILState_Ensure(); + try + { + using var pyself = new PyObject(self.CheckRun()); + using var pyvalue = pyself.GetAttr(propertyName); + return pyvalue.As(); + } + finally + { + Runtime.PyGILState_Release(gs); + } + } + + public static void InvokeSetProperty(IPythonDerivedType obj, string propertyName, T value) + { + var self = GetPyObj(obj); + + if (null == self.Ref) + { + throw new NullReferenceException("Instance must be specified when setting a property"); + } + + PyGILState gs = Runtime.PyGILState_Ensure(); + try + { + using var pyself = new PyObject(self.CheckRun()); + using var pyvalue = Converter.ToPythonImplicit(value).MoveToPyObject(); + pyself.SetAttr(propertyName, pyvalue); + } + finally + { + Runtime.PyGILState_Release(gs); + } + } + + public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, object[] args) + { + var selfRef = GetPyObj(obj); + if (selfRef.Ref == null) + { + // this might happen when the object is created from .NET + using var _ = Py.GIL(); + // In the end we decrement the python object's reference count. + // This doesn't actually destroy the object, it just sets the reference to this object + // to be a weak reference and it will be destroyed when the C# object is destroyed. + using var self = CLRObject.GetReference(obj, obj.GetType()); + SetPyObj(obj, self.Borrow()); + } + + // call the base constructor + obj.GetType().InvokeMember(origCtorName, + BindingFlags.InvokeMethod, + null, + obj, + args); + } + + public static void PyFinalize(IPythonDerivedType obj) + { + // the C# object is being destroyed which must mean there are no more + // references to the Python object as well + var self = GetPyObj(obj); + Finalizer.Instance.AddDerivedFinalizedObject(ref self.RawObj, self.Run); + } + + internal static void Finalize(IntPtr derived) + { + var @ref = NewReference.DangerousFromPointer(derived); + + ClassBase.tp_clear(@ref.Borrow()); + + var type = Runtime.PyObject_TYPE(@ref.Borrow()); + + if (!Runtime.HostedInPython || Runtime.TypeManagerInitialized) + { + // rare case when it's needed + // matches correspdonging PyObject_GC_UnTrack + // in ClassDerivedObject.tp_dealloc + Runtime.PyObject_GC_Del(@ref.Steal()); + + // must decref our type + Runtime.XDecref(StolenReference.DangerousFromPointer(type.DangerousGetAddress())); + } + } + + internal static FieldInfo? GetPyObjField(Type type) => type.GetField(PyObjName, PyObjFlags); + + internal static UnsafeReferenceWithRun GetPyObj(IPythonDerivedType obj) + { + FieldInfo fi = GetPyObjField(obj.GetType())!; + return (UnsafeReferenceWithRun)fi.GetValue(obj); + } + + internal static void SetPyObj(IPythonDerivedType obj, BorrowedReference pyObj) + { + FieldInfo fi = GetPyObjField(obj.GetType())!; + fi.SetValue(obj, new UnsafeReferenceWithRun(pyObj)); + } + } +} +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +#pragma warning disable CS0618 // Type or member is obsolete. OK for internal use +using static Python.Runtime.PythonDerivedType; +#pragma warning restore CS0618 // Type or member is obsolete + +namespace Python.Runtime +{ + /// + /// Managed class that provides the implementation for reflected types. + /// Managed classes and value types are represented in Python by actual + /// Python type objects. Each of those type objects is associated with + /// an instance of ClassObject, which provides its implementation. + /// + /// + /// interface used to identify which C# types were dynamically created as python subclasses + /// + public interface IPythonDerivedType + { + } + + [Serializable] + internal class ClassDerivedObject : ClassObject + { + private static Dictionary assemblyBuilders; + private static Dictionary, ModuleBuilder> moduleBuilders; + + static ClassDerivedObject() + { + assemblyBuilders = new Dictionary(); + moduleBuilders = new Dictionary, ModuleBuilder>(); + } + + public static void Reset() + { + assemblyBuilders = new Dictionary(); + moduleBuilders = new Dictionary, ModuleBuilder>(); + } + + internal ClassDerivedObject(Type tp) : base(tp) + { + } + + protected override NewReference NewObjectToPython(object obj, BorrowedReference tp) + { + var self = base.NewObjectToPython(obj, tp); + + SetPyObj((IPythonDerivedType)obj, self.Borrow()); + + // Decrement the python object's reference count. + // This doesn't actually destroy the object, it just sets the reference to this object + // to be a weak reference and it will be destroyed when the C# object is destroyed. + Runtime.XDecref(self.Steal()); + + return Converter.ToPython(obj, type.Value); + } + + protected override void SetTypeNewSlot(BorrowedReference pyType, SlotsHolder slotsHolder) + { + // Python derived types rely on base tp_new and overridden __init__ + } + + public new static void tp_dealloc(NewReference ob) + { + var self = (CLRObject?)GetManagedObject(ob.Borrow()); + + // don't let the python GC destroy this object + Runtime.PyObject_GC_UnTrack(ob.Borrow()); + + // self may be null after Shutdown begun + if (self is not null) + { + // The python should now have a ref count of 0, but we don't actually want to + // deallocate the object until the C# object that references it is destroyed. + // So we don't call PyObject_GC_Del here and instead we set the python + // reference to a weak reference so that the C# object can be collected. + GCHandle oldHandle = GetGCHandle(ob.Borrow()); + GCHandle gc = GCHandle.Alloc(self, GCHandleType.Weak); + SetGCHandle(ob.Borrow(), gc); + oldHandle.Free(); + } + } + + /// + /// No-op clear. Real cleanup happens in + /// + public new static int tp_clear(BorrowedReference ob) => 0; + + /// + /// Called from Converter.ToPython for types that are python subclasses of managed types. + /// The referenced python object is returned instead of a new wrapper. + /// + internal static NewReference ToPython(IPythonDerivedType obj) + { + // derived types have a __pyobj__ field that gets set to the python + // object in the overridden constructor + BorrowedReference self; + try + { + self = GetPyObj(obj).CheckRun(); + } + catch (RuntimeShutdownException e) + { + Exceptions.SetError(e); + return default; + } + + var result = new NewReference(self); + + // when the C# constructor creates the python object it starts as a weak + // reference with a reference count of 0. Now we're passing this object + // to Python the reference count needs to be incremented and the reference + // needs to be replaced with a strong reference to stop the C# object being + // collected while Python still has a reference to it. + if (Runtime.Refcount(self) == 1) + { + Runtime._Py_NewReference(self); + GCHandle weak = GetGCHandle(self); + var clrObject = GetManagedObject(self); + GCHandle gc = GCHandle.Alloc(clrObject, GCHandleType.Normal); + SetGCHandle(self, gc); + weak.Free(); + + // now the object has a python reference it's safe for the python GC to track it + Runtime.PyObject_GC_Track(self); + } + + return result; + } + /// /// Creates a new managed type derived from a base type with any virtual /// methods overridden to call out to python if the associated python diff --git a/src/runtime/Types/ManagedType.cs b/src/runtime/Types/ManagedType.cs index 97a19497c..5de133ecb 100644 --- a/src/runtime/Types/ManagedType.cs +++ b/src/runtime/Types/ManagedType.cs @@ -1,267 +1,267 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Runtime.InteropServices; -using System.Linq; - -namespace Python.Runtime -{ - /// - /// Common base class for all objects that are implemented in managed - /// code. It defines the common fields that associate CLR and Python - /// objects and common utilities to convert between those identities. - /// - [Serializable] - internal abstract class ManagedType - { - /// - /// Given a Python object, return the associated managed object or null. - /// - internal static ManagedType? GetManagedObject(BorrowedReference ob) - { - if (ob != null) - { - BorrowedReference tp = Runtime.PyObject_TYPE(ob); - var flags = PyType.GetFlags(tp); - if ((flags & TypeFlags.HasClrInstance) != 0) - { - var gc = TryGetGCHandle(ob, tp); - return (ManagedType?)gc?.Target; - } - } - return null; - } - - internal static bool IsInstanceOfManagedType(BorrowedReference ob) - { - if (ob != null) - { - BorrowedReference tp = Runtime.PyObject_TYPE(ob); - if (tp == Runtime.PyTypeType || tp == Runtime.PyCLRMetaType) - { - tp = ob; - } - - return IsManagedType(tp); - } - return false; - } - - internal static bool IsManagedType(BorrowedReference type) - { - var flags = PyType.GetFlags(type); - return (flags & TypeFlags.HasClrInstance) != 0; - } - - internal static BorrowedReference GetUnmanagedBaseType(BorrowedReference managedType) - { - Debug.Assert(managedType != null && IsManagedType(managedType)); - do - { - managedType = PyType.GetBase(managedType); - Debug.Assert(managedType != null); - } while (IsManagedType(managedType)); - return managedType; - } - - internal unsafe static int PyVisit(BorrowedReference ob, IntPtr visit, IntPtr arg) - { - if (ob == null) - { - return 0; - } - var visitFunc = (delegate* unmanaged[Cdecl])(visit); - return visitFunc(ob, arg); - } - - internal static unsafe void DecrefTypeAndFree(StolenReference ob) - { - if (ob == null) throw new ArgumentNullException(nameof(ob)); - var borrowed = new BorrowedReference(ob.DangerousGetAddress()); - - var type = Runtime.PyObject_TYPE(borrowed); - - var freePtr = Util.ReadIntPtr(type, TypeOffset.tp_free); - Debug.Assert(freePtr != IntPtr.Zero); - var free = (delegate* unmanaged[Cdecl])freePtr; - free(ob); - - Runtime.XDecref(StolenReference.DangerousFromPointer(type.DangerousGetAddress())); - } - - internal static int CallClear(BorrowedReference ob) - => CallTypeClear(ob, Runtime.PyObject_TYPE(ob)); - - /// - /// Wrapper for calling tp_clear - /// - internal static unsafe int CallTypeClear(BorrowedReference ob, BorrowedReference tp) - { - if (ob == null) throw new ArgumentNullException(nameof(ob)); - if (tp == null) throw new ArgumentNullException(nameof(tp)); - - var clearPtr = Util.ReadIntPtr(tp, TypeOffset.tp_clear); - if (clearPtr == IntPtr.Zero) - { - return 0; - } - var clearFunc = (delegate* unmanaged[Cdecl])clearPtr; - return clearFunc(ob); - } - - internal Dictionary? Save(BorrowedReference ob) - { - return OnSave(ob); - } - - internal void Load(BorrowedReference ob, Dictionary? context) - { - OnLoad(ob, context); - } - - protected virtual Dictionary? OnSave(BorrowedReference ob) => null; - protected virtual void OnLoad(BorrowedReference ob, Dictionary? context) { } - - /// - /// Initializes given object, or returns false and sets Python error on failure - /// - public virtual bool Init(BorrowedReference obj, BorrowedReference args, BorrowedReference kw) - { - // this just calls obj.__init__(*args, **kw) - using var init = Runtime.PyObject_GetAttr(obj, PyIdentifier.__init__); - Runtime.PyErr_Clear(); - - if (!init.IsNull()) - { - using var result = Runtime.PyObject_Call(init.Borrow(), args, kw); - - if (result.IsNull()) - { - return false; - } - } - - return true; - } - - protected static void ClearObjectDict(BorrowedReference ob) - { - BorrowedReference type = Runtime.PyObject_TYPE(ob); - int instanceDictOffset = Util.ReadInt32(type, TypeOffset.tp_dictoffset); - // Debug.Assert(instanceDictOffset > 0); - if (instanceDictOffset > 0) - Runtime.Py_CLEAR(ob, instanceDictOffset); - } - - protected static BorrowedReference GetObjectDict(BorrowedReference ob) - { - BorrowedReference type = Runtime.PyObject_TYPE(ob); - int instanceDictOffset = Util.ReadInt32(type, TypeOffset.tp_dictoffset); - Debug.Assert(instanceDictOffset > 0); - return Util.ReadRef(ob, instanceDictOffset); - } - - protected static void SetObjectDict(BorrowedReference ob, StolenReference value) - { - if (value.Pointer == IntPtr.Zero) throw new ArgumentNullException(nameof(value)); - SetObjectDictNullable(ob, value.AnalyzerWorkaround()); - } - protected static void SetObjectDictNullable(BorrowedReference ob, StolenReference value) - { - BorrowedReference type = Runtime.PyObject_TYPE(ob); - int instanceDictOffset = Util.ReadInt32(type, TypeOffset.tp_dictoffset); - Debug.Assert(instanceDictOffset > 0); - Runtime.ReplaceReference(ob, instanceDictOffset, value.AnalyzerWorkaround()); - } - - internal static void GetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, out IntPtr handle) - { - Debug.Assert(reflectedClrObject != null); - Debug.Assert(IsManagedType(type) || IsManagedType(reflectedClrObject)); - Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); - - int gcHandleOffset = Util.ReadInt32(type, Offsets.tp_clr_inst_offset); - Debug.Assert(gcHandleOffset > 0); - - handle = Util.ReadIntPtr(reflectedClrObject, gcHandleOffset); - } - - internal static GCHandle? TryGetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) - { - GetGCHandle(reflectedClrObject, type, out IntPtr handle); - return handle == IntPtr.Zero ? null : (GCHandle)handle; - } - internal static GCHandle? TryGetGCHandle(BorrowedReference reflectedClrObject) - { - BorrowedReference reflectedType = Runtime.PyObject_TYPE(reflectedClrObject); - - return TryGetGCHandle(reflectedClrObject, reflectedType); - } - - internal static GCHandle GetGCHandle(BorrowedReference reflectedClrObject) - => TryGetGCHandle(reflectedClrObject) ?? throw new InvalidOperationException(); - internal static GCHandle GetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) - => TryGetGCHandle(reflectedClrObject, type) ?? throw new InvalidOperationException(); - - internal static void InitGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, GCHandle handle) - { - Debug.Assert(TryGetGCHandle(reflectedClrObject) == null); - - SetGCHandle(reflectedClrObject, type: type, handle); - } - internal static void InitGCHandle(BorrowedReference reflectedClrObject, GCHandle handle) - => InitGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject), handle); - - internal static void SetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, GCHandle newHandle) - { - Debug.Assert(type != null); - Debug.Assert(reflectedClrObject != null); - Debug.Assert(IsManagedType(type) || IsManagedType(reflectedClrObject)); - Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); - - int offset = Util.ReadInt32(type, Offsets.tp_clr_inst_offset); - Debug.Assert(offset > 0); - - Util.WriteIntPtr(reflectedClrObject, offset, (IntPtr)newHandle); - } - internal static void SetGCHandle(BorrowedReference reflectedClrObject, GCHandle newHandle) - => SetGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject), newHandle); - - internal static bool TryFreeGCHandle(BorrowedReference reflectedClrObject) - => TryFreeGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject)); - - internal static bool TryFreeGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) - { - Debug.Assert(type != null); - Debug.Assert(reflectedClrObject != null); - Debug.Assert(IsManagedType(type) || IsManagedType(reflectedClrObject)); - Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); - - int offset = Util.ReadInt32(type, Offsets.tp_clr_inst_offset); - Debug.Assert(offset > 0); - - IntPtr raw = Util.ReadIntPtr(reflectedClrObject, offset); - if (raw == IntPtr.Zero) return false; - - var handle = (GCHandle)raw; - handle.Free(); - - Util.WriteIntPtr(reflectedClrObject, offset, IntPtr.Zero); - return true; - } - - internal static class Offsets - { - static Offsets() - { - int pyTypeSize = Util.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize); - if (pyTypeSize < 0) throw new InvalidOperationException(); - - tp_clr_inst_offset = pyTypeSize; - tp_clr_inst = tp_clr_inst_offset + IntPtr.Size; - } - public static int tp_clr_inst_offset { get; } - public static int tp_clr_inst { get; } - } - } -} +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Linq; + +namespace Python.Runtime +{ + /// + /// Common base class for all objects that are implemented in managed + /// code. It defines the common fields that associate CLR and Python + /// objects and common utilities to convert between those identities. + /// + [Serializable] + internal abstract class ManagedType + { + /// + /// Given a Python object, return the associated managed object or null. + /// + internal static ManagedType? GetManagedObject(BorrowedReference ob) + { + if (ob != null) + { + BorrowedReference tp = Runtime.PyObject_TYPE(ob); + var flags = PyType.GetFlags(tp); + if ((flags & TypeFlags.HasClrInstance) != 0) + { + var gc = TryGetGCHandle(ob, tp); + return (ManagedType?)gc?.Target; + } + } + return null; + } + + internal static bool IsInstanceOfManagedType(BorrowedReference ob) + { + if (ob != null) + { + BorrowedReference tp = Runtime.PyObject_TYPE(ob); + if (tp == Runtime.PyTypeType || tp == Runtime.PyCLRMetaType) + { + tp = ob; + } + + return IsManagedType(tp); + } + return false; + } + + internal static bool IsManagedType(BorrowedReference type) + { + var flags = PyType.GetFlags(type); + return (flags & TypeFlags.HasClrInstance) != 0; + } + + internal static BorrowedReference GetUnmanagedBaseType(BorrowedReference managedType) + { + Debug.Assert(managedType != null && IsManagedType(managedType)); + do + { + managedType = PyType.GetBase(managedType); + Debug.Assert(managedType != null); + } while (IsManagedType(managedType)); + return managedType; + } + + internal unsafe static int PyVisit(BorrowedReference ob, IntPtr visit, IntPtr arg) + { + if (ob == null) + { + return 0; + } + var visitFunc = (delegate* unmanaged[Cdecl])(visit); + return visitFunc(ob, arg); + } + + internal static unsafe void DecrefTypeAndFree(StolenReference ob) + { + if (ob == null) throw new ArgumentNullException(nameof(ob)); + var borrowed = new BorrowedReference(ob.DangerousGetAddress()); + + var type = Runtime.PyObject_TYPE(borrowed); + + var freePtr = Util.ReadIntPtr(type, TypeOffset.tp_free); + Debug.Assert(freePtr != IntPtr.Zero); + var free = (delegate* unmanaged[Cdecl])freePtr; + free(ob); + + Runtime.XDecref(StolenReference.DangerousFromPointer(type.DangerousGetAddress())); + } + + internal static int CallClear(BorrowedReference ob) + => CallTypeClear(ob, Runtime.PyObject_TYPE(ob)); + + /// + /// Wrapper for calling tp_clear + /// + internal static unsafe int CallTypeClear(BorrowedReference ob, BorrowedReference tp) + { + if (ob == null) throw new ArgumentNullException(nameof(ob)); + if (tp == null) throw new ArgumentNullException(nameof(tp)); + + var clearPtr = Util.ReadIntPtr(tp, TypeOffset.tp_clear); + if (clearPtr == IntPtr.Zero) + { + return 0; + } + var clearFunc = (delegate* unmanaged[Cdecl])clearPtr; + return clearFunc(ob); + } + + internal Dictionary? Save(BorrowedReference ob) + { + return OnSave(ob); + } + + internal void Load(BorrowedReference ob, Dictionary? context) + { + OnLoad(ob, context); + } + + protected virtual Dictionary? OnSave(BorrowedReference ob) => null; + protected virtual void OnLoad(BorrowedReference ob, Dictionary? context) { } + + /// + /// Initializes given object, or returns false and sets Python error on failure + /// + public virtual bool Init(BorrowedReference obj, BorrowedReference args, BorrowedReference kw) + { + // this just calls obj.__init__(*args, **kw) + using var init = Runtime.PyObject_GetAttr(obj, PyIdentifier.__init__); + Runtime.PyErr_Clear(); + + if (!init.IsNull()) + { + using var result = Runtime.PyObject_Call(init.Borrow(), args, kw); + + if (result.IsNull()) + { + return false; + } + } + + return true; + } + + protected static void ClearObjectDict(BorrowedReference ob) + { + BorrowedReference type = Runtime.PyObject_TYPE(ob); + int instanceDictOffset = Util.ReadInt32(type, TypeOffset.tp_dictoffset); + // Debug.Assert(instanceDictOffset > 0); + if (instanceDictOffset > 0) + Runtime.Py_CLEAR(ob, instanceDictOffset); + } + + protected static BorrowedReference GetObjectDict(BorrowedReference ob) + { + BorrowedReference type = Runtime.PyObject_TYPE(ob); + int instanceDictOffset = Util.ReadInt32(type, TypeOffset.tp_dictoffset); + Debug.Assert(instanceDictOffset > 0); + return Util.ReadRef(ob, instanceDictOffset); + } + + protected static void SetObjectDict(BorrowedReference ob, StolenReference value) + { + if (value.Pointer == IntPtr.Zero) throw new ArgumentNullException(nameof(value)); + SetObjectDictNullable(ob, value.AnalyzerWorkaround()); + } + protected static void SetObjectDictNullable(BorrowedReference ob, StolenReference value) + { + BorrowedReference type = Runtime.PyObject_TYPE(ob); + int instanceDictOffset = Util.ReadInt32(type, TypeOffset.tp_dictoffset); + Debug.Assert(instanceDictOffset > 0); + Runtime.ReplaceReference(ob, instanceDictOffset, value.AnalyzerWorkaround()); + } + + internal static void GetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, out IntPtr handle) + { + Debug.Assert(reflectedClrObject != null); + Debug.Assert(IsManagedType(type) || IsManagedType(reflectedClrObject)); + Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); + + int gcHandleOffset = Util.ReadInt32(type, Offsets.tp_clr_inst_offset); + Debug.Assert(gcHandleOffset > 0); + + handle = Util.ReadIntPtr(reflectedClrObject, gcHandleOffset); + } + + internal static GCHandle? TryGetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) + { + GetGCHandle(reflectedClrObject, type, out IntPtr handle); + return handle == IntPtr.Zero ? null : (GCHandle)handle; + } + internal static GCHandle? TryGetGCHandle(BorrowedReference reflectedClrObject) + { + BorrowedReference reflectedType = Runtime.PyObject_TYPE(reflectedClrObject); + + return TryGetGCHandle(reflectedClrObject, reflectedType); + } + + internal static GCHandle GetGCHandle(BorrowedReference reflectedClrObject) + => TryGetGCHandle(reflectedClrObject) ?? throw new InvalidOperationException(); + internal static GCHandle GetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) + => TryGetGCHandle(reflectedClrObject, type) ?? throw new InvalidOperationException(); + + internal static void InitGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, GCHandle handle) + { + Debug.Assert(TryGetGCHandle(reflectedClrObject) == null); + + SetGCHandle(reflectedClrObject, type: type, handle); + } + internal static void InitGCHandle(BorrowedReference reflectedClrObject, GCHandle handle) + => InitGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject), handle); + + internal static void SetGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type, GCHandle newHandle) + { + Debug.Assert(type != null); + Debug.Assert(reflectedClrObject != null); + Debug.Assert(IsManagedType(type) || IsManagedType(reflectedClrObject)); + Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); + + int offset = Util.ReadInt32(type, Offsets.tp_clr_inst_offset); + Debug.Assert(offset > 0); + + Util.WriteIntPtr(reflectedClrObject, offset, (IntPtr)newHandle); + } + internal static void SetGCHandle(BorrowedReference reflectedClrObject, GCHandle newHandle) + => SetGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject), newHandle); + + internal static bool TryFreeGCHandle(BorrowedReference reflectedClrObject) + => TryFreeGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject)); + + internal static bool TryFreeGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) + { + Debug.Assert(type != null); + Debug.Assert(reflectedClrObject != null); + Debug.Assert(IsManagedType(type) || IsManagedType(reflectedClrObject)); + Debug.Assert(Runtime.PyObject_TypeCheck(reflectedClrObject, type)); + + int offset = Util.ReadInt32(type, Offsets.tp_clr_inst_offset); + Debug.Assert(offset > 0); + + IntPtr raw = Util.ReadIntPtr(reflectedClrObject, offset); + if (raw == IntPtr.Zero) return false; + + var handle = (GCHandle)raw; + handle.Free(); + + Util.WriteIntPtr(reflectedClrObject, offset, IntPtr.Zero); + return true; + } + + internal static class Offsets + { + static Offsets() + { + int pyTypeSize = Util.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize); + if (pyTypeSize < 0) throw new InvalidOperationException(); + + tp_clr_inst_offset = pyTypeSize; + tp_clr_inst = tp_clr_inst_offset + IntPtr.Size; + } + public static int tp_clr_inst_offset { get; } + public static int tp_clr_inst { get; } + } + } +} diff --git a/src/runtime/Util/CodeGenerator.cs b/src/runtime/Util/CodeGenerator.cs index 6e0859da0..30403ceb4 100644 --- a/src/runtime/Util/CodeGenerator.cs +++ b/src/runtime/Util/CodeGenerator.cs @@ -1,100 +1,100 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Reflection; -using System.Reflection.Emit; -using System.Threading; - -namespace Python.Runtime -{ - /// - /// Several places in the runtime generate code on the fly to support - /// dynamic functionality. The CodeGenerator class manages the dynamic - /// assembly used for code generation and provides utility methods for - /// certain repetitive tasks. - /// - internal class CodeGenerator - { - private readonly AssemblyBuilder aBuilder; - private readonly ModuleBuilder mBuilder; - - const string NamePrefix = "__Python_Runtime_Generated_"; - - internal CodeGenerator() - { - var aname = new AssemblyName { Name = GetUniqueAssemblyName(NamePrefix + "Assembly") }; - var aa = AssemblyBuilderAccess.Run; - - aBuilder = Thread.GetDomain().DefineDynamicAssembly(aname, aa); - mBuilder = aBuilder.DefineDynamicModule(NamePrefix + "Module"); - } - - /// - /// DefineType is a shortcut utility to get a new TypeBuilder. - /// - internal TypeBuilder DefineType(string name) - { - var attrs = TypeAttributes.Public; - return mBuilder.DefineType(name, attrs); - } - - /// - /// DefineType is a shortcut utility to get a new TypeBuilder. - /// - internal TypeBuilder DefineType(string name, Type basetype) - { - var attrs = TypeAttributes.Public; - return mBuilder.DefineType(name, attrs, basetype); - } - - /// - /// Generates code, that copies potentially modified objects in args array - /// back to the corresponding byref arguments - /// - internal static void GenerateMarshalByRefsBack(ILGenerator il, IReadOnlyList argTypes) - { - // assumes argument array is in loc_0 - for (int i = 0; i < argTypes.Count; ++i) - { - var type = argTypes[i]; - if (type.IsByRef) - { - type = type.GetElementType(); - - il.Emit(OpCodes.Ldarg, i + 1); // for stobj/stind later at the end - - il.Emit(OpCodes.Ldloc_0); - il.Emit(OpCodes.Ldc_I4, i); - il.Emit(OpCodes.Ldelem_Ref); - - if (type.IsValueType) - { - il.Emit(OpCodes.Unbox_Any, type); - il.Emit(OpCodes.Stobj, type); - } - else - { - il.Emit(OpCodes.Castclass, type); - il.Emit(OpCodes.Stind_Ref); - } - } - } - } - - static string GetUniqueAssemblyName(string name) - { - var taken = new HashSet(AppDomain.CurrentDomain - .GetAssemblies() - .Select(a => a.GetName().Name)); - for (int i = 0; i < int.MaxValue; i++) - { - string candidate = name + i.ToString(CultureInfo.InvariantCulture); - if (!taken.Contains(candidate)) - return candidate; - } - - throw new NotSupportedException("Too many assemblies"); - } - } -} +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Reflection; +using System.Reflection.Emit; +using System.Threading; + +namespace Python.Runtime +{ + /// + /// Several places in the runtime generate code on the fly to support + /// dynamic functionality. The CodeGenerator class manages the dynamic + /// assembly used for code generation and provides utility methods for + /// certain repetitive tasks. + /// + internal class CodeGenerator + { + private readonly AssemblyBuilder aBuilder; + private readonly ModuleBuilder mBuilder; + + const string NamePrefix = "__Python_Runtime_Generated_"; + + internal CodeGenerator() + { + var aname = new AssemblyName { Name = GetUniqueAssemblyName(NamePrefix + "Assembly") }; + var aa = AssemblyBuilderAccess.Run; + + aBuilder = Thread.GetDomain().DefineDynamicAssembly(aname, aa); + mBuilder = aBuilder.DefineDynamicModule(NamePrefix + "Module"); + } + + /// + /// DefineType is a shortcut utility to get a new TypeBuilder. + /// + internal TypeBuilder DefineType(string name) + { + var attrs = TypeAttributes.Public; + return mBuilder.DefineType(name, attrs); + } + + /// + /// DefineType is a shortcut utility to get a new TypeBuilder. + /// + internal TypeBuilder DefineType(string name, Type basetype) + { + var attrs = TypeAttributes.Public; + return mBuilder.DefineType(name, attrs, basetype); + } + + /// + /// Generates code, that copies potentially modified objects in args array + /// back to the corresponding byref arguments + /// + internal static void GenerateMarshalByRefsBack(ILGenerator il, IReadOnlyList argTypes) + { + // assumes argument array is in loc_0 + for (int i = 0; i < argTypes.Count; ++i) + { + var type = argTypes[i]; + if (type.IsByRef) + { + type = type.GetElementType(); + + il.Emit(OpCodes.Ldarg, i + 1); // for stobj/stind later at the end + + il.Emit(OpCodes.Ldloc_0); + il.Emit(OpCodes.Ldc_I4, i); + il.Emit(OpCodes.Ldelem_Ref); + + if (type.IsValueType) + { + il.Emit(OpCodes.Unbox_Any, type); + il.Emit(OpCodes.Stobj, type); + } + else + { + il.Emit(OpCodes.Castclass, type); + il.Emit(OpCodes.Stind_Ref); + } + } + } + } + + static string GetUniqueAssemblyName(string name) + { + var taken = new HashSet(AppDomain.CurrentDomain + .GetAssemblies() + .Select(a => a.GetName().Name)); + for (int i = 0; i < int.MaxValue; i++) + { + string candidate = name + i.ToString(CultureInfo.InvariantCulture); + if (!taken.Contains(candidate)) + return candidate; + } + + throw new NotSupportedException("Too many assemblies"); + } + } +} diff --git a/src/runtime/Util/EventHandlerCollection.cs b/src/runtime/Util/EventHandlerCollection.cs index 0cd03d0fd..f6b120b06 100644 --- a/src/runtime/Util/EventHandlerCollection.cs +++ b/src/runtime/Util/EventHandlerCollection.cs @@ -1,128 +1,128 @@ -using System; -using System.Collections.Generic; -using System.Reflection; -using System.Runtime.Serialization; -using System.Security.Permissions; - -namespace Python.Runtime; - -[Serializable] -internal class EventHandlerCollection: Dictionary> -{ - readonly EventInfo info; - public EventHandlerCollection(EventInfo @event) - { - info = @event; - } - - /// - /// Register a new Python object event handler with the event. - /// - internal bool AddEventHandler(BorrowedReference target, PyObject handler) - { - object? obj = null; - if (target != null) - { - var co = (CLRObject)ManagedType.GetManagedObject(target)!; - obj = co.inst; - } - - // Create a true delegate instance of the appropriate type to - // wrap the Python handler. Note that wrapper delegate creation - // always succeeds, though calling the wrapper may fail. - Type type = info.EventHandlerType; - Delegate d = PythonEngine.DelegateManager.GetDelegate(type, handler); - - // Now register the handler in a mapping from instance to pairs - // of (handler hash, delegate) so we can lookup to remove later. - object key = obj ?? info.ReflectedType; - if (!TryGetValue(key, out var list)) - { - list = new List(); - this[key] = list; - } - list.Add(new Handler(Runtime.PyObject_Hash(handler), d)); - - // Note that AddEventHandler helper only works for public events, - // so we have to get the underlying add method explicitly. - object[] args = { d }; - MethodInfo mi = info.GetAddMethod(true); - mi.Invoke(obj, BindingFlags.Default, null, args, null); - - return true; - } - - - /// - /// Remove the given Python object event handler. - /// - internal bool RemoveEventHandler(BorrowedReference target, BorrowedReference handler) - { - object? obj = null; - if (target != null) - { - var co = (CLRObject)ManagedType.GetManagedObject(target)!; - obj = co.inst; - } - - nint hash = Runtime.PyObject_Hash(handler); - if (hash == -1 && Exceptions.ErrorOccurred()) - { - return false; - } - - object key = obj ?? info.ReflectedType; - - if (!TryGetValue(key, out var list)) - { - Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); - return false; - } - - object?[] args = { null }; - MethodInfo mi = info.GetRemoveMethod(true); - - for (var i = 0; i < list.Count; i++) - { - var item = (Handler)list[i]; - if (item.hash != hash) - { - continue; - } - args[0] = item.del; - try - { - mi.Invoke(obj, BindingFlags.Default, null, args, null); - } - catch - { - continue; - } - list.RemoveAt(i); - if (list.Count == 0) - { - Remove(key); - } - return true; - } - - Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); - return false; - } - - #region Serializable - [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] - protected EventHandlerCollection(SerializationInfo info, StreamingContext context) - : base(info, context) - { - this.info = (EventInfo)info.GetValue("event", typeof(EventInfo)); - } - [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] - public override void GetObjectData(SerializationInfo info, StreamingContext context) - { - base.GetObjectData(info, context); - - info.AddValue("event", this.info); - } - #endregion -} +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Runtime.Serialization; +using System.Security.Permissions; + +namespace Python.Runtime; + +[Serializable] +internal class EventHandlerCollection: Dictionary> +{ + readonly EventInfo info; + public EventHandlerCollection(EventInfo @event) + { + info = @event; + } + + /// + /// Register a new Python object event handler with the event. + /// + internal bool AddEventHandler(BorrowedReference target, PyObject handler) + { + object? obj = null; + if (target != null) + { + var co = (CLRObject)ManagedType.GetManagedObject(target)!; + obj = co.inst; + } + + // Create a true delegate instance of the appropriate type to + // wrap the Python handler. Note that wrapper delegate creation + // always succeeds, though calling the wrapper may fail. + Type type = info.EventHandlerType; + Delegate d = PythonEngine.DelegateManager.GetDelegate(type, handler); + + // Now register the handler in a mapping from instance to pairs + // of (handler hash, delegate) so we can lookup to remove later. + object key = obj ?? info.ReflectedType; + if (!TryGetValue(key, out var list)) + { + list = new List(); + this[key] = list; + } + list.Add(new Handler(Runtime.PyObject_Hash(handler), d)); + + // Note that AddEventHandler helper only works for public events, + // so we have to get the underlying add method explicitly. + object[] args = { d }; + MethodInfo mi = info.GetAddMethod(true); + mi.Invoke(obj, BindingFlags.Default, null, args, null); + + return true; + } + + + /// + /// Remove the given Python object event handler. + /// + internal bool RemoveEventHandler(BorrowedReference target, BorrowedReference handler) + { + object? obj = null; + if (target != null) + { + var co = (CLRObject)ManagedType.GetManagedObject(target)!; + obj = co.inst; + } + + nint hash = Runtime.PyObject_Hash(handler); + if (hash == -1 && Exceptions.ErrorOccurred()) + { + return false; + } + + object key = obj ?? info.ReflectedType; + + if (!TryGetValue(key, out var list)) + { + Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); + return false; + } + + object?[] args = { null }; + MethodInfo mi = info.GetRemoveMethod(true); + + for (var i = 0; i < list.Count; i++) + { + var item = (Handler)list[i]; + if (item.hash != hash) + { + continue; + } + args[0] = item.del; + try + { + mi.Invoke(obj, BindingFlags.Default, null, args, null); + } + catch + { + continue; + } + list.RemoveAt(i); + if (list.Count == 0) + { + Remove(key); + } + return true; + } + + Exceptions.SetError(Exceptions.ValueError, "unknown event handler"); + return false; + } + + #region Serializable + [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] + protected EventHandlerCollection(SerializationInfo info, StreamingContext context) + : base(info, context) + { + this.info = (EventInfo)info.GetValue("event", typeof(EventInfo)); + } + [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] + public override void GetObjectData(SerializationInfo info, StreamingContext context) + { + base.GetObjectData(info, context); + + info.AddValue("event", this.info); + } + #endregion +} diff --git a/src/testing/Properties/launchSettings.json b/src/testing/Properties/launchSettings.json new file mode 100644 index 000000000..781c177ab --- /dev/null +++ b/src/testing/Properties/launchSettings.json @@ -0,0 +1,7 @@ +{ + "profiles": { + "Python.Test": { + "commandName": "Project" + } + } +} \ No newline at end of file diff --git a/src/testing/interfacetest.cs b/src/testing/interfacetest.cs index a1a7106b5..9e5be4313 100644 --- a/src/testing/interfacetest.cs +++ b/src/testing/interfacetest.cs @@ -1,117 +1,117 @@ -namespace Python.Test -{ - /// - /// Supports CLR class unit tests. - /// - public interface IPublicInterface - { - } - - internal interface IInternalInterface - { - } - - public interface ISayHello1 - { - string SayHello(); - } - - public interface ISayHello2 - { - string SayHello(); - } - - public class InterfaceTest : ISayHello1, ISayHello2 - { - public InterfaceTest() - { - } - - public string HelloProperty - { - get { return "hello"; } - } - - string ISayHello1.SayHello() - { - return "hello 1"; - } - - string ISayHello2.SayHello() - { - return "hello 2"; - } - - public ISayHello1 GetISayHello1() - { - return this; - } - - public void GetISayHello2(out ISayHello2 hello2) - { - hello2 = this; - } - - public ISayHello1 GetNoSayHello(out ISayHello2 hello2) - { - hello2 = null; - return null; - } - - public ISayHello1 [] GetISayHello1Array() - { - return new[] { this }; - } - - public interface IPublic - { - } - - protected interface IProtected - { - } - - internal interface IInternal - { - } - - private interface IPrivate - { - } - } - - public interface IOutArg - { - string MyMethod_Out(string name, out int index); - } - - public class OutArgCaller - { - public static int CallMyMethod_Out(IOutArg myInterface) - { - myInterface.MyMethod_Out("myclient", out int index); - return index; - } - } - - public interface IGenericInterface - { - public T Get(T x); - } - - public class SpecificInterfaceUser - { - public SpecificInterfaceUser(IGenericInterface some, int x) - { - some.Get(x); - } - } - - public class GenericInterfaceUser - { - public GenericInterfaceUser(IGenericInterface some, T x) - { - some.Get(x); - } - } -} +namespace Python.Test +{ + /// + /// Supports CLR class unit tests. + /// + public interface IPublicInterface + { + } + + internal interface IInternalInterface + { + } + + public interface ISayHello1 + { + string SayHello(); + } + + public interface ISayHello2 + { + string SayHello(); + } + + public class InterfaceTest : ISayHello1, ISayHello2 + { + public InterfaceTest() + { + } + + public string HelloProperty + { + get { return "hello"; } + } + + string ISayHello1.SayHello() + { + return "hello 1"; + } + + string ISayHello2.SayHello() + { + return "hello 2"; + } + + public ISayHello1 GetISayHello1() + { + return this; + } + + public void GetISayHello2(out ISayHello2 hello2) + { + hello2 = this; + } + + public ISayHello1 GetNoSayHello(out ISayHello2 hello2) + { + hello2 = null; + return null; + } + + public ISayHello1 [] GetISayHello1Array() + { + return new[] { this }; + } + + public interface IPublic + { + } + + protected interface IProtected + { + } + + internal interface IInternal + { + } + + private interface IPrivate + { + } + } + + public interface IOutArg + { + string MyMethod_Out(string name, out int index); + } + + public class OutArgCaller + { + public static int CallMyMethod_Out(IOutArg myInterface) + { + myInterface.MyMethod_Out("myclient", out int index); + return index; + } + } + + public interface IGenericInterface + { + public T Get(T x); + } + + public class SpecificInterfaceUser + { + public SpecificInterfaceUser(IGenericInterface some, int x) + { + some.Get(x); + } + } + + public class GenericInterfaceUser + { + public GenericInterfaceUser(IGenericInterface some, T x) + { + some.Get(x); + } + } +} diff --git a/tests/conftest.py b/tests/conftest.py index 1ac20e1dd..ff82e555c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,120 +1,120 @@ -# -*- coding: utf-8 -*- -# TODO: move tests one out of src to project root. -# TODO: travis has numpy on their workers. Maybe add tests? - -"""Helpers for testing.""" - -import ctypes -import os -import sys -import sysconfig -from pathlib import Path -from subprocess import check_call -from tempfile import mkdtemp -import shutil - -import pytest - -# Add path for `Python.Test` -cwd = Path(__file__).parent -fixtures_path = cwd / "fixtures" -sys.path.append(str(fixtures_path)) - - -def pytest_addoption(parser): - parser.addoption( - "--runtime", - action="store", - default="default", - help="Must be one of default, coreclr, netfx and mono", - ) - - -collect_ignore = [] - - -def pytest_configure(config): - global bin_path - if "clr" in sys.modules: - # Already loaded (e.g. by the C# test runner), skip build - import clr - - clr.AddReference("Python.Test") - return - - runtime_opt = config.getoption("runtime") - if runtime_opt not in ["coreclr", "netfx", "mono", "default"]: - raise RuntimeError(f"Invalid runtime: {runtime_opt}") - - test_proj_path = cwd.parent / "src" / "testing" - bin_path = Path(mkdtemp()) - - fw = "netstandard2.0" - runtime_params = {} - - if runtime_opt == "coreclr": - # This is optional now: - # - # fw = "net6.0" - # runtime_params["runtime_config"] = str( - # bin_path / "Python.Test.runtimeconfig.json" - # ) - collect_ignore.append("domain_tests/test_domain_reload.py") - else: - domain_tests_dir = cwd / "domain_tests" - domain_bin_path = domain_tests_dir / "bin" - build_cmd = [ - "dotnet", - "build", - str(domain_tests_dir), - "-o", - str(domain_bin_path), - ] - is_64bits = sys.maxsize > 2**32 - if not is_64bits: - build_cmd += ["/p:Prefer32Bit=True"] - check_call(build_cmd) - - check_call( - ["dotnet", "publish", "-f", fw, "-o", str(bin_path), str(test_proj_path)] - ) - - import os - os.environ["PYTHONNET_RUNTIME"] = runtime_opt - for k, v in runtime_params.items(): - os.environ[f"PYTHONNET_{runtime_opt.upper()}_{k.upper()}"] = v - - import clr - - sys.path.append(str(bin_path)) - clr.AddReference("Python.Test") - - -def pytest_unconfigure(config): - global bin_path - try: - shutil.rmtree(bin_path) - except Exception: - pass - - -def pytest_report_header(config): - """Generate extra report headers""" - # FIXME: https://github.com/pytest-dev/pytest/issues/2257 - is_64bits = sys.maxsize > 2**32 - arch = "x64" if is_64bits else "x86" - ucs = ctypes.sizeof(ctypes.c_wchar) - libdir = sysconfig.get_config_var("LIBDIR") - - return f"Arch: {arch}, UCS: {ucs}, LIBDIR: {libdir}" - - -@pytest.fixture() -def filepath(): - """Returns full filepath for file in `fixtures` directory.""" - - def make_filepath(filename): - # http://stackoverflow.com/questions/18011902/parameter-to-a-fixture - return os.path.join(fixtures_path, filename) - - return make_filepath +# -*- coding: utf-8 -*- +# TODO: move tests one out of src to project root. +# TODO: travis has numpy on their workers. Maybe add tests? + +"""Helpers for testing.""" + +import ctypes +import os +import sys +import sysconfig +from pathlib import Path +from subprocess import check_call +from tempfile import mkdtemp +import shutil + +import pytest + +# Add path for `Python.Test` +cwd = Path(__file__).parent +fixtures_path = cwd / "fixtures" +sys.path.append(str(fixtures_path)) + + +def pytest_addoption(parser): + parser.addoption( + "--runtime", + action="store", + default="default", + help="Must be one of default, coreclr, netfx and mono", + ) + + +collect_ignore = [] + + +def pytest_configure(config): + global bin_path + if "clr" in sys.modules: + # Already loaded (e.g. by the C# test runner), skip build + import clr + + clr.AddReference("Python.Test") + return + + runtime_opt = config.getoption("runtime") + if runtime_opt not in ["coreclr", "netfx", "mono", "default"]: + raise RuntimeError(f"Invalid runtime: {runtime_opt}") + + test_proj_path = cwd.parent / "src" / "testing" + bin_path = Path(mkdtemp()) + + fw = "netstandard2.0" + runtime_params = {} + + if runtime_opt == "coreclr": + # This is optional now: + # + # fw = "net6.0" + # runtime_params["runtime_config"] = str( + # bin_path / "Python.Test.runtimeconfig.json" + # ) + collect_ignore.append("domain_tests/test_domain_reload.py") + else: + domain_tests_dir = cwd / "domain_tests" + domain_bin_path = domain_tests_dir / "bin" + build_cmd = [ + "dotnet", + "build", + str(domain_tests_dir), + "-o", + str(domain_bin_path), + ] + is_64bits = sys.maxsize > 2**32 + if not is_64bits: + build_cmd += ["/p:Prefer32Bit=True"] + check_call(build_cmd) + + check_call( + ["dotnet", "publish", "-f", fw, "-o", str(bin_path), str(test_proj_path)] + ) + + import os + os.environ["PYTHONNET_RUNTIME"] = runtime_opt + for k, v in runtime_params.items(): + os.environ[f"PYTHONNET_{runtime_opt.upper()}_{k.upper()}"] = v + + import clr + + sys.path.append(str(bin_path)) + clr.AddReference("Python.Test") + + +def pytest_unconfigure(config): + global bin_path + try: + shutil.rmtree(bin_path) + except Exception: + pass + + +def pytest_report_header(config): + """Generate extra report headers""" + # FIXME: https://github.com/pytest-dev/pytest/issues/2257 + is_64bits = sys.maxsize > 2**32 + arch = "x64" if is_64bits else "x86" + ucs = ctypes.sizeof(ctypes.c_wchar) + libdir = sysconfig.get_config_var("LIBDIR") + + return f"Arch: {arch}, UCS: {ucs}, LIBDIR: {libdir}" + + +@pytest.fixture() +def filepath(): + """Returns full filepath for file in `fixtures` directory.""" + + def make_filepath(filename): + # http://stackoverflow.com/questions/18011902/parameter-to-a-fixture + return os.path.join(fixtures_path, filename) + + return make_filepath diff --git a/tests/test_subclass.py b/tests/test_subclass.py index c6ab7650f..ca507e962 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -6,6 +6,335 @@ """Test sub-classing managed types""" +import System +import pytest +from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, + FunctionsTest, IGenericInterface) +from System.Collections.Generic import List + + +def interface_test_class_fixture(subnamespace): + """Delay creation of class until test starts.""" + + class InterfaceTestClass(IInterfaceTest): + """class that implements the test interface""" + __namespace__ = "Python.Test." + subnamespace + + def foo(self): + return "InterfaceTestClass" + + def bar(self, x, i): + return "/".join([x] * i) + + return InterfaceTestClass + + +def interface_generic_class_fixture(subnamespace): + + class GenericInterfaceImpl(IGenericInterface[int]): + __namespace__ = "Python.Test." + subnamespace + + def Get(self, x): + return x + + return GenericInterfaceImpl + + +def derived_class_fixture(subnamespace): + """Delay creation of class until test starts.""" + + class DerivedClass(SubClassTest): + """class that derives from a class deriving from IInterfaceTest""" + __namespace__ = "Python.Test." + subnamespace + + def foo(self): + return "DerivedClass" + + def base_foo(self): + return SubClassTest.foo(self) + + def super_foo(self): + return super(DerivedClass, self).foo() + + def bar(self, x, i): + return "_".join([x] * i) + + def return_list(self): + l = List[str]() + l.Add("A") + l.Add("B") + l.Add("C") + return l + + return DerivedClass + +def broken_derived_class_fixture(subnamespace): + """Delay creation of class until test starts.""" + + class DerivedClass(SubClassTest): + """class that derives from a class deriving from IInterfaceTest""" + __namespace__ = 3 + + return DerivedClass + +def derived_event_test_class_fixture(subnamespace): + """Delay creation of class until test starts.""" + + class DerivedEventTest(IInterfaceTest): + """class that implements IInterfaceTest.TestEvent""" + __namespace__ = "Python.Test." + subnamespace + + def __init__(self): + self.event_handlers = [] + + # event handling + def add_TestEvent(self, handler): + self.event_handlers.append(handler) + + def remove_TestEvent(self, handler): + self.event_handlers.remove(handler) + + def OnTestEvent(self, value): + args = EventArgsTest(value) + for handler in self.event_handlers: + handler(self, args) + + return DerivedEventTest + + +def test_base_class(): + """Test base class managed type""" + ob = SubClassTest() + assert ob.foo() == "foo" + assert FunctionsTest.test_foo(ob) == "foo" + assert ob.bar("bar", 2) == "bar" + assert FunctionsTest.test_bar(ob, "bar", 2) == "bar" + assert ob.not_overriden() == "not_overriden" + assert list(ob.return_list()) == ["a", "b", "c"] + assert list(SubClassTest.test_list(ob)) == ["a", "b", "c"] + + +def test_interface(): + """Test python classes can derive from C# interfaces""" + InterfaceTestClass = interface_test_class_fixture(test_interface.__name__) + ob = InterfaceTestClass() + assert ob.foo() == "InterfaceTestClass" + assert FunctionsTest.test_foo(ob) == "InterfaceTestClass" + assert ob.bar("bar", 2) == "bar/bar" + assert FunctionsTest.test_bar(ob, "bar", 2) == "bar/bar" + + # pass_through will convert from InterfaceTestClass -> IInterfaceTest, + # causing a new wrapper object to be created. Hence id will differ. + x = FunctionsTest.pass_through_interface(ob) + assert id(x) != id(ob) + + +def test_derived_class(): + """Test python class derived from managed type""" + DerivedClass = derived_class_fixture(test_derived_class.__name__) + ob = DerivedClass() + assert ob.foo() == "DerivedClass" + assert ob.base_foo() == "foo" + assert ob.super_foo() == "foo" + assert FunctionsTest.test_foo(ob) == "DerivedClass" + assert ob.bar("bar", 2) == "bar_bar" + assert FunctionsTest.test_bar(ob, "bar", 2) == "bar_bar" + assert ob.not_overriden() == "not_overriden" + assert list(ob.return_list()) == ["A", "B", "C"] + assert list(SubClassTest.test_list(ob)) == ["A", "B", "C"] + + x = FunctionsTest.pass_through(ob) + assert id(x) == id(ob) + +def test_broken_derived_class(): + """Test python class derived from managed type with invalid namespace""" + with pytest.raises(TypeError): + DerivedClass = broken_derived_class_fixture(test_derived_class.__name__) + ob = DerivedClass() + +def test_derived_traceback(): + """Test python exception traceback in class derived from managed base""" + class DerivedClass(SubClassTest): + __namespace__ = "Python.Test.traceback" + + def foo(self): + print (xyzname) + return None + + import sys,traceback + ob = DerivedClass() + + # direct call + try: + ob.foo() + assert False + except: + e = sys.exc_info() + assert "xyzname" in str(e[1]) + location = traceback.extract_tb(e[2])[-1] + assert location[2] == "foo" + + # call through managed code + try: + FunctionsTest.test_foo(ob) + assert False + except: + e = sys.exc_info() + assert "xyzname" in str(e[1]) + location = traceback.extract_tb(e[2])[-1] + assert location[2] == "foo" + + +def test_create_instance(): + """Test derived instances can be created from managed code""" + DerivedClass = derived_class_fixture(test_create_instance.__name__) + ob = FunctionsTest.create_instance(DerivedClass) + assert ob.foo() == "DerivedClass" + assert FunctionsTest.test_foo(ob) == "DerivedClass" + assert ob.bar("bar", 2) == "bar_bar" + assert FunctionsTest.test_bar(ob, "bar", 2) == "bar_bar" + assert ob.not_overriden() == "not_overriden" + + x = FunctionsTest.pass_through(ob) + assert id(x) == id(ob) + + InterfaceTestClass = interface_test_class_fixture(test_create_instance.__name__) + ob2 = FunctionsTest.create_instance_interface(InterfaceTestClass) + assert ob2.foo() == "InterfaceTestClass" + assert FunctionsTest.test_foo(ob2) == "InterfaceTestClass" + assert ob2.bar("bar", 2) == "bar/bar" + assert FunctionsTest.test_bar(ob2, "bar", 2) == "bar/bar" + + y = FunctionsTest.pass_through_interface(ob2) + assert id(y) != id(ob2) + + +def test_events(): + class EventHandler(object): + def handler(self, x, args): + self.value = args.value + + event_handler = EventHandler() + + x = SubClassTest() + x.TestEvent += event_handler.handler + assert FunctionsTest.test_event(x, 1) == 1 + assert event_handler.value == 1 + + InterfaceTestClass = interface_test_class_fixture(test_events.__name__) + i = InterfaceTestClass() + with pytest.raises(System.NotImplementedException): + FunctionsTest.test_event(i, 2) + + DerivedEventTest = derived_event_test_class_fixture(test_events.__name__) + d = DerivedEventTest() + d.add_TestEvent(event_handler.handler) + assert FunctionsTest.test_event(d, 3) == 3 + assert event_handler.value == 3 + assert len(d.event_handlers) == 1 + + +def test_isinstance_check(): + a = [str(x) for x in range(0, 1000)] + b = [System.String(x) for x in a] + + for x in a: + assert not isinstance(x, System.Object) + assert not isinstance(x, System.String) + + for x in b: + assert isinstance(x, System.Object) + assert isinstance(x, System.String) + +def test_namespace_and_init(): + calls = [] + class TestX(System.Object): + __namespace__ = "test_clr_subclass_with_init_args" + def __init__(self, *args, **kwargs): + calls.append((args, kwargs)) + t = TestX(1,2,3,foo="bar") + assert len(calls) == 1 + assert calls[0][0] == (1,2,3) + assert calls[0][1] == {"foo":"bar"} + +def test_namespace_and_argless_init(): + calls = [] + class TestX(System.Object): + __namespace__ = "test_clr_subclass_without_init_args" + def __init__(self): + calls.append(True) + t = TestX() + assert len(calls) == 1 + assert calls[0] == True + + +def test_namespace_and_no_init(): + class TestX(System.Object): + __namespace__ = "test_clr_subclass_without_init" + q = 1 + t = TestX() + assert t.q == 1 + +def test_construction_from_clr(): + import clr + calls = [] + class TestX(System.Object): + __namespace__ = "test_clr_subclass_init_from_clr" + @clr.clrmethod(None, [int, str]) + def __init__(self, i, s): + calls.append((i, s)) + + # Construct a TestX from Python + t = TestX(1, "foo") + assert len(calls) == 1 + assert calls[0][0] == 1 + assert calls[0][1] == "foo" + + # Reset calls and construct a TestX from CLR + calls = [] + tp = t.GetType() + t2 = tp.GetConstructors()[0].Invoke(None) + assert len(calls) == 0 + + # The object has only been constructed, now it needs to be initialized as well + tp.GetMethod("__init__").Invoke(t2, [1, "foo"]) + assert len(calls) == 1 + assert calls[0][0] == 1 + assert calls[0][1] == "foo" + +# regression test for https://github.com/pythonnet/pythonnet/issues/1565 +def test_can_be_collected_by_gc(): + from Python.Test import BaseClass + + class Derived(BaseClass): + __namespace__ = 'test_can_be_collected_by_gc' + + inst = Derived() + cycle = [inst] + del inst + cycle.append(cycle) + del cycle + + import gc + gc.collect() + +def test_generic_interface(): + from System import Int32 + from Python.Test import GenericInterfaceUser, SpecificInterfaceUser + + GenericInterfaceImpl = interface_generic_class_fixture(test_generic_interface.__name__) + + obj = GenericInterfaceImpl() + SpecificInterfaceUser(obj, Int32(0)) + GenericInterfaceUser[Int32](obj, Int32(0)) +# -*- coding: utf-8 -*- +# FIXME: This test module randomly passes/fails even if all tests are skipped. +# Something fishy is going on with the Test fixtures. Behavior seen on CI on +# both Linux and Windows +# TODO: Remove delay of class creations. Adding SetUp/TearDown may help + +"""Test sub-classing managed types""" + import System import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py old mode 100755 new mode 100644 index 6d80bcfa6..a19bc67de --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -31,6 +31,371 @@ } +def _check_output(*args, **kwargs): + return subprocess.check_output(*args, **kwargs, encoding="utf8") + + +class AstParser(object): + """Walk an AST and determine the members of all structs""" + + def __init__(self): + self._typedefs = {} + self._typedecls = {} + self._structs = {} + self._struct_stack = [] + self._struct_members_stack = [] + self._ptr_decl_depth = 0 + self._struct_members = {} + self._decl_names = {} + + def get_struct_members(self, name): + """return a list of (name, type) of struct members""" + defs = self._typedefs.get(name) + if defs is None: + return None + node = self._get_leaf_node(defs) + name = node.name + if name is None: + name = defs.declname + return self._struct_members.get(name) + + def visit(self, node): + if isinstance(node, c_ast.FileAST): + self.visit_ast(node) + elif isinstance(node, c_ast.Typedef): + self.visit_typedef(node) + elif isinstance(node, c_ast.TypeDecl): + self.visit_typedecl(node) + elif isinstance(node, c_ast.Struct): + self.visit_struct(node) + elif isinstance(node, c_ast.Decl): + self.visit_decl(node) + elif isinstance(node, c_ast.FuncDecl): + self.visit_funcdecl(node) + elif isinstance(node, c_ast.PtrDecl): + self.visit_ptrdecl(node) + elif isinstance(node, c_ast.IdentifierType): + self.visit_identifier(node) + + def visit_ast(self, ast): + for _name, node in ast.children(): + self.visit(node) + + def visit_typedef(self, typedef): + self._typedefs[typedef.name] = typedef.type + self.visit(typedef.type) + + def visit_typedecl(self, typedecl): + self._decl_names[typedecl.type] = typedecl.declname + self.visit(typedecl.type) + + def visit_struct(self, struct): + if struct.decls: + self._structs[self._get_struct_name(struct)] = struct + # recurse into the struct + self._struct_stack.insert(0, struct) + for decl in struct.decls: + self._struct_members_stack.insert(0, decl.name) + self.visit(decl) + self._struct_members_stack.pop(0) + self._struct_stack.pop(0) + elif self._ptr_decl_depth or self._struct_members_stack: + # the struct is empty, but add it as a member to the current + # struct as the current member maybe a pointer to it. + self._add_struct_member(struct.name) + + def visit_decl(self, decl): + self.visit(decl.type) + + def visit_funcdecl(self, funcdecl): + self.visit(funcdecl.type) + + def visit_ptrdecl(self, ptrdecl): + self._ptr_decl_depth += 1 + self.visit(ptrdecl.type) + self._ptr_decl_depth -= 1 + + def visit_identifier(self, identifier): + type_name = " ".join(identifier.names) + self._add_struct_member(type_name) + + def _add_struct_member(self, type_name): + if not (self._struct_stack and self._struct_members_stack): + return + + # add member to current struct + current_struct = self._struct_stack[0] + member_name = self._struct_members_stack[0] + struct_members = self._struct_members.setdefault( + self._get_struct_name(current_struct), [] + ) + + # get the node associated with this type + node = None + if type_name in self._typedefs: + node = self._get_leaf_node(self._typedefs[type_name]) + # If the struct was only declared when the typedef was created, its member + # information will not have been recorded and we have to look it up in the + # structs + if isinstance(node, c_ast.Struct) and node.decls is None: + if node.name in self._structs: + node = self._structs[node.name] + elif type_name in self._structs: + node = self._structs[type_name] + + # If it's a struct (and not a pointer to a struct) expand + # it into the current struct definition + if not self._ptr_decl_depth and isinstance(node, c_ast.Struct): + for decl in node.decls or []: + self._struct_members_stack.insert(0, decl.name) + self.visit(decl) + self._struct_members_stack.pop(0) + else: + # otherwise add it as a single member + struct_members.append((member_name, type_name)) + + def _get_leaf_node(self, node): + if isinstance(node, c_ast.Typedef): + return self._get_leaf_node(node.type) + if isinstance(node, c_ast.TypeDecl): + return self._get_leaf_node(node.type) + return node + + def _get_struct_name(self, node): + return node.name or self._decl_names.get(node) or "_struct_%d" % id(node) + + +class Writer(object): + def __init__(self): + self._stream = StringIO() + + def append(self, indent=0, code=""): + self._stream.write("%s%s\n" % (indent * " ", code)) + + def extend(self, s): + self._stream.write(s) + + def to_string(self): + return self._stream.getvalue() + + +def preprocess_python_headers(*, cc=None, include_py=None): + """Return Python.h pre-processed, ready for parsing. + Requires clang. + """ + this_path = Path(__file__).parent + + fake_libc_include = this_path / "fake_libc_include" + include_dirs = [fake_libc_include] + + if cc is None: + cc = shutil.which("clang") + if cc is None: + cc = shutil.which("gcc") + if cc is None: + raise RuntimeError("No suitable C compiler found, need clang or gcc") + + if include_py is None: + include_py = sysconfig.get_config_var("INCLUDEPY") + include_py = Path(include_py) + + include_dirs.append(include_py) + + include_args = [c for p in include_dirs for c in ["-I", str(p)]] + + # fmt: off + defines = [ + "-D", "__attribute__(x)=", + "-D", "__inline__=inline", + "-D", "__asm__=;#pragma asm", + "-D", "__int64=long long", + "-D", "_POSIX_THREADS", + ] + + if sys.platform == "win32": + defines.extend([ + "-D", "__inline=inline", + "-D", "__ptr32=", + "-D", "__ptr64=", + "-D", "__declspec(x)=", + ]) + #fmt: on + + if hasattr(sys, "abiflags"): + if "d" in sys.abiflags: + defines.extend(("-D", "PYTHON_WITH_PYDEBUG")) + if "u" in sys.abiflags: + defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) + + python_h = include_py / "Python.h" + cmd = [cc, "-pthread"] + include_args + defines + ["-E", str(python_h)] + + # normalize as the parser doesn't like windows line endings. + lines = [] + for line in _check_output(cmd).splitlines(): + if line.startswith("#"): + line = line.replace("\\", "/") + lines.append(line) + return "\n".join(lines) + + +def gen_interop_head(writer, version, abi_flags): + filename = os.path.basename(__file__) + class_definition = f""" +// Auto-generated by {filename}. +// DO NOT MODIFY BY HAND. + +// Python {".".join(version[:2])}: ABI flags: '{abi_flags}' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{{""" + writer.extend(class_definition) + + +def gen_interop_tail(writer): + tail = """} +""" + writer.extend(tail) + + +def gen_heap_type_members(parser, writer, type_name): + """Generate the TypeOffset C# class""" + members = parser.get_struct_members("PyHeapTypeObject") + class_definition = f""" + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class {type_name} : GeneratedTypeOffsets, ITypeOffsets + {{ + public {type_name}() {{ }} + // Auto-generated from PyHeapTypeObject in Python.h +""" + + # All the members are sizeof(void*) so we don't need to do any + # extra work to determine the size based on the type. + for name, _type in members: + name = _typeoffset_member_renames.get(name, name) + class_definition += " public int %s { get; private set; }\n" % name + + class_definition += """ } +""" + writer.extend(class_definition) + + +def gen_structure_code(parser, writer, type_name, indent): + members = parser.get_struct_members(type_name) + if members is None: + return False + out = writer.append + out(indent, "[StructLayout(LayoutKind.Sequential)]") + out(indent, f"internal struct {type_name}") + out(indent, "{") + for name, _type in members: + out(indent + 1, f"public IntPtr {name};") + out(indent, "}") + out() + return True + + +def main(*, cc=None, include_py=None, version=None, out=None): + # preprocess Python.h and build the AST + python_h = preprocess_python_headers(cc=cc, include_py=include_py) + parser = c_parser.CParser() + ast = parser.parse(python_h) + + # extract struct members from the AST + ast_parser = AstParser() + ast_parser.visit(ast) + + writer = Writer() + + if include_py and not version: + raise RuntimeError("If the include path is overridden, version must be " + "defined" + ) + + if version: + version = version.split('.') + else: + version = sys.version_info + + # generate the C# code + abi_flags = getattr(sys, "abiflags", "").replace("m", "") + gen_interop_head(writer, version, abi_flags) + + type_name = f"TypeOffset{version[0]}{version[1]}{abi_flags}" + gen_heap_type_members(ast_parser, writer, type_name) + + gen_interop_tail(writer) + + interop_cs = writer.to_string() + if not out or out == "-": + print(interop_cs) + else: + with open(out, "w") as fh: + fh.write(interop_cs) + + +if __name__ == "__main__": + import argparse + + a = argparse.ArgumentParser("Interop file generator for Python.NET") + a.add_argument("--cc", help="C compiler to use, either clang or gcc") + a.add_argument("--include-py", help="Include path of Python") + a.add_argument("--version", help="Python version") + a.add_argument("--out", help="Output path", default="-") + args = a.parse_args() + + sys.exit(main( + cc=args.cc, + include_py=args.include_py, + out=args.out, + version=args.version + )) +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +TypeOffset is a C# class that mirrors the in-memory layout of heap +allocated Python objects. + +This script parses the Python C headers and outputs the TypeOffset +C# class. + +Requirements: + - pycparser + - clang +""" + +import os +import shutil +import sys +import sysconfig +import subprocess + +from io import StringIO +from pathlib import Path +from pycparser import c_ast, c_parser + +# rename some members from their C name when generating the C# +_typeoffset_member_renames = { + "ht_name": "name", + "ht_qualname": "qualname", + "getitem": "spec_cache_getitem", +} + + def _check_output(*args, **kwargs): return subprocess.check_output(*args, **kwargs, encoding="utf8")