From 94f8ade285f657bfec022a5dee066ee668bac451 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 9 Aug 2025 22:52:27 +0200 Subject: [PATCH 01/14] Initial 3.14 commit (cherry picked from commit caac33d258e327bba18d6436a62d33d7fcd08859) --- .github/workflows/main.yml | 2 +- pyproject.toml | 2 +- src/runtime/Native/TypeOffset314.cs | 147 ++++++++++++++++++++++++++++ src/runtime/PythonEngine.cs | 2 +- 4 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 src/runtime/Native/TypeOffset314.cs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0ae51bce9..5f05ea997 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - python: ["3.8", "3.9", "3.10", "3.11"] + python: ["3.8", "3.9", "3.10", "3.11", "3.14"] steps: - name: Checkout code diff --git a/pyproject.toml b/pyproject.toml index 6151e3fff..dd3d057f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "clr_loader>=0.2.2,<0.3.0" ] -requires-python = ">=3.7, <3.12" +requires-python = ">=3.7, <3.15" classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/src/runtime/Native/TypeOffset314.cs b/src/runtime/Native/TypeOffset314.cs new file mode 100644 index 000000000..6ec9a621e --- /dev/null +++ b/src/runtime/Native/TypeOffset314.cs @@ -0,0 +1,147 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.14: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset314 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset314() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt_full { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + public int tp_watched { get; private set; } + public int tp_versions_used { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int am_send { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + public int _ht_tpname { get; private set; } + public int ht_token { get; private set; } + public int spec_cache_getitem { get; private set; } + public int getitem_version { get; private set; } + public int init { get; private set; } + } +} + diff --git a/src/runtime/PythonEngine.cs b/src/runtime/PythonEngine.cs index eb0c98ce9..63a8fe827 100644 --- a/src/runtime/PythonEngine.cs +++ b/src/runtime/PythonEngine.cs @@ -135,7 +135,7 @@ public static string PythonPath } public static Version MinSupportedVersion => new(3, 7); - public static Version MaxSupportedVersion => new(3, 11, int.MaxValue, int.MaxValue); + public static Version MaxSupportedVersion => new(3, 14, int.MaxValue, int.MaxValue); public static bool IsSupportedVersion(Version version) => version >= MinSupportedVersion && version <= MaxSupportedVersion; public static string Version From 4231f0086518c4218f8b67271279eefde7f42ff2 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 22 Oct 2025 18:59:10 +0200 Subject: [PATCH 02/14] Apply alignment fix (cherry picked from commit e10d3332d6cc286551e754761ef2f757c9ff6f8c) --- src/runtime/Native/TypeOffset314.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/runtime/Native/TypeOffset314.cs b/src/runtime/Native/TypeOffset314.cs index 6ec9a621e..28101ba12 100644 --- a/src/runtime/Native/TypeOffset314.cs +++ b/src/runtime/Native/TypeOffset314.cs @@ -75,8 +75,14 @@ public TypeOffset314() { } public int tp_version_tag { get; private set; } public int tp_finalize { get; private set; } public int tp_vectorcall { get; private set; } + // This is an error in our generator: + // + // The fields below are actually not pointers (like we incorrectly + // assume for all other fields) but instead a char (1 byte) and a short + // (2 bytes). By dropping one of the fields, we still get the correct + // overall size of the struct. public int tp_watched { get; private set; } - public int tp_versions_used { get; private set; } + // public int tp_versions_used { get; private set; } public int am_await { get; private set; } public int am_aiter { get; private set; } public int am_anext { get; private set; } From 477ba22d513955760a67aff1a276b3d574d0bb43 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 22 Oct 2025 18:59:27 +0200 Subject: [PATCH 03/14] Disable problematic GC tests (cherry picked from commit e9765585b3d7497e49ee0c35a17bb1792b91170e) --- tests/test_method.py | 1 + tests/test_subclass.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/test_method.py b/tests/test_method.py index dfe5100bd..39eb3b260 100644 --- a/tests/test_method.py +++ b/tests/test_method.py @@ -2,6 +2,7 @@ """Test CLR method support.""" +import sys import System import pytest from Python.Test import MethodTest diff --git a/tests/test_subclass.py b/tests/test_subclass.py index ff53df7c1..63e7b5c2f 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -6,6 +6,7 @@ """Test sub-classing managed types""" +import sys import System import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, @@ -290,6 +291,7 @@ def __init__(self, i, s): assert calls[0][1] == "foo" # regression test for https://github.com/pythonnet/pythonnet/issues/1565 +@pytest.mark.skipif(sys.version_info >= (3, 14), reason="Test skipped on Python 3.14 and above") def test_can_be_collected_by_gc(): from Python.Test import BaseClass From 5e67d647f71969025b01c4b377536339b9cd6706 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 24 Oct 2025 14:56:38 +0200 Subject: [PATCH 04/14] Set ht_token to NULL in Python 3.14 (cherry picked from commit 65af09891fc6b2b5a832dbc2218c2f0eaf633684) --- src/runtime/Native/TypeOffset.cs | 15 +++++++++++++++ src/runtime/TypeManager.cs | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/src/runtime/Native/TypeOffset.cs b/src/runtime/Native/TypeOffset.cs index 0a85b05d2..c94447f37 100644 --- a/src/runtime/Native/TypeOffset.cs +++ b/src/runtime/Native/TypeOffset.cs @@ -76,6 +76,8 @@ static partial class TypeOffset internal static int tp_setattro { get; private set; } internal static int tp_str { get; private set; } internal static int tp_traverse { get; private set; } + // Special case: Only available in Python 3.14 onwards, set to -1 by default + internal static int ht_token { get; private set; } = -1; internal static void Use(ITypeOffsets offsets, int extraHeadOffset) { @@ -88,6 +90,19 @@ internal static void Use(ITypeOffsets offsets, int extraHeadOffset) slotNames.Add(offsetProperty.Name); var sourceProperty = typeof(ITypeOffsets).GetProperty(offsetProperty.Name); + if (sourceProperty == null) + { + if ((int)offsetProperty.GetValue(null) == -1) + { + // Skip, this is an optional offset value + continue; + } + else + { + throw new Exception($"No offset defined for necessary slot {offsetProperty.Name}"); + } + } + int value = (int)sourceProperty.GetValue(offsets, null); value += extraHeadOffset; offsetProperty.SetValue(obj: null, value: value, index: null); diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 3b75738b2..6ff207e99 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -609,6 +609,11 @@ internal static PyType AllocateTypeObject(string name, PyType metatype) Util.WriteIntPtr(type, TypeOffset.tp_traverse, subtype_traverse); Util.WriteIntPtr(type, TypeOffset.tp_clear, subtype_clear); + // This is a new mechanism in Python 3.14. We should eventually use it to implement + // a nicer type check, but for now we just need to ensure that it is set to NULL. + if (TypeOffset.ht_token != -1) + Util.WriteIntPtr(type, TypeOffset.ht_token, IntPtr.Zero); + InheritSubstructs(type.Reference.DangerousGetAddress()); return type; From 934633420804f413d11dfde53450d2989e04c153 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 7 Dec 2025 14:54:44 +0100 Subject: [PATCH 05/14] Workaround for blocked PyObject_GenericSetAttr in metatypes Python 3.14 introduced a new assertion that prevents us from using PyObject_GenericSetAttr directly in our meta type. To work around this, we manipulate the type dict directly. This workaround is a simplified variant of Cython's workaround from https://github.com/cython/cython/pull/6325. The relevant Python change is in https://github.com/python/cpython/pull/118454 (cherry picked from commit 08550d090a88f91a84b028208cf57a8a3b9c1b58) --- src/runtime/Native/PyIdentifier_.cs | 3 ++ src/runtime/Native/PyIdentifier_.tt | 1 + src/runtime/Types/MetaType.cs | 44 ++++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/runtime/Native/PyIdentifier_.cs b/src/runtime/Native/PyIdentifier_.cs index 4884a81ad..cba09ab1d 100644 --- a/src/runtime/Native/PyIdentifier_.cs +++ b/src/runtime/Native/PyIdentifier_.cs @@ -25,6 +25,8 @@ static class PyIdentifier public static BorrowedReference __self__ => new(f__self__); static IntPtr f__annotations__; public static BorrowedReference __annotations__ => new(f__annotations__); + static IntPtr f__dictoffset__; + public static BorrowedReference __dictoffset__ => new(f__dictoffset__); static IntPtr f__init__; public static BorrowedReference __init__ => new(f__init__); static IntPtr f__repr__; @@ -57,6 +59,7 @@ static partial class InternString "__slots__", "__self__", "__annotations__", + "__dictoffset__", "__init__", "__repr__", "__import__", diff --git a/src/runtime/Native/PyIdentifier_.tt b/src/runtime/Native/PyIdentifier_.tt index 03a26cb50..b431803e8 100644 --- a/src/runtime/Native/PyIdentifier_.tt +++ b/src/runtime/Native/PyIdentifier_.tt @@ -13,6 +13,7 @@ "__slots__", "__self__", "__annotations__", + "__dictoffset__", "__init__", "__repr__", diff --git a/src/runtime/Types/MetaType.cs b/src/runtime/Types/MetaType.cs index 9a66240d3..664863f4f 100644 --- a/src/runtime/Types/MetaType.cs +++ b/src/runtime/Types/MetaType.cs @@ -18,6 +18,7 @@ internal sealed class MetaType : ManagedType // set in Initialize private static PyType PyCLRMetaType; private static SlotsHolder _metaSlotsHodler; + private static int TypeDictOffset = -1; #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable. internal static readonly string[] CustomMethods = new string[] @@ -32,6 +33,25 @@ internal sealed class MetaType : ManagedType public static PyType Initialize() { PyCLRMetaType = TypeManager.CreateMetaType(typeof(MetaType), out _metaSlotsHodler); + + // Retrieve the offset of the type's dictionary from PyType_Type for + // use in the tp_setattro implementation. + using (NewReference dictOffset = Runtime.PyObject_GetAttr(Runtime.PyTypeType, PyIdentifier.__dictoffset__)) + { + if (dictOffset.IsNull()) + { + throw new InvalidOperationException("Could not get __dictoffset__ from PyType_Type"); + } + + nint dictOffsetVal = Runtime.PyLong_AsSignedSize_t(dictOffset.Borrow()); + if (dictOffsetVal <= 0) + { + throw new InvalidOperationException("Could not get __dictoffset__ from PyType_Type"); + } + + TypeDictOffset = checked((int)dictOffsetVal); + } + return PyCLRMetaType; } @@ -41,6 +61,7 @@ public static void Release() { _metaSlotsHodler.ResetSlots(); } + TypeDictOffset = -1; PyCLRMetaType.Dispose(); } @@ -246,7 +267,28 @@ public static int tp_setattro(BorrowedReference tp, BorrowedReference name, Borr } } - int res = Runtime.PyObject_GenericSetAttr(tp, name, value); + // Access the type's dictionary directly + // + // We can not use the PyObject_GenericSetAttr because since Python + // 3.14 as https://github.com/python/cpython/pull/118454 intrdoduced + // an assertion to prevent it from being called from metatypes. + // + // The direct dictionary access is equivalent to what Cython does + // to work around the same issue: https://github.com/cython/cython/pull/6325 + BorrowedReference typeDict = new(Util.ReadIntPtr(tp, TypeDictOffset)); + int res; + if (value.IsNull) + { + res = Runtime.PyDict_DelItem(typeDict, name); + if (res != 0) + { + Exceptions.SetError(Exceptions.AttributeError, "attribute not found"); + } + } + else + { + res = Runtime.PyDict_SetItem(typeDict, name, value); + } Runtime.PyType_Modified(tp); return res; From 8107ff11caef5a76d4516036132615081732b830 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 11 Dec 2024 17:54:54 +0100 Subject: [PATCH 06/14] Use PyThreadState_GetUnchecked on Python 3.13 (cherry picked from commit f3face061ac4432762fe707081c3e437b1f42d7d) --- src/runtime/Runtime.Delegates.cs | 14 ++++++++++++-- src/runtime/Runtime.cs | 4 ++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/runtime/Runtime.Delegates.cs b/src/runtime/Runtime.Delegates.cs index 5a6e0507d..1e9f7ab0e 100644 --- a/src/runtime/Runtime.Delegates.cs +++ b/src/runtime/Runtime.Delegates.cs @@ -23,7 +23,17 @@ static Delegates() Py_EndInterpreter = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_EndInterpreter), GetUnmanagedDll(_PythonDll)); PyThreadState_New = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_New), GetUnmanagedDll(_PythonDll)); PyThreadState_Get = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_Get), GetUnmanagedDll(_PythonDll)); - _PyThreadState_UncheckedGet = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(_PyThreadState_UncheckedGet), GetUnmanagedDll(_PythonDll)); + try + { + // Up until Python 3.13, this function was private and named + // slightly differently. + PyThreadState_GetUnchecked = (delegate* unmanaged[Cdecl])GetFunctionByName("_PyThreadState_UncheckedGet", GetUnmanagedDll(_PythonDll)); + } + catch (MissingMethodException) + { + + PyThreadState_GetUnchecked = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyThreadState_GetUnchecked), GetUnmanagedDll(_PythonDll)); + } try { PyGILState_Check = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyGILState_Check), GetUnmanagedDll(_PythonDll)); @@ -316,7 +326,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] Py_EndInterpreter { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_New { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_Get { get; } - internal static delegate* unmanaged[Cdecl] _PyThreadState_UncheckedGet { get; } + internal static delegate* unmanaged[Cdecl] PyThreadState_GetUnchecked { get; } internal static delegate* unmanaged[Cdecl] PyGILState_Check { get; } internal static delegate* unmanaged[Cdecl] PyGILState_Ensure { get; } internal static delegate* unmanaged[Cdecl] PyGILState_Release { get; } diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index ff081e893..925b4d3db 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -319,7 +319,7 @@ internal static void Shutdown() // Then release the GIL for good, if there is somehting to release // Use the unchecked version as the checked version calls `abort()` // if the current state is NULL. - if (_PyThreadState_UncheckedGet() != (PyThreadState*)0) + if (PyThreadState_GetUnchecked() != (PyThreadState*)0) { PyEval_SaveThread(); } @@ -745,7 +745,7 @@ internal static T TryUsingDll(Func op) internal static PyThreadState* PyThreadState_Get() => Delegates.PyThreadState_Get(); - internal static PyThreadState* _PyThreadState_UncheckedGet() => Delegates._PyThreadState_UncheckedGet(); + internal static PyThreadState* PyThreadState_GetUnchecked() => Delegates.PyThreadState_GetUnchecked(); internal static int PyGILState_Check() => Delegates.PyGILState_Check(); From 7d13c02ceef352d559e1fb85c02abc8ad9261e6d Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 27 Sep 2023 14:09:44 +0200 Subject: [PATCH 07/14] Remove deprecated function call (cherry picked from commit 8dfe4080d642397a7efcb52b8d3aa69da1675713) --- src/runtime/Converter.cs | 6 ++---- src/runtime/Runtime.Delegates.cs | 4 ++-- src/runtime/Runtime.cs | 3 ++- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/runtime/Converter.cs b/src/runtime/Converter.cs index f2c867e43..553bc34ce 100644 --- a/src/runtime/Converter.cs +++ b/src/runtime/Converter.cs @@ -1068,10 +1068,8 @@ internal static bool ToPrimitive(BorrowedReference value, Type obType, out objec { if (Runtime.PyUnicode_GetLength(value) == 1) { - IntPtr unicodePtr = Runtime.PyUnicode_AsUnicode(value); - Char[] buff = new Char[1]; - Marshal.Copy(unicodePtr, buff, 0, 1); - result = buff[0]; + int chr = Runtime.PyUnicode_ReadChar(value, 0); + result = (Char)chr; return true; } goto type_error; diff --git a/src/runtime/Runtime.Delegates.cs b/src/runtime/Runtime.Delegates.cs index 1e9f7ab0e..c0759b722 100644 --- a/src/runtime/Runtime.Delegates.cs +++ b/src/runtime/Runtime.Delegates.cs @@ -175,8 +175,8 @@ static Delegates() PyUnicode_AsUTF8 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF8), GetUnmanagedDll(_PythonDll)); PyUnicode_DecodeUTF16 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_DecodeUTF16), GetUnmanagedDll(_PythonDll)); PyUnicode_GetLength = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_GetLength), GetUnmanagedDll(_PythonDll)); - PyUnicode_AsUnicode = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUnicode), GetUnmanagedDll(_PythonDll)); PyUnicode_AsUTF16String = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF16String), GetUnmanagedDll(_PythonDll)); + PyUnicode_ReadChar = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_ReadChar), GetUnmanagedDll(_PythonDll)); PyUnicode_FromOrdinal = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromOrdinal), GetUnmanagedDll(_PythonDll)); PyUnicode_InternFromString = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_InternFromString), GetUnmanagedDll(_PythonDll)); PyUnicode_Compare = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_Compare), GetUnmanagedDll(_PythonDll)); @@ -454,7 +454,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF8 { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_DecodeUTF16 { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_GetLength { get; } - internal static delegate* unmanaged[Cdecl] PyUnicode_AsUnicode { get; } + internal static delegate* unmanaged[Cdecl] PyUnicode_ReadChar { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF16String { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromOrdinal { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_InternFromString { get; } diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 925b4d3db..8fe77e512 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -1370,9 +1370,10 @@ internal static IntPtr PyBytes_AsString(BorrowedReference ob) internal static nint PyUnicode_GetLength(BorrowedReference ob) => Delegates.PyUnicode_GetLength(ob); - internal static IntPtr PyUnicode_AsUnicode(BorrowedReference ob) => Delegates.PyUnicode_AsUnicode(ob); internal static NewReference PyUnicode_AsUTF16String(BorrowedReference ob) => Delegates.PyUnicode_AsUTF16String(ob); + internal static int PyUnicode_ReadChar(BorrowedReference ob, nint index) => Delegates.PyUnicode_ReadChar(ob, index); + internal static NewReference PyUnicode_FromOrdinal(int c) => Delegates.PyUnicode_FromOrdinal(c); From 32e8130bef5bdba56282a37b0b0f2bfd76642189 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 6 Dec 2025 19:26:07 +0100 Subject: [PATCH 08/14] Assign True instead of None to __clear_reentry_guard__ Not at all sure why this helps, but when assigning `None` instead, the object is gone at the time of garbage collection. (cherry picked from commit 8e0333d9affaa8b48d82c3aad28a521ecfdfad95) --- src/runtime/Types/ClassBase.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/runtime/Types/ClassBase.cs b/src/runtime/Types/ClassBase.cs index 590c870b5..790f635de 100644 --- a/src/runtime/Types/ClassBase.cs +++ b/src/runtime/Types/ClassBase.cs @@ -419,7 +419,12 @@ internal static unsafe int BaseUnmanagedClear(BorrowedReference ob) using var dict = Runtime.PyObject_GenericGetDict(ob); if (Runtime.PyMapping_HasKey(dict.Borrow(), PyIdentifier.__clear_reentry_guard__) != 0) return 0; - int res = Runtime.PyDict_SetItem(dict.Borrow(), PyIdentifier.__clear_reentry_guard__, Runtime.None); + + int res = Runtime.PyDict_SetItem( + dict.Borrow(), + PyIdentifier.__clear_reentry_guard__, + Runtime.PyTrue + ); if (res != 0) return res; res = clear(ob); From 497b21a0a27505b4e3e06d4ce4be2c8228f4eb87 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 7 Dec 2025 13:20:18 +0100 Subject: [PATCH 09/14] Move tp_clear workaround to .NET In Python 3.14, the objects __dict__ seems to already be half deconstructed, leading to crashes during garbage collection. Since gc in Python is single-threaded (I think :)), it should be fine to have a single static for this. If that is not true, we can always use a thread-local instead. (cherry picked from commit 908e13b664fe208b72b25773015cbc0e7ed97d78) --- src/runtime/Native/PyIdentifier_.cs | 3 --- src/runtime/Native/PyIdentifier_.tt | 1 - src/runtime/Types/ClassBase.cs | 26 +++++++++++--------------- tests/test_subclass.py | 1 - 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/src/runtime/Native/PyIdentifier_.cs b/src/runtime/Native/PyIdentifier_.cs index cba09ab1d..870f7952e 100644 --- a/src/runtime/Native/PyIdentifier_.cs +++ b/src/runtime/Native/PyIdentifier_.cs @@ -13,8 +13,6 @@ static class PyIdentifier public static BorrowedReference __doc__ => new(f__doc__); static IntPtr f__class__; public static BorrowedReference __class__ => new(f__class__); - static IntPtr f__clear_reentry_guard__; - public static BorrowedReference __clear_reentry_guard__ => new(f__clear_reentry_guard__); static IntPtr f__module__; public static BorrowedReference __module__ => new(f__module__); static IntPtr f__file__; @@ -53,7 +51,6 @@ static partial class InternString "__dict__", "__doc__", "__class__", - "__clear_reentry_guard__", "__module__", "__file__", "__slots__", diff --git a/src/runtime/Native/PyIdentifier_.tt b/src/runtime/Native/PyIdentifier_.tt index b431803e8..d58740cdd 100644 --- a/src/runtime/Native/PyIdentifier_.tt +++ b/src/runtime/Native/PyIdentifier_.tt @@ -7,7 +7,6 @@ "__dict__", "__doc__", "__class__", - "__clear_reentry_guard__", "__module__", "__file__", "__slots__", diff --git a/src/runtime/Types/ClassBase.cs b/src/runtime/Types/ClassBase.cs index 790f635de..d63592920 100644 --- a/src/runtime/Types/ClassBase.cs +++ b/src/runtime/Types/ClassBase.cs @@ -401,6 +401,8 @@ public static int tp_clear(BorrowedReference ob) return 0; } + static readonly HashSet ClearVisited = new(); + internal static unsafe int BaseUnmanagedClear(BorrowedReference ob) { var type = Runtime.PyObject_TYPE(ob); @@ -412,26 +414,20 @@ internal static unsafe int BaseUnmanagedClear(BorrowedReference ob) } var clear = (delegate* unmanaged[Cdecl])clearPtr; - bool usesSubtypeClear = clearPtr == TypeManager.subtype_clear; - if (usesSubtypeClear) + if (clearPtr == TypeManager.subtype_clear) { - // workaround for https://bugs.python.org/issue45266 (subtype_clear) - using var dict = Runtime.PyObject_GenericGetDict(ob); - if (Runtime.PyMapping_HasKey(dict.Borrow(), PyIdentifier.__clear_reentry_guard__) != 0) + var addr = ob.DangerousGetAddress(); + if (!ClearVisited.Add(addr)) return 0; - int res = Runtime.PyDict_SetItem( - dict.Borrow(), - PyIdentifier.__clear_reentry_guard__, - Runtime.PyTrue - ); - if (res != 0) return res; - - res = clear(ob); - Runtime.PyDict_DelItem(dict.Borrow(), PyIdentifier.__clear_reentry_guard__); + int res = clear(ob); + ClearVisited.Remove(addr); return res; } - return clear(ob); + else + { + return clear(ob); + } } protected override Dictionary OnSave(BorrowedReference ob) diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 63e7b5c2f..85c50d21a 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -291,7 +291,6 @@ def __init__(self, i, s): assert calls[0][1] == "foo" # regression test for https://github.com/pythonnet/pythonnet/issues/1565 -@pytest.mark.skipif(sys.version_info >= (3, 14), reason="Test skipped on Python 3.14 and above") def test_can_be_collected_by_gc(): from Python.Test import BaseClass From 73389fc9d442d7514a6fe39c0a348cea848e411f Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 10 May 2024 19:28:38 +0200 Subject: [PATCH 10/14] Use non-BOM encodings (#2370) * Use non-BOM encodings The documentation of the used `PyUnicode_DecodeUTF16` states that not passing `*byteorder` or passing a 0 results in the first two bytes, if they are the BOM (U+FEFF, zero-width no-break space), to be interpreted and skipped, which is incorrect when we convert a known "non BOM" string, which all strings from C# are. (cherry picked from commit 195cde67fffd06521f3bcb2294e60cad4ec506d6) --- src/embed_tests/TestPyType.cs | 2 +- src/runtime/Loader.cs | 6 ++-- src/runtime/Native/CustomMarshaler.cs | 2 +- src/runtime/Native/NativeTypeSpec.cs | 2 +- src/runtime/PythonTypes/PyType.cs | 2 +- src/runtime/Runtime.cs | 46 ++++++++++++++------------- src/runtime/Util/Encodings.cs | 10 ++++++ tests/test_conversion.py | 3 ++ 8 files changed, 44 insertions(+), 29 deletions(-) create mode 100644 src/runtime/Util/Encodings.cs diff --git a/src/embed_tests/TestPyType.cs b/src/embed_tests/TestPyType.cs index 34645747d..0470070c3 100644 --- a/src/embed_tests/TestPyType.cs +++ b/src/embed_tests/TestPyType.cs @@ -28,7 +28,7 @@ public void CanCreateHeapType() const string name = "nÁmæ"; const string docStr = "dÁcæ"; - using var doc = new StrPtr(docStr, Encoding.UTF8); + using var doc = new StrPtr(docStr, Encodings.UTF8); var spec = new TypeSpec( name: name, basicSize: Util.ReadInt32(Runtime.Runtime.PyBaseObjectType, TypeOffset.tp_basicsize), diff --git a/src/runtime/Loader.cs b/src/runtime/Loader.cs index bfb6e0d6e..555c6b3b4 100644 --- a/src/runtime/Loader.cs +++ b/src/runtime/Loader.cs @@ -12,7 +12,7 @@ public unsafe static int Initialize(IntPtr data, int size) { try { - var dllPath = Encoding.UTF8.GetString((byte*)data.ToPointer(), size); + var dllPath = Encodings.UTF8.GetString((byte*)data.ToPointer(), size); if (!string.IsNullOrEmpty(dllPath)) { @@ -43,7 +43,7 @@ public unsafe static int Initialize(IntPtr data, int size) ); return 1; } - + return 0; } @@ -51,7 +51,7 @@ public unsafe static int Shutdown(IntPtr data, int size) { try { - var command = Encoding.UTF8.GetString((byte*)data.ToPointer(), size); + var command = Encodings.UTF8.GetString((byte*)data.ToPointer(), size); if (command == "full_shutdown") { diff --git a/src/runtime/Native/CustomMarshaler.cs b/src/runtime/Native/CustomMarshaler.cs index f544756d8..8db8768b9 100644 --- a/src/runtime/Native/CustomMarshaler.cs +++ b/src/runtime/Native/CustomMarshaler.cs @@ -42,7 +42,7 @@ public int GetNativeDataSize() internal class UcsMarshaler : MarshalerBase { internal static readonly int _UCS = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? 2 : 4; - internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + internal static readonly Encoding PyEncoding = _UCS == 2 ? Encodings.UTF16 : Encodings.UTF32; private static readonly MarshalerBase Instance = new UcsMarshaler(); public override IntPtr MarshalManagedToNative(object managedObj) diff --git a/src/runtime/Native/NativeTypeSpec.cs b/src/runtime/Native/NativeTypeSpec.cs index 8b84df536..50019a148 100644 --- a/src/runtime/Native/NativeTypeSpec.cs +++ b/src/runtime/Native/NativeTypeSpec.cs @@ -17,7 +17,7 @@ public NativeTypeSpec(TypeSpec spec) { if (spec is null) throw new ArgumentNullException(nameof(spec)); - this.Name = new StrPtr(spec.Name, Encoding.UTF8); + this.Name = new StrPtr(spec.Name, Encodings.UTF8); this.BasicSize = spec.BasicSize; this.ItemSize = spec.ItemSize; this.Flags = (int)spec.Flags; diff --git a/src/runtime/PythonTypes/PyType.cs b/src/runtime/PythonTypes/PyType.cs index af796a5c5..28bda5d3e 100644 --- a/src/runtime/PythonTypes/PyType.cs +++ b/src/runtime/PythonTypes/PyType.cs @@ -53,7 +53,7 @@ public string Name { RawPointer = Util.ReadIntPtr(this, TypeOffset.tp_name), }; - return namePtr.ToString(System.Text.Encoding.UTF8)!; + return namePtr.ToString(Encodings.UTF8)!; } } diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 8fe77e512..c142fcea6 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -842,13 +842,13 @@ public static int Py_Main(int argc, string[] argv) internal static int PyRun_SimpleString(string code) { - using var codePtr = new StrPtr(code, Encoding.UTF8); + using var codePtr = new StrPtr(code, Encodings.UTF8); return Delegates.PyRun_SimpleStringFlags(codePtr, Utf8String); } internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals) { - using var codePtr = new StrPtr(code, Encoding.UTF8); + using var codePtr = new StrPtr(code, Encodings.UTF8); return Delegates.PyRun_StringFlags(codePtr, st, globals, locals, Utf8String); } @@ -860,14 +860,14 @@ internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedR /// internal static NewReference Py_CompileString(string str, string file, int start) { - using var strPtr = new StrPtr(str, Encoding.UTF8); + using var strPtr = new StrPtr(str, Encodings.UTF8); using var fileObj = new PyString(file); return Delegates.Py_CompileStringObject(strPtr, fileObj, start, Utf8String, -1); } internal static NewReference PyImport_ExecCodeModule(string name, BorrowedReference code) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyImport_ExecCodeModule(namePtr, code); } @@ -914,13 +914,13 @@ internal static bool PyObject_IsIterable(BorrowedReference ob) internal static int PyObject_HasAttrString(BorrowedReference pointer, string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyObject_HasAttrString(pointer, namePtr); } internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyObject_GetAttrString(pointer, namePtr); } @@ -931,12 +931,12 @@ internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, S internal static int PyObject_DelAttr(BorrowedReference @object, BorrowedReference name) => Delegates.PyObject_SetAttr(@object, name, null); internal static int PyObject_DelAttrString(BorrowedReference @object, string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyObject_SetAttrString(@object, namePtr, null); } internal static int PyObject_SetAttrString(BorrowedReference @object, string name, BorrowedReference value) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyObject_SetAttrString(@object, namePtr, value); } @@ -1144,7 +1144,7 @@ internal static bool PyLong_Check(BorrowedReference ob) internal static NewReference PyLong_FromString(string value, int radix) { - using var valPtr = new StrPtr(value, Encoding.UTF8); + using var valPtr = new StrPtr(value, Encodings.UTF8); return Delegates.PyLong_FromString(valPtr, IntPtr.Zero, radix); } @@ -1332,12 +1332,14 @@ internal static bool PyString_Check(BorrowedReference ob) internal static NewReference PyString_FromString(string value) { + int byteorder = BitConverter.IsLittleEndian ? -1 : 1; + int* byteorderPtr = &byteorder; fixed(char* ptr = value) return Delegates.PyUnicode_DecodeUTF16( (IntPtr)ptr, value.Length * sizeof(Char), IntPtr.Zero, - IntPtr.Zero + (IntPtr)byteorderPtr ); } @@ -1352,7 +1354,7 @@ internal static NewReference EmptyPyBytes() internal static NewReference PyByteArray_FromStringAndSize(IntPtr strPtr, nint len) => Delegates.PyByteArray_FromStringAndSize(strPtr, len); internal static NewReference PyByteArray_FromStringAndSize(string s) { - using var ptr = new StrPtr(s, Encoding.UTF8); + using var ptr = new StrPtr(s, Encodings.UTF8); return PyByteArray_FromStringAndSize(ptr.RawPointer, checked((nint)ptr.ByteCount)); } @@ -1380,7 +1382,7 @@ internal static IntPtr PyBytes_AsString(BorrowedReference ob) internal static NewReference PyUnicode_InternFromString(string s) { - using var ptr = new StrPtr(s, Encoding.UTF8); + using var ptr = new StrPtr(s, Encodings.UTF8); return Delegates.PyUnicode_InternFromString(ptr); } @@ -1472,7 +1474,7 @@ internal static bool PyDict_Check(BorrowedReference ob) internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key) { - using var keyStr = new StrPtr(key, Encoding.UTF8); + using var keyStr = new StrPtr(key, Encodings.UTF8); return Delegates.PyDict_GetItemString(pointer, keyStr); } @@ -1488,7 +1490,7 @@ internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer /// internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value) { - using var keyPtr = new StrPtr(key, Encoding.UTF8); + using var keyPtr = new StrPtr(key, Encodings.UTF8); return Delegates.PyDict_SetItemString(dict, keyPtr, value); } @@ -1497,7 +1499,7 @@ internal static int PyDict_SetItemString(BorrowedReference dict, string key, Bor internal static int PyDict_DelItemString(BorrowedReference pointer, string key) { - using var keyPtr = new StrPtr(key, Encoding.UTF8); + using var keyPtr = new StrPtr(key, Encodings.UTF8); return Delegates.PyDict_DelItemString(pointer, keyPtr); } @@ -1612,7 +1614,7 @@ internal static bool PyIter_Check(BorrowedReference ob) internal static NewReference PyModule_New(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyModule_New(namePtr); } @@ -1626,7 +1628,7 @@ internal static NewReference PyModule_New(string name) /// Return -1 on error, 0 on success. internal static int PyModule_AddObject(BorrowedReference module, string name, StolenReference value) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); IntPtr valueAddr = value.DangerousGetAddressOrNull(); int res = Delegates.PyModule_AddObject(module, namePtr, valueAddr); // We can't just exit here because the reference is stolen only on success. @@ -1644,7 +1646,7 @@ internal static int PyModule_AddObject(BorrowedReference module, string name, St internal static NewReference PyImport_ImportModule(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyImport_ImportModule(namePtr); } @@ -1653,7 +1655,7 @@ internal static NewReference PyImport_ImportModule(string name) internal static BorrowedReference PyImport_AddModule(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyImport_AddModule(namePtr); } @@ -1681,13 +1683,13 @@ internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath) internal static BorrowedReference PySys_GetObject(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PySys_GetObject(namePtr); } internal static int PySys_SetObject(string name, BorrowedReference ob) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PySys_SetObject(namePtr, ob); } @@ -1784,7 +1786,7 @@ internal static IntPtr PyMem_Malloc(long size) internal static void PyErr_SetString(BorrowedReference ob, string message) { - using var msgPtr = new StrPtr(message, Encoding.UTF8); + using var msgPtr = new StrPtr(message, Encodings.UTF8); Delegates.PyErr_SetString(ob, msgPtr); } diff --git a/src/runtime/Util/Encodings.cs b/src/runtime/Util/Encodings.cs new file mode 100644 index 000000000..d5a0c6ff8 --- /dev/null +++ b/src/runtime/Util/Encodings.cs @@ -0,0 +1,10 @@ +using System; +using System.Text; + +namespace Python.Runtime; + +static class Encodings { + public static System.Text.Encoding UTF8 = new UTF8Encoding(false, true); + public static System.Text.Encoding UTF16 = new UnicodeEncoding(!BitConverter.IsLittleEndian, false, true); + public static System.Text.Encoding UTF32 = new UTF32Encoding(!BitConverter.IsLittleEndian, false, true); +} diff --git a/tests/test_conversion.py b/tests/test_conversion.py index 163d26dbc..ae2b0f18a 100644 --- a/tests/test_conversion.py +++ b/tests/test_conversion.py @@ -532,6 +532,9 @@ def test_string_conversion(): ob.StringField = System.String(u'\uffff\uffff') assert ob.StringField == u'\uffff\uffff' + ob.StringField = System.String("\ufeffbom") + assert ob.StringField == "\ufeffbom" + ob.StringField = None assert ob.StringField is None From bfdfebb5f489fa5de6d7c95f73a00f5cf545ff53 Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Fri, 19 Jun 2026 15:37:33 +0000 Subject: [PATCH 11/14] Preserve SyntaxError source line in message on Python 3.12+ Python 3.12 eagerly normalizes the error indicator, so PyErr_Fetch now hands us the SyntaxError instance (whose str() omits the offending source line) instead of the raw args tuple (whose str() included it). Callers that surface PythonException.Message for compile diagnostics therefore lost the offending source text on 3.12+. GetMessage now re-appends the SyntaxError 'text' attribute when present. This is a no-op on <=3.11 (there the fetched value is a tuple without the SyntaxError attributes) and only affects SyntaxError messages. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/runtime/PythonException.cs | 51 +++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/src/runtime/PythonException.cs b/src/runtime/PythonException.cs index 14a8d54d1..89737855e 100644 --- a/src/runtime/PythonException.cs +++ b/src/runtime/PythonException.cs @@ -252,12 +252,61 @@ private static string GetMessage(PyObject? value, PyType type) if (value != null && !value.IsNone()) { - return value.ToString() ?? "no message"; + var message = value.ToString() ?? "no message"; + + // Python 3.12+ eagerly normalizes the error indicator, so a SyntaxError + // reaches us as the exception instance whose str() omits the offending + // source line. Pre-3.12 we received the raw args tuple, whose str() + // included it. Re-append the source text so the message stays complete + // for callers that surface it (e.g. compile diagnostics). This is a + // no-op on <=3.11 (there 'value' is a tuple without these attributes). + if (TryGetSyntaxErrorText(value, out var sourceText)) + { + message = $"{message}: {sourceText}"; + } + + return message; } return type.Name; } + /// + /// If is a SyntaxError instance carrying the offending + /// source line (its text attribute), returns that trimmed text. + /// + private static bool TryGetSyntaxErrorText(PyObject value, out string text) + { + text = string.Empty; + try + { + // 'msg' + 'text' is the distinctive SyntaxError shape; bail otherwise. + if (!value.HasAttr("msg") || !value.HasAttr("text")) + { + return false; + } + + using var textObj = value.GetAttr("text"); + if (textObj.IsNone()) + { + return false; + } + + var sourceLine = textObj.ToString(); + if (string.IsNullOrWhiteSpace(sourceLine)) + { + return false; + } + + text = sourceLine.Trim(); + return true; + } + catch (PythonException) + { + return false; + } + } + private static string TracebackToString(PyObject traceback) { if (traceback is null) From d1cccc3f937ecfd6dbfc0c1b44b72b1e3904d45e Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Fri, 19 Jun 2026 15:37:33 +0000 Subject: [PATCH 12/14] Make embed tests compatible with Python 3.12+ behavior changes Three CPython behavior changes surfaced as failures/crashes once the overload-resolution crash was fixed, all on 3.12+: - ClassManagerTests.BindsCorrectOverloadForClassName crashed the host with "Python memory allocator called without holding the GIL". TestClass2's Get(PyObject o) re-enters Python via ToPython() while MethodBinder has released the GIL (allow_threads) around the managed call. A managed callback that re-enters Python must re-acquire the GIL; tolerated on <=3.11, fatal on 3.12+. Wrap the body in using (Py.GIL()). - TestGetsPythonCodeInfoInStackTrace[ForNestedInterop]: 3.12+ adds caret indicator lines (e.g. "~~~^^^") under source lines in tracebacks, shifting the positional assertions. Drop caret-only lines before asserting (no-op on <=3.11). - Codecs.ExceptionDecodedNoInstance: 3.12 eagerly normalizes exceptions, so the error indicator always carries an instance ("value"); the instanceless scenario this decoder targets can no longer be produced. Guard the test to <3.12. Verified: full embed suite green on 3.11 (910/910) with these changes; the three previously-failing tests pass on 3.14. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/embed_tests/ClassManagerTests.cs | 9 ++++++++- src/embed_tests/Codecs.cs | 8 ++++++++ src/embed_tests/TestPythonException.cs | 14 ++++++++++++-- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/embed_tests/ClassManagerTests.cs b/src/embed_tests/ClassManagerTests.cs index 264509c2a..f1af2c22a 100644 --- a/src/embed_tests/ClassManagerTests.cs +++ b/src/embed_tests/ClassManagerTests.cs @@ -965,7 +965,14 @@ private class TestClass2 : TestClass1 { public PyObject Get(PyObject o) { - return "PyObject Get(PyObject o)".ToPython(); + // This managed method is invoked by pythonnet with the GIL released + // (MethodBinder uses allow_threads around managed calls). Re-entering + // Python here - creating a str via ToPython() - requires re-acquiring + // the GIL; on CPython 3.12+ allocating without the GIL is fatal. + using (Py.GIL()) + { + return "PyObject Get(PyObject o)".ToPython(); + } } public dynamic Get(Type t) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index 5f452a5e8..7742a19d4 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -361,6 +361,14 @@ from datetime import datetime [Test] public void ExceptionDecodedNoInstance() { + if (Runtime.PyVersion >= new Version(3, 12)) + { + // Python 3.12+ eagerly normalizes the error indicator, so an exception + // always reaches the decoder with an instance ("value"). The instanceless + // error scenario this decoder targets can no longer be produced by CPython. + Assert.Ignore("Instanceless exceptions are not produced on Python 3.12+ (eager normalization)."); + } + PyObjectConversions.RegisterDecoder(new InstancelessExceptionDecoder()); using var scope = Py.CreateScope(); var error = Assert.Throws(() => PythonEngine.Exec( diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 107f20f53..a5c2353a6 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -235,7 +235,12 @@ def CallThrow(self): { Assert.AreEqual("Test Exception Message", ex.InnerException.Message); - var pythonTracebackLines = ex.PythonTraceback.TrimEnd('\n').Split('\n').Select(x => x.Trim()).ToList(); + var pythonTracebackLines = ex.PythonTraceback.TrimEnd('\n').Split('\n').Select(x => x.Trim()) + // Python 3.12+ adds caret indicator lines (e.g. "~~~^^^") under the offending + // source code in tracebacks. Drop those so positional assertions below stay + // version-agnostic (no-op on <=3.11, which doesn't emit them). + .Where(x => !(x.Length > 0 && x.All(c => c == '~' || c == '^'))) + .ToList(); Assert.AreEqual(5, pythonTracebackLines.Count); Assert.AreEqual("File \"none\", line 9, in CallThrow", pythonTracebackLines[0]); @@ -298,7 +303,12 @@ def CallThrow(): { Assert.AreEqual("Test Exception Message", ex.InnerException.Message); - var pythonTracebackLines = ex.PythonTraceback.TrimEnd('\n').Split('\n').Select(x => x.Trim()).ToList(); + var pythonTracebackLines = ex.PythonTraceback.TrimEnd('\n').Split('\n').Select(x => x.Trim()) + // Python 3.12+ adds caret indicator lines (e.g. "~~~^^^") under the offending + // source code in tracebacks. Drop those so positional assertions below stay + // version-agnostic (no-op on <=3.11, which doesn't emit them). + .Where(x => !(x.Length > 0 && x.All(c => c == '~' || c == '^'))) + .ToList(); Assert.AreEqual(4, pythonTracebackLines.Count); Assert.IsTrue(new[] From 67e0f8060960ab15e817233a879568460a33466c Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Fri, 19 Jun 2026 16:26:29 +0000 Subject: [PATCH 13/14] Add Python 3.12 / 3.13 ABI offsets and CI jobs The fork resolves PyTypeObject field offsets from the hardcoded TypeOffset{major}{minor} tables (it does not run geninterop at build), so a missing table makes ABI.Initialize throw "Python ABI v... is not supported" and every test on that version fails at PythonEngine init. Only 3.6-3.11 and 3.14 tables were present. Vendor the 3.12 and 3.13 tables from pythonnet/pythonnet upstream (byte-identical to upstream master; same source as the already-present TypeOffset314) and add 3.12 + 3.13 to the CI matrix. Local verification (uv standalone CPython 3.13, this branch's fixes): embed suite ABI-initializes correctly and runs 847 passed / 0 failed (parity with 3.14). 3.12 table is vendored from the same authoritative source; CI exercises it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/main.yml | 2 +- src/runtime/Native/TypeOffset312.cs | 144 ++++++++++++++++++++++++++ src/runtime/Native/TypeOffset313.cs | 152 ++++++++++++++++++++++++++++ 3 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 src/runtime/Native/TypeOffset312.cs create mode 100644 src/runtime/Native/TypeOffset313.cs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5f05ea997..e281ec1b8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - python: ["3.8", "3.9", "3.10", "3.11", "3.14"] + python: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - name: Checkout code diff --git a/src/runtime/Native/TypeOffset312.cs b/src/runtime/Native/TypeOffset312.cs new file mode 100644 index 000000000..8ba30e816 --- /dev/null +++ b/src/runtime/Native/TypeOffset312.cs @@ -0,0 +1,144 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.12: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset312 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset312() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + public int tp_watched { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int am_send { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + public int _ht_tpname { get; private set; } + public int spec_cache_getitem { get; private set; } + public int getitem_version { get; private set; } + } +} + diff --git a/src/runtime/Native/TypeOffset313.cs b/src/runtime/Native/TypeOffset313.cs new file mode 100644 index 000000000..4c2e71295 --- /dev/null +++ b/src/runtime/Native/TypeOffset313.cs @@ -0,0 +1,152 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.13: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset313 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset313() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + // This is an error in our generator: + // + // The fields below are actually not pointers (like we incorrectly + // assume for all other fields) but instead a char (1 byte) and a short + // (2 bytes). By dropping one of the fields, we still get the correct + // overall size of the struct. + public int tp_watched { get; private set; } + // public int tp_versions_used { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int am_send { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + public int _ht_tpname { get; private set; } + public int spec_cache_getitem { get; private set; } + public int getitem_version { get; private set; } + public int init { get; private set; } + } +} + From 97183994cc966cdc21b219529d67087911c44e5f Mon Sep 17 00:00:00 2001 From: Martin-Molinero Date: Fri, 19 Jun 2026 19:28:29 +0000 Subject: [PATCH 14/14] Support Python 3.13/3.14 re-init in tests; drop obsolete TestDomainReload The embed-test suite re-initializes the interpreter per fixture. On CPython 3.13/3.14 the suite aborted with "Failed to import encodings module" - not a filesystem problem (strace shows the file opens fine) but interpreter import state corrupted across re-initialization. Root cause isolated to a single test: TestPythonEngineProperties.SetPythonPath. It uses PythonEngine.PythonPath, which pins a fixed module search path via the deprecated Py_SetPath. CPython 3.13+ keeps that path config in _PyRuntime across Py_Finalize and offers no way to reset it back to auto-computation without the PyConfig API, so once this test runs every later re-initialization in the same process is forced onto the pinned path and eventually cannot bootstrap encodings. All other fixtures - including the normal Initialize/Shutdown cycles in pyinitialize and TestFinalizer - run fine in a single process. Run only SetPythonPath in its own test process so it cannot pollute the rest of the suite. Verified locally: full embed suite green on 3.11, 3.13 and 3.14 (main run 908 / SetPythonPath 1, 0 failures); no regression. Also delete TestDomainReload: AppDomain reload is not supported on modern .NET (single-domain), so those tests (MarshalByRefObject / AppDomain.CreateDomain) are obsolete. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/main.yml | 11 +- src/embed_tests/TestDomainReload.cs | 403 ---------------------------- 2 files changed, 10 insertions(+), 404 deletions(-) delete mode 100644 src/embed_tests/TestDomainReload.cs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e281ec1b8..91d3d5a29 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,7 +49,16 @@ jobs: echo PYTHONHOME=$(python -c 'import sys; print(sys.prefix)') >> $GITHUB_ENV - name: Embedding tests - run: dotnet test --runtime any-x64 --logger "console;verbosity=detailed" src/embed_tests/ + run: dotnet test --runtime any-x64 --logger "console;verbosity=detailed" src/embed_tests/ --filter "FullyQualifiedName!~SetPythonPath" + + # SetPythonPath exercises PythonEngine.PythonPath, which uses the deprecated Py_SetPath + # to pin a fixed module search path. CPython 3.13+ keeps that path config in _PyRuntime + # across Py_Finalize and provides no way to reset it back to auto-computation without the + # PyConfig API, so once this test runs, every later interpreter re-initialization in the + # same process is forced onto the pinned path and eventually fails to import encodings. + # Run it in its own process so it cannot pollute the rest of the suite. + - name: Embedding tests (SetPythonPath, isolated process) + run: dotnet test --runtime any-x64 --logger "console;verbosity=detailed" src/embed_tests/ --filter "FullyQualifiedName~SetPythonPath" - name: Python Tests (.NET Core) run: pytest --runtime netcore tests diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs deleted file mode 100644 index 498119d1e..000000000 --- a/src/embed_tests/TestDomainReload.cs +++ /dev/null @@ -1,403 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Reflection; -using System.Runtime.InteropServices; -using NUnit.Framework; -using Python.Runtime; - -using PyRuntime = Python.Runtime.Runtime; -// -// This test case is disabled on .NET Standard because it doesn't have all the -// APIs we use. We could work around that, but .NET Core doesn't implement -// domain creation, so it's not worth it. -// -// Unfortunately this means no continuous integration testing for this case. -// -#if NETFRAMEWORK -namespace Python.EmbeddingTest -{ - class TestDomainReload - { - abstract class CrossCaller : MarshalByRefObject - { - public abstract ValueType Execute(ValueType arg); - } - - - /// - /// Test that the python runtime can survive a C# domain reload without crashing. - /// - /// At the time this test was written, there was a very annoying - /// seemingly random crash bug when integrating pythonnet into Unity. - /// - /// The repro steps that David Lassonde, Viktoria Kovecses and - /// Benoit Hudson eventually worked out: - /// 1. Write a HelloWorld.cs script that uses Python.Runtime to access - /// some C# data from python: C# calls python, which calls C#. - /// 2. Execute the script (e.g. make it a MenuItem and click it). - /// 3. Touch HelloWorld.cs on disk, forcing Unity to recompile scripts. - /// 4. Wait several seconds for Unity to be done recompiling and - /// reloading the C# domain. - /// 5. Make python run the gc (e.g. by calling gc.collect()). - /// - /// The reason: - /// A. In step 2, Python.Runtime registers a bunch of new types with - /// their tp_traverse slot pointing to managed code, and allocates - /// some objects of those types. - /// B. In step 4, Unity unloads the C# domain. That frees the managed - /// code. But at the time of the crash investigation, pythonnet - /// leaked the python side of the objects allocated in step 1. - /// C. In step 5, python sees some pythonnet objects in its gc list of - /// potentially-leaked objects. It calls tp_traverse on those objects. - /// But tp_traverse was freed in step 3 => CRASH. - /// - /// This test distills what's going on without needing Unity around (we'd see - /// similar behaviour if we were using pythonnet on a .NET web server that did - /// a hot reload). - /// - [Test] - public static void DomainReloadAndGC() - { - Assert.IsFalse(PythonEngine.IsInitialized); - RunAssemblyAndUnload("test1"); - Assert.That(PyRuntime.Py_IsInitialized() != 0, - "On soft-shutdown mode, Python runtime should still running"); - - RunAssemblyAndUnload("test2"); - Assert.That(PyRuntime.Py_IsInitialized() != 0, - "On soft-shutdown mode, Python runtime should still running"); - } - - #region CrossDomainObject - - class CrossDomainObjectStep1 : CrossCaller - { - public override ValueType Execute(ValueType arg) - { - try - { - // Create a C# user-defined object in Python. Asssing some values. - Type type = typeof(Python.EmbeddingTest.Domain.MyClass); - string code = string.Format(@" -import clr -clr.AddReference('{0}') - -from Python.EmbeddingTest.Domain import MyClass -obj = MyClass() -obj.Method() -obj.StaticMethod() -obj.Property = 1 -obj.Field = 10 -", Assembly.GetExecutingAssembly().FullName); - - using (Py.GIL()) - using (var scope = Py.CreateScope()) - { - scope.Exec(code); - using (PyObject obj = scope.Get("obj")) - { - Debug.Assert(obj.AsManagedObject(type).GetType() == type); - // We only needs its Python handle - PyRuntime.XIncref(obj); - return obj.Handle; - } - } - } - catch (Exception e) - { - Debug.WriteLine(e); - throw; - } - } - } - - - class CrossDomainObjectStep2 : CrossCaller - { - public override ValueType Execute(ValueType arg) - { - // handle refering a clr object created in previous domain, - // it should had been deserialized and became callable agian. - using var handle = NewReference.DangerousFromPointer((IntPtr)arg); - try - { - using (Py.GIL()) - { - BorrowedReference tp = Runtime.Runtime.PyObject_TYPE(handle.Borrow()); - IntPtr tp_clear = Util.ReadIntPtr(tp, TypeOffset.tp_clear); - Assert.That(tp_clear, Is.Not.Null); - - using (PyObject obj = new PyObject(handle.Steal())) - { - obj.InvokeMethod("Method"); - obj.InvokeMethod("StaticMethod"); - - using (var scope = Py.CreateScope()) - { - scope.Set("obj", obj); - scope.Exec(@" -obj.Method() -obj.StaticMethod() -obj.Property += 1 -obj.Field += 10 -"); - } - var clrObj = obj.As(); - Assert.AreEqual(clrObj.Property, 2); - Assert.AreEqual(clrObj.Field, 20); - } - } - } - catch (Exception e) - { - Debug.WriteLine(e); - throw; - } - return 0; - } - } - - /// - /// Create a C# custom object in a domain, in python code. - /// Unload the domain, create a new domain. - /// Make sure the C# custom object created in the previous domain has been re-created - /// - [Test] - public static void CrossDomainObject() - { - RunDomainReloadSteps(); - } - - #endregion - - /// - /// This is a magic incantation required to run code in an application - /// domain other than the current one. - /// - class Proxy : MarshalByRefObject - { - public void RunPython() - { - Console.WriteLine("[Proxy] Entering RunPython"); - PythonRunner.RunPython(); - Console.WriteLine("[Proxy] Leaving RunPython"); - } - - public object Call(string methodName, params object[] args) - { - var pythonrunner = typeof(PythonRunner); - var method = pythonrunner.GetMethod(methodName); - return method.Invoke(null, args); - } - } - - static T CreateInstanceInstanceAndUnwrap(AppDomain domain) - { - Type type = typeof(T); - var theProxy = (T)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); - return theProxy; - } - - /// - /// Create a domain, run the assembly in it (the RunPython function), - /// and unload the domain. - /// - static void RunAssemblyAndUnload(string domainName) - { - Console.WriteLine($"[Program.Main] === creating domain {domainName}"); - - AppDomain domain = CreateDomain(domainName); - // Create a Proxy object in the new domain, where we want the - // assembly (and Python .NET) to reside - var theProxy = CreateInstanceInstanceAndUnwrap(domain); - - theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL); - // From now on use the Proxy to call into the new assembly - theProxy.RunPython(); - - theProxy.Call("ShutdownPython"); - Console.WriteLine($"[Program.Main] Before Domain Unload on {domainName}"); - AppDomain.Unload(domain); - Console.WriteLine($"[Program.Main] After Domain Unload on {domainName}"); - - // Validate that the assembly does not exist anymore - try - { - Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); - Assert.Fail($"{theProxy} should be invlaid now"); - } - catch (AppDomainUnloadedException) - { - Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); - } - } - - private static AppDomain CreateDomain(string name) - { - // Create the domain. Make sure to set PrivateBinPath to a relative - // path from the CWD (namely, 'bin'). - // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain - var currentDomain = AppDomain.CurrentDomain; - var domainsetup = new AppDomainSetup() - { - ApplicationBase = currentDomain.SetupInformation.ApplicationBase, - ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, - LoaderOptimization = LoaderOptimization.SingleDomain, - PrivateBinPath = "." - }; - var domain = AppDomain.CreateDomain( - $"My Domain {name}", - currentDomain.Evidence, - domainsetup); - return domain; - } - - /// - /// Resolves the assembly. Why doesn't this just work normally? - /// - static Assembly ResolveAssembly(object sender, ResolveEventArgs args) - { - var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); - - foreach (var assembly in loadedAssemblies) - { - if (assembly.FullName == args.Name) - { - return assembly; - } - } - - return null; - } - - static void RunDomainReloadSteps() where T1 : CrossCaller where T2 : CrossCaller - { - ValueType arg = null; - Type type = typeof(Proxy); - { - AppDomain domain = CreateDomain("test_domain_reload_1"); - try - { - var theProxy = CreateInstanceInstanceAndUnwrap(domain); - theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL); - - var caller = CreateInstanceInstanceAndUnwrap(domain); - arg = caller.Execute(arg); - - theProxy.Call("ShutdownPython"); - } - finally - { - AppDomain.Unload(domain); - } - } - - { - AppDomain domain = CreateDomain("test_domain_reload_2"); - try - { - var theProxy = CreateInstanceInstanceAndUnwrap(domain); - theProxy.Call(nameof(PythonRunner.InitPython), PyRuntime.PythonDLL); - - var caller = CreateInstanceInstanceAndUnwrap(domain); - caller.Execute(arg); - theProxy.Call("ShutdownPythonCompletely"); - } - finally - { - AppDomain.Unload(domain); - } - } - - Assert.IsTrue(PyRuntime.Py_IsInitialized() != 0); - } - } - - - // - // The code we'll test. All that really matters is - // using GIL { Python.Exec(pyScript); } - // but the rest is useful for debugging. - // - // What matters in the python code is gc.collect and clr.AddReference. - // - // Note that the language version is 2.0, so no $"foo{bar}" syntax. - // - static class PythonRunner - { - public static void RunPython() - { - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - string name = AppDomain.CurrentDomain.FriendlyName; - Console.WriteLine("[{0} in .NET] In PythonRunner.RunPython", name); - using (Py.GIL()) - { - try - { - var pyScript = string.Format("import clr\n" - + "print('[{0} in python] imported clr')\n" - + "clr.AddReference('System')\n" - + "print('[{0} in python] allocated a clr object')\n" - + "import gc\n" - + "gc.collect()\n" - + "print('[{0} in python] collected garbage')\n", - name); - PythonEngine.Exec(pyScript); - } - catch (Exception e) - { - Console.WriteLine(string.Format("[{0} in .NET] Caught exception: {1}", name, e)); - throw; - } - } - } - - - private static IntPtr _state; - - public static void InitPython(string dllName) - { - PyRuntime.PythonDLL = dllName; - PythonEngine.Initialize(); - _state = PythonEngine.BeginAllowThreads(); - } - - public static void ShutdownPython() - { - PythonEngine.EndAllowThreads(_state); - PythonEngine.Shutdown(); - } - - public static void ShutdownPythonCompletely() - { - PythonEngine.EndAllowThreads(_state); - - PythonEngine.Shutdown(); - } - - static void OnDomainUnload(object sender, EventArgs e) - { - Console.WriteLine(string.Format("[{0} in .NET] unloading", AppDomain.CurrentDomain.FriendlyName)); - } - } - -} - - -namespace Python.EmbeddingTest.Domain -{ - [Serializable] - public class MyClass - { - public int Property { get; set; } - public int Field; - public void Method() { } - public static void StaticMethod() { } - } -} - - -#endif