From 12718257622d19d98a96a861ebeedf938e3a339a Mon Sep 17 00:00:00 2001 From: midnightstardust1203 Date: Fri, 2 Aug 2024 22:31:47 +0800 Subject: [PATCH 1/6] Remove leading whitespace email --- Lib/email/_policybase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index c7694a44e26639..58f3eb66d465a4 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -307,7 +307,7 @@ def header_source_parse(self, sourcelines): """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = value.lstrip('\t') + ''.join(sourcelines[1:]) return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): From 425e7638ef5936630dd457656cbaa6365cb128cf Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:37:05 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2024-08-02-14-37-04.gh-issue-109252._NyO6i.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2024-08-02-14-37-04.gh-issue-109252._NyO6i.rst diff --git a/Misc/NEWS.d/next/Library/2024-08-02-14-37-04.gh-issue-109252._NyO6i.rst b/Misc/NEWS.d/next/Library/2024-08-02-14-37-04.gh-issue-109252._NyO6i.rst new file mode 100644 index 00000000000000..481e58795ae393 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-02-14-37-04.gh-issue-109252._NyO6i.rst @@ -0,0 +1 @@ +Fixed an existing issue with email header lib adding whitespace From 4b63cd170e5dd840bffc80922f09f2d69932ff5c Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 2 Aug 2024 12:11:44 -0400 Subject: [PATCH 3/6] gh-122527: Fix a crash on deallocation of `PyStructSequence` (GH-122577) The `PyStructSequence` destructor would crash if it was deallocated after its type's dictionary was cleared by the GC, because it couldn't compute the "real size" of the instance. This could occur with relatively straightforward code in the free-threaded build or with a reference cycle involving the type in the default build, due to differing orders in which `tp_clear()` was called. Account for the non-sequence fields in `tp_basicsize` and use that, along with `Py_SIZE()`, to compute the "real" size of a `PyStructSequence` in the dealloc function. This avoids the accesses to the type's dictionary during dealloc, which were unsafe. --- Lib/test/test_structseq.py | 13 +++++++++ Lib/test/test_sys.py | 3 +- ...-08-01-19-13-58.gh-issue-122527.eztso6.rst | 4 +++ Objects/structseq.c | 28 +++++++++++++++---- 4 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-08-01-19-13-58.gh-issue-122527.eztso6.rst diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index 6aec63e2603412..d0bc0bd7b61520 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -2,8 +2,10 @@ import os import pickle import re +import textwrap import time import unittest +from test.support import script_helper class StructSeqTest(unittest.TestCase): @@ -342,6 +344,17 @@ def test_copy_replace_with_unnamed_fields(self): with self.assertRaisesRegex(TypeError, error_message): copy.replace(r, st_mode=1, error=2) + def test_reference_cycle(self): + # gh-122527: Check that a structseq that's part of a reference cycle + # with its own type doesn't crash. Previously, if the type's dictionary + # was cleared first, the structseq instance would crash in the + # destructor. + script_helper.assert_python_ok("-c", textwrap.dedent(r""" + import time + t = time.gmtime() + type(t).refcyle = t + """)) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 7e4bc980b390f7..709355e293f2fc 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1823,7 +1823,8 @@ def test_pythontypes(self): # symtable entry # XXX # sys.flags - check(sys.flags, vsize('') + self.P * len(sys.flags)) + # FIXME: The +1 will not be necessary once gh-122575 is fixed + check(sys.flags, vsize('') + self.P * (1 + len(sys.flags))) def test_asyncgen_hooks(self): old = sys.get_asyncgen_hooks() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-01-19-13-58.gh-issue-122527.eztso6.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-01-19-13-58.gh-issue-122527.eztso6.rst new file mode 100644 index 00000000000000..f697ed99d0c523 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-01-19-13-58.gh-issue-122527.eztso6.rst @@ -0,0 +1,4 @@ +Fix a crash that occurred when a ``PyStructSequence`` was deallocated after +its type's dictionary was cleared by the GC. The type's +:c:member:`~PyTypeObject.tp_basicsize` now accounts for non-sequence fields +that aren't included in the :c:macro:`Py_SIZE` of the sequence. diff --git a/Objects/structseq.c b/Objects/structseq.c index d8289f2638db0f..94f09b3ee0a337 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -41,12 +41,20 @@ get_type_attr_as_size(PyTypeObject *tp, PyObject *name) get_type_attr_as_size(tp, &_Py_ID(n_sequence_fields)) #define REAL_SIZE_TP(tp) \ get_type_attr_as_size(tp, &_Py_ID(n_fields)) -#define REAL_SIZE(op) REAL_SIZE_TP(Py_TYPE(op)) +#define REAL_SIZE(op) get_real_size((PyObject *)op) #define UNNAMED_FIELDS_TP(tp) \ get_type_attr_as_size(tp, &_Py_ID(n_unnamed_fields)) #define UNNAMED_FIELDS(op) UNNAMED_FIELDS_TP(Py_TYPE(op)) +static Py_ssize_t +get_real_size(PyObject *op) +{ + // Compute the real size from the visible size (i.e., Py_SIZE()) and the + // number of non-sequence fields accounted for in tp_basicsize. + Py_ssize_t hidden = Py_TYPE(op)->tp_basicsize - offsetof(PyStructSequence, ob_item); + return Py_SIZE(op) + hidden / sizeof(PyObject *); +} PyObject * PyStructSequence_New(PyTypeObject *type) @@ -120,6 +128,9 @@ structseq_dealloc(PyStructSequence *obj) PyObject_GC_UnTrack(obj); PyTypeObject *tp = Py_TYPE(obj); + // gh-122527: We can't use REAL_SIZE_TP() or any macros that access the + // type's dictionary here, because the dictionary may have already been + // cleared by the garbage collector. size = REAL_SIZE(obj); for (i = 0; i < size; ++i) { Py_XDECREF(obj->ob_item[i]); @@ -565,10 +576,14 @@ initialize_members(PyStructSequence_Desc *desc, static void initialize_static_fields(PyTypeObject *type, PyStructSequence_Desc *desc, - PyMemberDef *tp_members, unsigned long tp_flags) + PyMemberDef *tp_members, Py_ssize_t n_members, + unsigned long tp_flags) { type->tp_name = desc->name; - type->tp_basicsize = sizeof(PyStructSequence) - sizeof(PyObject *); + // Account for hidden members in tp_basicsize because they are not + // included in the variable size. + Py_ssize_t n_hidden = n_members - desc->n_in_sequence; + type->tp_basicsize = sizeof(PyStructSequence) + (n_hidden - 1) * sizeof(PyObject *); type->tp_itemsize = sizeof(PyObject *); type->tp_dealloc = (destructor)structseq_dealloc; type->tp_repr = (reprfunc)structseq_repr; @@ -621,7 +636,7 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, if (members == NULL) { goto error; } - initialize_static_fields(type, desc, members, tp_flags); + initialize_static_fields(type, desc, members, n_members, tp_flags); _Py_SetImmortal((PyObject *)type); } @@ -684,7 +699,7 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) if (members == NULL) { return -1; } - initialize_static_fields(type, desc, members, 0); + initialize_static_fields(type, desc, members, n_members, 0); if (initialize_static_type(type, desc, n_members, n_unnamed_members) < 0) { PyMem_Free(members); return -1; @@ -760,7 +775,8 @@ _PyStructSequence_NewType(PyStructSequence_Desc *desc, unsigned long tp_flags) /* The name in this PyType_Spec is statically allocated so it is */ /* expected that it'll outlive the PyType_Spec */ spec.name = desc->name; - spec.basicsize = sizeof(PyStructSequence) - sizeof(PyObject *); + Py_ssize_t hidden = n_members - desc->n_in_sequence; + spec.basicsize = (int)(sizeof(PyStructSequence) + (hidden - 1) * sizeof(PyObject *)); spec.itemsize = sizeof(PyObject *); spec.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | tp_flags; spec.slots = slots; From fe0a28d850943cf2ba132c9b0a933bb0c98ff0ae Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 2 Aug 2024 23:56:51 +0100 Subject: [PATCH 4/6] gh-122560: add test that comprehension loop var appears only in one scope of the symtable (#122582) --- Lib/test/test_symtable.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index bd367c1591c744..24d89b09d946ad 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -528,6 +528,27 @@ def test_symtable_entry_repr(self): self.assertEqual(repr(self.top._table), expected) +class ComprehensionTests(unittest.TestCase): + def get_identifiers_recursive(self, st, res): + res.extend(st.get_identifiers()) + for ch in st.get_children(): + self.get_identifiers_recursive(ch, res) + + def test_loopvar_in_only_one_scope(self): + # ensure that the loop variable appears only once in the symtable + comps = [ + "[x for x in [1]]", + "{x for x in [1]}", + "{x:x*x for x in [1]}", + ] + for comp in comps: + with self.subTest(comp=comp): + st = symtable.symtable(comp, "?", "exec") + ids = [] + self.get_identifiers_recursive(st, ids) + self.assertEqual(len([x for x in ids if x == 'x']), 1) + + class CommandLineTest(unittest.TestCase): maxDiff = None From f3b855392e3237fc759ef8d88f51655782ff3817 Mon Sep 17 00:00:00 2001 From: midnightstardust1203 Date: Fri, 2 Aug 2024 22:31:47 +0800 Subject: [PATCH 5/6] Remove leading whitespace email --- Lib/email/_policybase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index c7694a44e26639..58f3eb66d465a4 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -307,7 +307,7 @@ def header_source_parse(self, sourcelines): """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip(' \t') + ''.join(sourcelines[1:]) + value = value.lstrip('\t') + ''.join(sourcelines[1:]) return (name, value.rstrip('\r\n')) def header_store_parse(self, name, value): From 1a2409bdc73a40715240f36d8a857500aa9c87f7 Mon Sep 17 00:00:00 2001 From: midnightstardust1203 Date: Sat, 3 Aug 2024 10:06:47 +0800 Subject: [PATCH 6/6] added leading check --- Lib/email/_policybase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index 58f3eb66d465a4..a29f51bbe502c5 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -307,8 +307,8 @@ def header_source_parse(self, sourcelines): """ name, value = sourcelines[0].split(':', 1) - value = value.lstrip('\t') + ''.join(sourcelines[1:]) - return (name, value.rstrip('\r\n')) + value = value + ''.join(sourcelines[1:]) + return (name, value.lstrip(" \r\n\t").rstrip('\r\n')) def header_store_parse(self, name, value): """+