From 85a67bafb4e0dfc9eeef5f75d6c961e4f6231ace Mon Sep 17 00:00:00 2001 From: Neko Asakura Date: Fri, 6 Nov 2026 14:34:13 +0800 Subject: [PATCH 001/229] gh-161419: fix weakref reviving a dying object in free-threading (#161431) --- Include/internal/pycore_object.h | 2 ++ Lib/test/test_weakref.py | 24 +++++++++++++++++++ ...-11-05-10-19-48.gh-issue-161419.wc4DV0.rst | 3 +++ 3 files changed, 29 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-10-19-48.gh-issue-161419.wc4DV0.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index f6a71fe04176014..a27e13f9c460bfa 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -712,6 +712,8 @@ _PyObject_ResurrectEnd(PyObject *op) if (_Py_IsOwnedByCurrentThread(op) && local == 1 && shared == 0) { // Fast-path: object has a single refcount and is owned by this thread _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, _Py_UNOWNED_TID); + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, _Py_REF_MERGED); # ifdef Py_TRACE_REFS _Py_ForgetReference(op); # endif diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index da4155d09f73283..5b4ace36c461437 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -1,4 +1,5 @@ import gc +import io import sys import doctest import unittest @@ -1056,6 +1057,29 @@ def __del__(self): del head self.assertEqual(found, []) + def test_ref_to_object_dying_after_finalizer(self): + cleared = [] + class Attr: + def __del__(self): + cleared.append(self.wr() is None) + + class SetWithDel(set): + __slots__ = ('a',) + def __del__(self): + pass + + class Raw(io.RawIOBase): + __slots__ = ('a',) + + for cls in (SetWithDel, Raw): + with self.subTest(cls=cls.__name__): + cleared.clear() + o = cls() + o.a = Attr() + o.a.wr = weakref.ref(o) + del o + self.assertEqual(cleared, [True]) + def test_callback_attribute(self): x = Object(1) callback = lambda ref: None diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-10-19-48.gh-issue-161419.wc4DV0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-10-19-48.gh-issue-161419.wc4DV0.rst new file mode 100644 index 000000000000000..0d6605a9eeb5503 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-10-19-48.gh-issue-161419.wc4DV0.rst @@ -0,0 +1,3 @@ +Fix a crash in the :term:`free-threaded build` where a weak reference could +revive an object whose finalizer had already run during deallocation, causing +it to be deallocated twice. From 4b50f88ba146d71f03f917215d95a320708151e9 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Fri, 6 Nov 2026 08:55:26 +0100 Subject: [PATCH 002/229] gh-161387: Check the type of the decimal context `ContextVar` (#161399) Co-authored-by: sobolevn --- Lib/test/test_decimal.py | 21 +++++++++++++++++++ ...-11-05-10-05-00.gh-issue-161387.def456.rst | 2 ++ Modules/_decimal/_decimal.c | 7 +++++++ 3 files changed, 30 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-11-05-10-05-00.gh-issue-161387.def456.rst diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index c4b24617af7c433..ef7aa5358b5cc64 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -3780,6 +3780,27 @@ def test_crashing_signal_dict_flags(self): @requires_cdecimal class CContextAPItests(ContextAPItests, unittest.TestCase): decimal = C + + def test_current_context_var_wrong_type(self): + # gh-161387 + if not self.decimal.HAVE_CONTEXTVAR: + self.skipTest("_decimal built without context variables") + value = object() + context_vars = [var for var in contextvars.copy_context() + if var.name == "decimal_context"] + self.assertTrue(context_vars) + tokens = [var.set(value) for var in context_vars] + + def restore(): + for var, token in zip(context_vars, tokens): + var.reset(token) + + self.addCleanup(restore) + with self.assertRaisesRegex(TypeError, "got object"): + self.decimal.Decimal(1) + self.decimal.Decimal(2) + for var in context_vars: + self.assertIs(var.get(), value) + class PyContextAPItests(ContextAPItests, unittest.TestCase): decimal = P diff --git a/Misc/NEWS.d/next/Library/2026-11-05-10-05-00.gh-issue-161387.def456.rst b/Misc/NEWS.d/next/Library/2026-11-05-10-05-00.gh-issue-161387.def456.rst new file mode 100644 index 000000000000000..f445980563853bb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-05-10-05-00.gh-issue-161387.def456.rst @@ -0,0 +1,2 @@ +Fix a crash in :mod:`decimal` when the internal ``decimal_context`` +:class:`contextvars.ContextVar` holds a value that is not a context. diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 098522b130ec874..3a128a13222f9b5 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -2003,6 +2003,13 @@ current_context(decimal_state *state) } if (tl_context != NULL) { + if (!PyDecContext_Check(state, tl_context)) { + PyErr_Format(PyExc_TypeError, + "current decimal context must be a Context " + "instance, got %T", tl_context); + Py_DECREF(tl_context); + return NULL; + } return tl_context; } From 3feccd170e647b00141b61742a61ae3a0651d527 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 6 Nov 2026 10:42:50 +0100 Subject: [PATCH 003/229] gh-161242: Add a freelist for traceback objects (#161243) * Add a freelist for traceback objects A traceback object is created for every raised exception. Keep up to 8 of them on a per-thread freelist, and replace the argument check in tb_create_raw() with asserts since all callers validate their arguments. Co-Authored-By: Claude Fable 5.1 * Apply suggestion from @eendebakpt * Apply suggestion from @eendebakpt --------- Co-authored-by: Claude Fable 5.1 --- Include/internal/pycore_freelist_state.h | 2 ++ Objects/object.c | 1 + Python/traceback.c | 16 +++++++++------- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 46e2a82ea034560..266703a63b61f17 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -31,6 +31,7 @@ extern "C" { # define Py_pycfunctionobject_MAXFREELIST 16 # define Py_pycmethodobject_MAXFREELIST 16 # define Py_pymethodobjects_MAXFREELIST 20 +# define Py_tracebacks_MAXFREELIST 8 // A generic freelist of either PyObjects or other data structures. struct _Py_freelist { @@ -66,6 +67,7 @@ struct _Py_freelists { struct _Py_freelist pycfunctionobject; struct _Py_freelist pycmethodobject; struct _Py_freelist pymethodobjects; + struct _Py_freelist tracebacks; }; #ifdef __cplusplus diff --git a/Objects/object.c b/Objects/object.c index f8206e5d3eca329..757e54aa1d487d7 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -956,6 +956,7 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) clear_freelist(&freelists->pycfunctionobject, is_finalization, PyObject_GC_Del); clear_freelist(&freelists->pycmethodobject, is_finalization, PyObject_GC_Del); clear_freelist(&freelists->pymethodobjects, is_finalization, free_object); + clear_freelist(&freelists->tracebacks, is_finalization, PyObject_GC_Del); } /* diff --git a/Python/traceback.c b/Python/traceback.c index 7dc72d482745673..7e99dcb5a99249a 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH +#include "pycore_freelist.h" // _Py_FREELIST_POP() #include "pycore_frame.h" // PyFrameObject #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_interpframe.h" // _PyFrame_GetCode() @@ -79,15 +80,16 @@ tb_create_raw(PyTracebackObject *next, PyFrameObject *frame, int lasti, int lineno) { PyTracebackObject *tb; - if ((next != NULL && !PyTraceBack_Check(next)) || - frame == NULL || !PyFrame_Check(frame)) { - PyErr_BadInternalCall(); - return NULL; + assert(next == NULL || PyTraceBack_Check(next)); + assert(frame != NULL); + assert(PyFrame_Check(frame)); + tb = _Py_FREELIST_POP(PyTracebackObject, tracebacks); + if (tb == NULL) { + tb = PyObject_GC_New(PyTracebackObject, &PyTraceBack_Type); } - tb = PyObject_GC_New(PyTracebackObject, &PyTraceBack_Type); if (tb != NULL) { tb->tb_next = (PyTracebackObject*)Py_XNewRef(next); - tb->tb_frame = (PyFrameObject*)Py_XNewRef(frame); + tb->tb_frame = (PyFrameObject*)Py_NewRef(frame); tb->tb_lasti = lasti; tb->tb_lineno = lineno; PyObject_GC_Track(tb); @@ -237,7 +239,7 @@ tb_dealloc(PyObject *op) PyObject_GC_UnTrack(tb); Py_XDECREF(tb->tb_next); Py_XDECREF(tb->tb_frame); - PyObject_GC_Del(tb); + _Py_FREELIST_FREE(tracebacks, tb, PyObject_GC_Del); } static int From b76b6e7808268232a3ec15c1fe3a886ba3d7f1f8 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Fri, 6 Nov 2026 12:30:44 +0000 Subject: [PATCH 004/229] gh-157847: Clean up another portion of the `turtle` documentation (#160620) Co-authored-by: Maciej Olko --- Doc/library/turtle.rst | 289 +++++++++-------- Lib/turtle.py | 697 +++++++++++++++++++++-------------------- 2 files changed, 515 insertions(+), 471 deletions(-) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 39f8e5c6f226eea..7828d5e00cf7051 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -551,9 +551,9 @@ Move and draw .. function:: stamp() - Stamp a copy of the turtle shape onto the canvas at the current turtle - position. Return a stamp_id for that stamp, which can be used to delete - it by calling ``clearstamp(stamp_id)``. + Stamp a copy of the turtle shape onto the canvas and return its id. The + stamp is placed at the current turtle position. The returned id can be + used to delete the stamp with :func:`clearstamp`. .. doctest:: :skipif: _tkinter is None @@ -565,10 +565,10 @@ Move and draw .. function:: clearstamp(stampid) - :param stampid: an integer, must be return value of previous - :func:`stamp` call + :param stampid: an integer - Delete stamp with given *stampid*. + Delete the stamp with the given *stampid*. *stampid* must be the return + value of a previous :func:`stamp` call. .. doctest:: :skipif: _tkinter is None @@ -587,11 +587,11 @@ Move and draw .. function:: clearstamps(n=None) - :param n: an integer (or ``None``) + :param n: an integer or ``None`` - Delete all or first/last *n* of turtle's stamps. If *n* is ``None``, delete - all stamps, if *n* > 0 delete first *n* stamps, else if *n* < 0 delete - last *n* stamps. + Delete all or the first or last *n* of the turtle's stamps. If *n* is not + given, delete all stamps. If *n* is positive, delete the first *n* stamps. + If *n* is negative, delete the last *n* stamps, so -2 deletes the last two. .. doctest:: :skipif: _tkinter is None @@ -845,46 +845,49 @@ Drawing state .. function:: pen(pen=None, **pendict) - :param pen: a dictionary with some or all of the below listed keys - :param pendict: one or more keyword-arguments with the below listed keys as keywords + :param pen: a dictionary or ``None`` + :param pendict: keyword arguments with the keys below - Return or set the pen's attributes in a "pen-dictionary" with the following - key/value pairs: + Return or set the pen's attributes. The attributes are returned as a + dictionary with the following keys and values: - * "shown": True/False - * "pendown": True/False - * "pencolor": color-string or color-tuple - * "fillcolor": color-string or color-tuple - * "pensize": positive number - * "speed": number in range 0..10 - * "resizemode": "auto" or "user" or "noresize" - * "stretchfactor": (positive number, positive number) - * "outline": positive number - * "tilt": number + * "shown": ``True`` or ``False`` + * "pendown": ``True`` or ``False`` + * "pencolor": a color string or a color tuple + * "fillcolor": a color string or a color tuple + * "pensize": a positive number + * "speed": a number from 0 to 10 + * "resizemode": "auto", "user" or "noresize" + * "stretchfactor": a pair of positive numbers + * "shearfactor": a number + * "outline": a positive number + * "tilt": a number - This dictionary can be used as argument for a subsequent call to :func:`pen` - to restore the former pen-state. Moreover one or more of these attributes - can be provided as keyword-arguments. This can be used to set several pen - attributes in one statement. + Such a dictionary can be passed as *pen* to restore a former pen state. The + attributes can also be given as keyword arguments, which can be used to set + several of them in one statement. .. doctest:: :skipif: _tkinter is None :options: +NORMALIZE_WHITESPACE >>> turtle.pen(fillcolor="black", pencolor="red", pensize=10) - >>> sorted(turtle.pen().items()) - [('fillcolor', 'black'), ('outline', 1), ('pencolor', 'red'), - ('pendown', True), ('pensize', 10), ('resizemode', 'noresize'), - ('shearfactor', 0.0), ('shown', True), ('speed', 9), - ('stretchfactor', (1.0, 1.0)), ('tilt', 0.0)] - >>> penstate=turtle.pen() + >>> turtle.pen() + {'shown': True, 'pendown': True, 'pencolor': 'red', 'fillcolor': 'black', + 'pensize': 10, 'speed': 9, 'resizemode': 'noresize', + 'stretchfactor': (1.0, 1.0), 'shearfactor': 0.0, 'outline': 1, + 'tilt': 0.0} + >>> penstate = turtle.pen() >>> turtle.color("yellow", "") >>> turtle.penup() - >>> sorted(turtle.pen().items())[:3] - [('fillcolor', ''), ('outline', 1), ('pencolor', 'yellow')] + >>> turtle.pen()["pencolor"] + 'yellow' + >>> # restore the saved pen state, but with a green fill >>> turtle.pen(penstate, fillcolor="green") - >>> sorted(turtle.pen().items())[:3] - [('fillcolor', 'green'), ('outline', 1), ('pencolor', 'red')] + >>> turtle.pen()["pencolor"] + 'red' + >>> turtle.pen()["fillcolor"] + 'green' .. function:: isdown() @@ -1051,6 +1054,7 @@ Filling >>> turtle.home() + .. function:: filling() Return ``True`` if the turtle is filling, ``False`` otherwise. @@ -1064,9 +1068,10 @@ Filling ... else: ... turtle.pensize(3) + .. function:: fill() - Fill the shape drawn in the ``with turtle.fill():`` block. + Fill the shape drawn in the :keyword:`with` block. .. doctest:: :skipif: _tkinter is None @@ -1075,8 +1080,8 @@ Filling >>> with turtle.fill(): ... turtle.circle(80) - Using :func:`!fill` is equivalent to adding the :func:`begin_fill` before the - fill-block and :func:`end_fill` after the fill-block: + :func:`fill` is a :term:`context manager`, so using it is the same as + calling :func:`begin_fill` before the block and :func:`end_fill` after it: .. doctest:: :skipif: _tkinter is None @@ -1091,17 +1096,18 @@ Filling .. function:: begin_fill() - To be called just before drawing a shape to be filled. + Call just before drawing a shape to be filled. The shape is filled by the + next call to :func:`end_fill`. .. function:: end_fill() Fill the shape drawn after the last call to :func:`begin_fill`. - Whether or not overlap regions for self-intersecting polygons - or multiple shapes are filled depends on the operating system graphics, - type of overlap, and number of overlaps. For example, the Turtle star - above may be either all yellow or have some white regions. + If the shape crosses itself or several shapes overlap, whether the + overlapping parts are filled depends on the operating system, the type of + overlap and how many there are. For example, the turtle star above may be + either all yellow or have some white regions. .. doctest:: :skipif: _tkinter is None @@ -1144,15 +1150,16 @@ More drawing control .. function:: write(arg, move=False, align="left", font=("Arial", 8, "normal")) - :param arg: object to be written to the TurtleScreen - :param move: True/False - :param align: one of the strings "left", "center" or right" - :param font: a triple (fontname, fontsize, fonttype) + :param arg: an object + :param move: a boolean + :param align: a string + :param font: a tuple of a string, an integer and a string - Write text - the string representation of *arg* - at the current turtle - position according to *align* ("left", "center" or "right") and with the given - font. If *move* is true, the pen is moved to the bottom-right corner of the - text. By default, *move* is ``False``. + Write text at the current turtle position. The text is the string + representation of *arg*. *align* is one of the strings "left", "center" or + "right" and says how the text is placed relative to the turtle position. + *font* is a triple of font name, font size and font type. If *move* is + ``True``, the pen is moved to the bottom-right corner of the text. .. doctest:: :skipif: _tkinter is None @@ -1280,14 +1287,15 @@ Appearance .. function:: shearfactor(shear=None) - :param shear: number (optional) + :param shear: a number or ``None`` - Set or return the current shearfactor. Shear the turtleshape according to - the given shearfactor shear, which is the tangent of the shear angle. - Do *not* change the turtle's heading (direction of movement). - If shear is not given: return the current shearfactor, i. e. the - tangent of the shear angle, by which lines parallel to the - heading of the turtle are sheared. + Return or set the current shear factor. Shearing slants the turtle shape, + like the slant of italic letters. The shear factor says how much it is + slanted: 0 is not slanted at all, and the larger the number, the stronger + the slant. Negative numbers slant the other way. If *shear* is not given, + return the current shear factor. Otherwise slant the turtle shape + accordingly and set resizemode to "user". The turtle's heading does not + change. Precisely, the shear factor is the tangent of the shear angle. .. doctest:: :skipif: _tkinter is None @@ -1303,8 +1311,9 @@ Appearance :param angle: a number - Rotate the turtleshape by *angle* from its current tilt-angle, but do *not* - change the turtle's heading (direction of movement). + Turn the turtle shape by *angle* from its current tilt angle. The turtle's + heading does not change, so the turtle keeps moving in the same direction. + See :func:`tiltangle`. .. doctest:: :skipif: _tkinter is None @@ -1320,15 +1329,15 @@ Appearance .. function:: tiltangle(angle=None) - :param angle: a number (optional) + :param angle: a number or ``None`` - Set or return the current tilt-angle. If angle is given, rotate the - turtleshape to point in the direction specified by angle, - regardless of its current tilt-angle. Do *not* change the turtle's - heading (direction of movement). - If angle is not given: return the current tilt-angle, i. e. the angle - between the orientation of the turtleshape and the heading of the - turtle (its direction of movement). + Return or set the current tilt angle. The tilt angle is the angle between + the direction the turtle shape points in and the direction the turtle moves + in. Normally the shape points in the direction of movement, so the tilt + angle is 0. If *angle* is given, turn the turtle shape to that tilt angle, + no matter what it was before, and set resizemode to "user". The turtle's + heading does not change. If *angle* is not given, return the current tilt + angle. .. doctest:: :skipif: _tkinter is None @@ -1343,21 +1352,18 @@ Appearance .. function:: shapetransform(t11=None, t12=None, t21=None, t22=None) - :param t11: a number (optional) - :param t12: a number (optional) - :param t21: a number (optional) - :param t12: a number (optional) + :param t11: a number or ``None`` + :param t12: a number or ``None`` + :param t21: a number or ``None`` + :param t22: a number or ``None`` - Set or return the current transformation matrix of the turtle shape. - - If none of the matrix elements are given, return the transformation - matrix as a tuple of 4 elements. - Otherwise set the given elements and transform the turtleshape - according to the matrix consisting of first row t11, t12 and - second row t21, t22. The determinant t11 * t22 - t12 * t21 must not be - zero, otherwise an error is raised. - Modify stretchfactor, shearfactor and tiltangle according to the - given matrix. + Return or set the transformation matrix of the turtle shape. The matrix + combines the stretch factors, the shear factor and the tilt angle of the + turtle shape into four numbers: *t11* and *t12* form its first row, *t21* + and *t22* its second. If none of them are given, return the current matrix + as a tuple of four numbers. Otherwise set the given elements and update the + stretch factors, shear factor and tilt angle accordingly. + ``t11 * t22 - t12 * t21`` must not be zero, otherwise an error is raised. .. doctest:: :skipif: _tkinter is None @@ -1372,8 +1378,9 @@ Appearance .. function:: get_shapepoly() - Return the current shape polygon as tuple of coordinate pairs. This - can be used to define a new shape or components of a compound shape. + Return the points that form the outline of the current turtle shape + as a tuple of ``(x, y)`` coordinate pairs. These points can be used + to define a new shape or parts of a :ref:`compound shape ` .. doctest:: :skipif: _tkinter is None @@ -1464,8 +1471,8 @@ Special Turtle methods .. function:: poly() - Record the vertices of a polygon drawn in the ``with turtle.poly():`` block. - The first and last vertices will be connected. + Record the vertices of a polygon drawn in the :keyword:`with` block. The + first and last vertices will be connected. .. doctest:: :skipif: _tkinter is None @@ -1475,6 +1482,18 @@ Special Turtle methods ... turtle.right(60) ... turtle.forward(100) + :func:`poly` is a :term:`context manager`, so using it is the same as + calling :func:`begin_poly` before the block and :func:`end_poly` after it: + + .. doctest:: + :skipif: _tkinter is None + + >>> turtle.begin_poly() + >>> turtle.forward(100) + >>> turtle.right(60) + >>> turtle.forward(100) + >>> turtle.end_poly() + .. versionadded:: 3.14 @@ -1638,19 +1657,19 @@ Window control .. function:: bgpic(picname=None) - :param picname: a string, name of an image file (PNG, GIF, PGM, and PPM) - or ``"nopic"``, or ``None`` + :param picname: a string or ``None`` - Set background image or return name of current backgroundimage. If *picname* - is a filename, set the corresponding image as background. If *picname* is - ``"nopic"``, delete background image, if present. If *picname* is ``None``, - return the filename of the current backgroundimage. :: + Set the background image or return the name of the current one. If + *picname* is the name of an image file (PNG, GIF, PGM, and PPM), set that + image as the background. If *picname* is ``"nopic"``, delete the background + image, if present. If *picname* is ``None``, return the filename of the + current background image. :: >>> screen.bgpic() 'nopic' >>> screen.bgpic("landscape.gif") >>> screen.bgpic() - "landscape.gif" + 'landscape.gif' .. function:: clear() @@ -1685,15 +1704,19 @@ Window control .. function:: screensize(canvwidth=None, canvheight=None, bg=None) - :param canvwidth: positive integer, new width of canvas in pixels - :param canvheight: positive integer, new height of canvas in pixels - :param bg: colorstring or color-tuple, new background color + :param canvwidth: a positive integer or ``None`` + :param canvheight: a positive integer or ``None`` + :param bg: a colorstring or a color tuple or ``None`` - If no arguments are given, return current (canvaswidth, canvasheight). Else - resize the canvas the turtles are drawing on. Do not alter the drawing - window. To observe hidden parts of the canvas, use the scrollbars. With this - method, one can make visible those parts of a drawing which were outside the - canvas before. + Resize the canvas the turtles are drawing on. *canvwidth* and *canvheight* + are the new width and height of the canvas in pixels, and *bg* is the new + background color. If no arguments are given, return the current canvas + width and height. + + The drawing window is not altered. To see hidden parts of the canvas, use + the scrollbars. This can make visible those parts of a drawing which were + outside the canvas before, for example to find a turtle that has wandered + off the canvas. .. doctest:: :skipif: _tkinter is None @@ -1704,8 +1727,6 @@ Window control >>> screen.screensize() (2000, 1500) - e.g. to search for an erroneously escaped turtle ;-) - .. function:: setworldcoordinates(llx, lly, urx, ury) @@ -1815,19 +1836,24 @@ Using screen events .. function:: listen(xdummy=None, ydummy=None) - Set focus on TurtleScreen (in order to collect key-events). Dummy arguments - are provided in order to be able to pass :func:`listen` to the onclick method. + :param xdummy: ignored + :param ydummy: ignored + + Set focus on the :class:`TurtleScreen` so that it receives key events. The + two dummy arguments are only there so that :func:`listen` can be passed to + :func:`onclick`. .. function:: onkey(fun, key) onkeyrelease(fun, key) :param fun: a function with no arguments or ``None`` - :param key: a string: key (e.g. "a") or key-symbol (e.g. "space") + :param key: a string - Bind *fun* to key-release event of key. If *fun* is ``None``, event bindings - are removed. Remark: in order to be able to register key-events, TurtleScreen - must have the focus. (See method :func:`listen`.) + Bind *fun* to the key-release event of *key*. *key* is a key, such as + "a", or a key symbol, such as "space". If *fun* is ``None``, the event + binding is removed. To be able to receive key events, the + :class:`TurtleScreen` must have focus, see :func:`listen`. .. doctest:: :skipif: _tkinter is None @@ -1839,26 +1865,34 @@ Using screen events >>> screen.onkey(f, "Up") >>> screen.listen() + The turtle can then be moved by repeatedly pressing the up-arrow key, + drawing a hexagon. + .. function:: onkeypress(fun, key=None) :param fun: a function with no arguments or ``None`` - :param key: a string: key (e.g. "a") or key-symbol (e.g. "space") + :param key: a string or ``None`` - Bind *fun* to key-press event of key if key is given, - or to any key-press-event if no key is given. - Remark: in order to be able to register key-events, TurtleScreen - must have focus. (See method :func:`listen`.) + Bind *fun* to the key-press event of *key*. *key* is a key, such as "a", + or a key symbol, such as "space". If *key* is not given, *fun* is bound to + the key-press event of any key. If *fun* is ``None``, the event binding is + removed. To be able to receive key events, the :class:`TurtleScreen` must + have focus, see :func:`listen`. .. doctest:: :skipif: _tkinter is None >>> def f(): ... fd(50) + ... lt(60) ... - >>> screen.onkey(f, "Up") + >>> screen.onkeypress(f, "Up") >>> screen.listen() + The turtle can then be moved by pressing or holding down the up-arrow key, + drawing a hexagon. + .. function:: onclick(fun, btn=1, add=None) onscreenclick(fun, btn=1, add=None) @@ -1892,7 +1926,7 @@ Using screen events .. function:: ontimer(fun, t=0) :param fun: a function with no arguments - :param t: a number >= 0 + :param t: a nonnegative number Install a timer that calls *fun* after *t* milliseconds. @@ -1905,7 +1939,7 @@ Using screen events ... fd(50) ... lt(60) ... screen.ontimer(f, 250) - >>> f() ### makes the turtle march around + >>> f() # makes the turtle march around >>> running = False @@ -2120,15 +2154,16 @@ Screen-only methods client script. -.. function:: save(filename, overwrite=False) +.. function:: save(filename, *, overwrite=False) - Save the current turtle drawing (and turtles) as a PostScript file. + :param filename: a string + :param overwrite: a boolean - :param filename: the path of the saved PostScript file - :param overwrite: if ``False`` and there already exists a file with the given - filename, then the function will raise a - ``FileExistsError``. If it is ``True``, the file will be - overwritten. + Save the current drawing and the turtles as a PostScript file. *filename* + is the path of the file to create. It must end with ``".ps"`` or + ``".eps"``. If a file with that name already exists, a + :exc:`FileExistsError` is raised, unless *overwrite* is ``True``. + *overwrite* must be given by name, as in ``overwrite=True``. .. doctest:: :skipif: _tkinter is None diff --git a/Lib/turtle.py b/Lib/turtle.py index fbd1ff4870003de..3d808eef72cfa63 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -339,33 +339,27 @@ def onResize(self, event): self.adjustScrolls() def bbox(self, *args): - """ 'forward' method, which canvas itself has inherited... - """ + """Forward to the canvas method of the same name.""" return self._canvas.bbox(*args) def cget(self, *args, **kwargs): - """ 'forward' method, which canvas itself has inherited... - """ + """Forward to the canvas method of the same name.""" return self._canvas.cget(*args, **kwargs) def config(self, *args, **kwargs): - """ 'forward' method, which canvas itself has inherited... - """ + """Forward to the canvas method of the same name.""" self._canvas.config(*args, **kwargs) def bind(self, *args, **kwargs): - """ 'forward' method, which canvas itself has inherited... - """ + """Forward to the canvas method of the same name.""" self._canvas.bind(*args, **kwargs) def unbind(self, *args, **kwargs): - """ 'forward' method, which canvas itself has inherited... - """ + """Forward to the canvas method of the same name.""" self._canvas.unbind(*args, **kwargs) def focus_force(self): - """ 'forward' method, which canvas itself has inherited... - """ + """Forward to the canvas method of the same name.""" self._canvas.focus_force() __forwardmethods(ScrolledCanvas, TK.Canvas, '_canvas') @@ -384,7 +378,7 @@ def _getcanvas(self): return self._canvas def set_geometry(self, width, height, startx, starty): - self.geometry("%dx%d%+d%+d"%(width, height, startx, starty)) + self.geometry("%dx%d%+d%+d" % (width, height, startx, starty)) def ondestroy(self, destroy): self.wm_protocol("WM_DELETE_WINDOW", destroy) @@ -518,8 +512,7 @@ def _delay(self, delay): self.cv.after(delay) def _iscolorstring(self, color): - """Check if the string color is a legal Tkinter color string. - """ + """Check if the string color is a legal Tkinter color string.""" try: self.cv.winfo_rgb(color) ok = True @@ -576,13 +569,13 @@ def _onrelease(self, item, fun, num=1, add=None): then _onscreensclick-event. """ if fun is None: - self.cv.tag_unbind(item, "" % num) + self.cv.tag_unbind(item, f"") else: def eventfun(event): x, y = (self.cv.canvasx(event.x)/self.xscale, -self.cv.canvasy(event.y)/self.yscale) fun(x, y) - self.cv.tag_bind(item, "" % num, + self.cv.tag_bind(item, f"", eventfun, add) def _ondrag(self, item, fun, num=1, add=None): @@ -595,7 +588,7 @@ def _ondrag(self, item, fun, num=1, add=None): mouse-click event on that turtle. """ if fun is None: - self.cv.tag_unbind(item, "" % num) + self.cv.tag_unbind(item, f"") else: def eventfun(event): try: @@ -604,7 +597,7 @@ def eventfun(event): fun(x, y) except Exception: pass - self.cv.tag_bind(item, "" % num, eventfun, add) + self.cv.tag_bind(item, f"", eventfun, add) def _onscreenclick(self, fun, num=1, add=None): """Bind fun to mouse-click event on canvas. @@ -616,24 +609,25 @@ def _onscreenclick(self, fun, num=1, add=None): then _onscreensclick-event. """ if fun is None: - self.cv.unbind("" % num) + self.cv.unbind(f"") else: def eventfun(event): x, y = (self.cv.canvasx(event.x)/self.xscale, -self.cv.canvasy(event.y)/self.yscale) fun(x, y) - self.cv.bind("" % num, eventfun, add) + self.cv.bind(f"", eventfun, add) def _onkeyrelease(self, fun, key): """Bind fun to key-release event of key. - Canvas must have focus. See method listen + + Canvas must have focus. See the listen() method. """ if fun is None: - self.cv.unbind("" % key, None) + self.cv.unbind(f"", None) else: def eventfun(event): fun() - self.cv.bind("" % key, eventfun) + self.cv.bind(f"", eventfun) def _onkeypress(self, fun, key=None): """If key is given, bind fun to key-press event of key. @@ -644,37 +638,32 @@ def _onkeypress(self, fun, key=None): if key is None: self.cv.unbind("", None) else: - self.cv.unbind("" % key, None) + self.cv.unbind(f"", None) else: def eventfun(event): fun() if key is None: self.cv.bind("", eventfun) else: - self.cv.bind("" % key, eventfun) + self.cv.bind(f"", eventfun) def _listen(self): - """Set focus on canvas (in order to collect key-events) - """ + """Set focus on canvas (in order to collect key-events).""" self.cv.focus_force() def _ontimer(self, fun, t): - """Install a timer, which calls fun after t milliseconds. - """ + """Install a timer, which calls fun after t milliseconds.""" if t == 0: self.cv.after_idle(fun) else: self.cv.after(t, fun) def _createimage(self, image): - """Create and return image item on canvas. - """ + """Create and return image item on canvas.""" return self.cv.create_image(0, 0, image=image) def _drawimage(self, item, pos, image): - """Configure image item as to draw image object - at position (x,y) on canvas) - """ + """Configure the canvas item to show image at position pos.""" x, y = pos self.cv.coords(item, (x * self.xscale, -y * self.yscale)) self.cv.itemconfig(item, image=image) @@ -683,14 +672,12 @@ def _setbgpic(self, item, image): """Configure image item as to draw image object at center of canvas. Set item to the first item in the displaylist, so it will be drawn below - any other item .""" + any other item.""" self.cv.itemconfig(item, image=image) self.cv.tag_lower(item) def _type(self, item): - """Return 'line' or 'polygon' or 'image' depending on - type of item. - """ + """Return 'line' or 'polygon' or 'image' depending on type of item.""" return self.cv.type(item) def _pointlist(self, item): @@ -804,12 +791,11 @@ def numinput(self, title, prompt, default=None, minval=None, maxval=None): class Terminator(Exception): - """Will be raised in TurtleScreen.update, if _RUNNING becomes False. + """Raised by turtle commands after the turtle screen has been closed. - This stops execution of a turtle graphics script. - Main purpose: use in the Demo-Viewer turtle.Demo.py. + This stops the execution of a turtle graphics script. It is mainly + used by the turtledemo viewer. """ - pass class TurtleGraphicsError(Exception): @@ -979,9 +965,6 @@ def clear(self): As a function, this method is only available under the name clearscreen. The function clear is derived from the Turtle method clear. - - Example (for a TurtleScreen instance named screen): - >>> screen.clearscreen() """ self._delayvalue = _CFG["delay"] self._colormode = _CFG["colormode"] @@ -1401,27 +1384,28 @@ def onclick(self, fun, btn=1, add=None): self._onscreenclick(fun, btn, add) def onkey(self, fun, key): - """Bind fun to key-release event of key. + """Bind fun to the key-release event of key. - Arguments: - fun -- a function with no arguments - key -- a string: key (e.g. "a") or key-symbol (e.g. "space") + Aliases: onkey | onkeyrelease - In order to be able to register key-events, TurtleScreen - must have focus. (See method listen.) + key is a key, such as "a", or a key symbol, such as "space". If + fun is None, the event binding is removed. To be able to receive + key events, the TurtleScreen must have focus, see listen(). - Example (for a TurtleScreen instance named screen): - - >>> def f(): - ... fd(50) - ... lt(60) - ... - >>> screen.onkey(f, "Up") - >>> screen.listen() + Arguments: + fun -- a function with no arguments or None + key -- a string - Subsequently the turtle can be moved by repeatedly pressing - the up-arrow key, consequently drawing a hexagon + Example (for a TurtleScreen instance named screen): + >>> def f(): + ... fd(50) + ... lt(60) + ... + >>> screen.onkey(f, "Up") + >>> screen.listen() + The turtle can then be moved by repeatedly pressing the up-arrow + key, drawing a hexagon. """ if fun is None: if key in self._keys: @@ -1431,29 +1415,28 @@ def onkey(self, fun, key): self._onkeyrelease(fun, key) def onkeypress(self, fun, key=None): - """Bind fun to key-press event of key if key is given, - or to any key-press-event if no key is given. + """Bind fun to the key-press event of key. - Arguments: - fun -- a function with no arguments - key -- a string: key (e.g. "a") or key-symbol (e.g. "space") + key is a key, such as "a", or a key symbol, such as "space". If + key is not given, fun is bound to the key-press event of any key. + If fun is None, the event binding is removed. To be able to + receive key events, the TurtleScreen must have focus, see + listen(). - In order to be able to register key-events, TurtleScreen - must have focus. (See method listen.) - - Example (for a TurtleScreen instance named screen - and a Turtle instance named turtle): + Arguments: + fun -- a function with no arguments or None + key -- a string or None - >>> def f(): - ... fd(50) - ... lt(60) - ... - >>> screen.onkeypress(f, "Up") - >>> screen.listen() + Example (for a TurtleScreen instance named screen): + >>> def f(): + ... fd(50) + ... lt(60) + ... + >>> screen.onkeypress(f, "Up") + >>> screen.listen() - Subsequently the turtle can be moved by repeatedly pressing - the up-arrow key, or by keeping pressed the up-arrow key. - consequently drawing a hexagon. + The turtle can then be moved by pressing or holding down the + up-arrow key, drawing a hexagon. """ if fun is None: if key in self._keys: @@ -1463,54 +1446,54 @@ def onkeypress(self, fun, key=None): self._onkeypress(fun, key) def listen(self, xdummy=None, ydummy=None): - """Set focus on TurtleScreen (in order to collect key-events) + """Set focus on the TurtleScreen so that it receives key events. - No arguments. - Dummy arguments are provided in order - to be able to pass listen to the onclick method. + The two dummy arguments are only there so that listen() can be + passed to onclick(). - Example (for a TurtleScreen instance named screen): - >>> screen.listen() + Arguments: + xdummy -- ignored + ydummy -- ignored """ self._listen() def ontimer(self, fun, t=0): - """Install a timer, which calls fun after t milliseconds. + """Install a timer that calls fun after t milliseconds. Arguments: - fun -- a function with no arguments. - t -- a number >= 0 + fun -- a function with no arguments + t -- a nonnegative number Example (for a TurtleScreen instance named screen): - - >>> running = True - >>> def f(): - ... if running: - ... fd(50) - ... lt(60) - ... screen.ontimer(f, 250) - ... - >>> f() # makes the turtle marching around - >>> running = False + >>> running = True + >>> def f(): + ... if running: + ... fd(50) + ... lt(60) + ... screen.ontimer(f, 250) + ... + >>> f() # makes the turtle march around + >>> running = False """ self._ontimer(fun, t) def bgpic(self, picname=None): - """Set background image or return name of current backgroundimage. + """Set the background image or return the name of the current one. - Optional argument: - picname -- a string, name of an image file (PNG, GIF, PGM, and PPM) or "nopic". + If picname is the name of an image file (PNG, GIF, PGM, and PPM), + set that image as the background. If picname is "nopic", delete + the background image, if present. If picname is None, return the + filename of the current background image. - If picname is a filename, set the corresponding image as background. - If picname is "nopic", delete backgroundimage, if present. - If picname is None, return the filename of the current backgroundimage. + Argument: + picname -- a string or None Example (for a TurtleScreen instance named screen): - >>> screen.bgpic() - 'nopic' - >>> screen.bgpic("landscape.gif") - >>> screen.bgpic() - 'landscape.gif' + >>> screen.bgpic() + 'nopic' + >>> screen.bgpic("landscape.gif") + >>> screen.bgpic() + 'landscape.gif' """ if picname is None: return self._bgpicname @@ -1522,34 +1505,44 @@ def bgpic(self, picname=None): def screensize(self, canvwidth=None, canvheight=None, bg=None): """Resize the canvas the turtles are drawing on. - Optional arguments: - canvwidth -- positive integer, new width of canvas in pixels - canvheight -- positive integer, new height of canvas in pixels - bg -- colorstring or color-tuple, new backgroundcolor - If no arguments are given, return current (canvaswidth, canvasheight) + canvwidth and canvheight are the new width and height of the + canvas in pixels, and bg is the new background color. If no + arguments are given, return the current canvas width and height. - Do not alter the drawing window. To observe hidden parts of - the canvas use the scrollbars. (Can make visible those parts - of a drawing, which were outside the canvas before!) + The drawing window is not altered. To see hidden parts of the + canvas, use the scrollbars. This can make visible those parts of a + drawing which were outside the canvas before, for example to find + a turtle that has wandered off the canvas. - Example (for a Turtle instance named turtle): - >>> turtle.screensize(2000,1500) - >>> # e.g. to search for an erroneously escaped turtle ;-) + Arguments: + canvwidth -- a positive integer or None + canvheight -- a positive integer or None + bg -- a colorstring or a color tuple or None + + Example (for a TurtleScreen instance named screen): + >>> screen.screensize() + (400, 300) + >>> screen.screensize(2000,1500) + >>> screen.screensize() + (2000, 1500) """ return self._resize(canvwidth, canvheight, bg) def save(self, filename, *, overwrite=False): - """Save the drawing as a PostScript file + """Save the current drawing and the turtles as a PostScript file. - Arguments: - filename -- a string, the path of the created file. - Must end with '.ps' or '.eps'. + filename is the path of the file to create. It must end with + ".ps" or ".eps". If a file with that name already exists, a + FileExistsError is raised, unless overwrite is True. overwrite + must be given by name, as in overwrite=True. - Optional arguments: - overwrite -- boolean, if true, then existing files will be overwritten + Arguments: + filename -- a string + overwrite -- a boolean Example (for a TurtleScreen instance named screen): - >>> screen.save('my_drawing.eps') + >>> screen.save("my_drawing.ps") + >>> screen.save("my_drawing.ps", overwrite=True) """ filename = Path(filename) if not filename.parent.exists(): @@ -1571,6 +1564,8 @@ def save(self, filename, *, overwrite=False): postscript = self.cv.postscript() filename.write_text(postscript) + # Aliases for the commonly used commands: + onscreenclick = onclick resetscreen = reset clearscreen = clear @@ -1819,7 +1814,7 @@ def xcor(self): return self._position[0] def ycor(self): - """Return the turtle's y coordinate + """Return the turtle's y coordinate. Example (for a Turtle instance named turtle): >>> reset() @@ -1841,7 +1836,7 @@ def goto(self, x, y=None): The turtle's heading does not change. Arguments: - x -- a number or a pair/vector of numbers + x -- a number or a pair of numbers y -- a number or None Example (for a Turtle instance named turtle): @@ -1924,7 +1919,7 @@ def distance(self, x, y=None): as returned by pos(), or another turtle. Arguments: - x -- a number or a pair/vector of numbers or a turtle instance + x -- a number or a pair of numbers or a turtle instance y -- a number if x is a number, else None Example (for a Turtle instance named turtle): @@ -1957,7 +1952,7 @@ def towards(self, x, y=None): see mode(). Arguments: - x -- a number or a pair/vector of numbers or a turtle instance + x -- a number or a pair of numbers or a turtle instance y -- a number if x is a number, else None Example (for a Turtle instance named turtle): @@ -2168,7 +2163,7 @@ def pensize(self, width=None): Example (for a Turtle instance named turtle): >>> turtle.pensize() 1 - >>> turtle.pensize(10) # from here on lines of width 10 are drawn + >>> turtle.pensize(10) # from here on lines of width 10 are drawn """ if width is None: return self._pensize @@ -2424,49 +2419,47 @@ def isvisible(self): def pen(self, pen=None, **pendict): """Return or set the pen's attributes. + The attributes are returned as a dictionary with the following + keys and values: + + - "shown": True or False + - "pendown": True or False + - "pencolor": a color string or a color tuple + - "fillcolor": a color string or a color tuple + - "pensize": a positive number + - "speed": a number from 0 to 10 + - "resizemode": "auto", "user" or "noresize" + - "stretchfactor": a pair of positive numbers + - "shearfactor": a number + - "outline": a positive number + - "tilt": a number + + Such a dictionary can be passed as pen to restore a former pen + state. The attributes can also be given as keyword arguments, + which can be used to set several of them in one statement. + Arguments: - pen -- a dictionary with some or all of the below listed keys. - **pendict -- one or more keyword-arguments with the below - listed keys as keywords. - - Return or set the pen's attributes in a 'pen-dictionary' - with the following key/value pairs: - "shown" : True/False - "pendown" : True/False - "pencolor" : color-string or color-tuple - "fillcolor" : color-string or color-tuple - "pensize" : positive number - "speed" : number in range 0..10 - "resizemode" : "auto" or "user" or "noresize" - "stretchfactor": (positive number, positive number) - "shearfactor": number - "outline" : positive number - "tilt" : number - - This dictionary can be used as argument for a subsequent - pen()-call to restore the former pen-state. Moreover one - or more of these attributes can be provided as keyword-arguments. - This can be used to set several pen attributes in one statement. - - - Examples (for a Turtle instance named turtle): - >>> turtle.pen(fillcolor="black", pencolor="red", pensize=10) - >>> turtle.pen() - {'pensize': 10, 'shown': True, 'resizemode': 'auto', 'outline': 1, - 'pencolor': 'red', 'pendown': True, 'fillcolor': 'black', - 'stretchfactor': (1,1), 'speed': 3, 'shearfactor': 0.0} - >>> penstate=turtle.pen() - >>> turtle.color("yellow","") - >>> turtle.penup() - >>> turtle.pen() - {'pensize': 10, 'shown': True, 'resizemode': 'auto', 'outline': 1, - 'pencolor': 'yellow', 'pendown': False, 'fillcolor': '', - 'stretchfactor': (1,1), 'speed': 3, 'shearfactor': 0.0} - >>> p.pen(penstate, fillcolor="green") - >>> p.pen() - {'pensize': 10, 'shown': True, 'resizemode': 'auto', 'outline': 1, - 'pencolor': 'red', 'pendown': True, 'fillcolor': 'green', - 'stretchfactor': (1,1), 'speed': 3, 'shearfactor': 0.0} + pen -- a dictionary or None + pendict -- keyword arguments with the keys above + + Example (for a Turtle instance named turtle): + >>> turtle.pen(fillcolor="black", pencolor="red", pensize=10) + >>> turtle.pen() + {'shown': True, 'pendown': True, 'pencolor': 'red', + 'fillcolor': 'black', 'pensize': 10, 'speed': 3, + 'resizemode': 'noresize', 'stretchfactor': (1.0, 1.0), + 'shearfactor': 0.0, 'outline': 1, 'tilt': 0.0} + >>> penstate = turtle.pen() + >>> turtle.color("yellow", "") + >>> turtle.penup() + >>> turtle.pen()["pencolor"] + 'yellow' + >>> # restore the saved pen state, but with a green fill + >>> turtle.pen(penstate, fillcolor="green") + >>> turtle.pen()["pencolor"] + 'red' + >>> turtle.pen()["fillcolor"] + 'green' """ _pd = {"shown" : self._shown, "pendown" : self._drawing, @@ -2718,9 +2711,6 @@ def clear(self): The state and position of the turtle as well as the drawings of other turtles are not affected. - - Example (for a Turtle instance named turtle): - >>> turtle.clear() """ self._clear() self._update() @@ -2939,55 +2929,60 @@ def shapesize(self, stretch_wid=None, stretch_len=None, outline=None): stretchfactor=stretchfactor, outline=outline) def shearfactor(self, shear=None): - """Set or return the current shearfactor. - - Optional argument: shear -- number, tangent of the shear angle - - Shear the turtleshape according to the given shearfactor shear, - which is the tangent of the shear angle. DO NOT change the - turtle's heading (direction of movement). - If shear is not given: return the current shearfactor, i. e. the - tangent of the shear angle, by which lines parallel to the - heading of the turtle are sheared. - - Examples (for a Turtle instance named turtle): - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.shearfactor(0.5) - >>> turtle.shearfactor() - >>> 0.5 + """Return or set the current shear factor. + + Shearing slants the turtle shape, like the slant of italic + letters. The shear factor says how much it is slanted: 0 is not + slanted at all, and the larger the number, the stronger the + slant. Negative numbers slant the other way. If shear is not + given, return the current shear factor. Otherwise slant the + turtle shape accordingly and set resizemode to "user". The + turtle's heading does not change. Precisely, the shear factor is + the tangent of the shear angle. + + Argument: + shear -- a number or None + + Example (for a Turtle instance named turtle): + >>> turtle.shape("circle") + >>> turtle.shapesize(5,2) + >>> turtle.shearfactor(0.5) + >>> turtle.shearfactor() + 0.5 """ if shear is None: return self._shearfactor self.pen(resizemode="user", shearfactor=shear) def tiltangle(self, angle=None): - """Set or return the current tilt-angle. - - Optional argument: angle -- number - - Rotate the turtleshape to point in the direction specified by angle, - regardless of its current tilt-angle. DO NOT change the turtle's - heading (direction of movement). - If angle is not given: return the current tilt-angle, i. e. the angle - between the orientation of the turtleshape and the heading of the - turtle (its direction of movement). - - Examples (for a Turtle instance named turtle): - >>> turtle.shape("circle") - >>> turtle.shapesize(5, 2) - >>> turtle.tiltangle() - 0.0 - >>> turtle.tiltangle(45) - >>> turtle.tiltangle() - 45.0 - >>> turtle.stamp() - >>> turtle.fd(50) - >>> turtle.tiltangle(-45) - >>> turtle.tiltangle() - 315.0 - >>> turtle.stamp() - >>> turtle.fd(50) + """Return or set the current tilt angle. + + The tilt angle is the angle between the direction the turtle shape + points in and the direction the turtle moves in. Normally the + shape points in the direction of movement, so the tilt angle is + 0. If angle is given, turn the turtle shape to that tilt angle, no + matter what it was before, and set resizemode to "user". The + turtle's heading does not change. If angle is not given, return + the current tilt angle. + + Argument: + angle -- a number or None + + Example (for a Turtle instance named turtle): + >>> turtle.shape("circle") + >>> turtle.shapesize(5, 2) + >>> turtle.tiltangle() + 0.0 + >>> turtle.tiltangle(45) + >>> turtle.tiltangle() + 45.0 + >>> turtle.stamp() + >>> turtle.fd(50) + >>> turtle.tiltangle(-45) + >>> turtle.tiltangle() + 315.0 + >>> turtle.stamp() + >>> turtle.fd(50) """ if angle is None: tilt = -math.degrees(self._tilt) * self._angleOrient @@ -2998,43 +2993,47 @@ def tiltangle(self, angle=None): self.pen(resizemode="user", tilt=tilt) def tilt(self, angle): - """Rotate the turtleshape by angle. + """Turn the turtle shape by angle from its current tilt angle. + + The turtle's heading does not change, so the turtle keeps moving + in the same direction. See tiltangle(). Argument: - angle - a number - - Rotate the turtleshape by angle from its current tilt-angle, - but do NOT change the turtle's heading (direction of movement). - - Examples (for a Turtle instance named turtle): - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.tilt(30) - >>> turtle.fd(50) - >>> turtle.tilt(30) - >>> turtle.fd(50) + angle -- a number + + Example (for a Turtle instance named turtle): + >>> turtle.shape("circle") + >>> turtle.shapesize(5,2) + >>> turtle.tilt(30) + >>> turtle.fd(50) + >>> turtle.tilt(30) + >>> turtle.fd(50) """ self.tiltangle(angle + self.tiltangle()) def shapetransform(self, t11=None, t12=None, t21=None, t22=None): - """Set or return the current transformation matrix of the turtle shape. - - Optional arguments: t11, t12, t21, t22 -- numbers. - - If none of the matrix elements are given, return the transformation - matrix. - Otherwise set the given elements and transform the turtleshape - according to the matrix consisting of first row t11, t12 and - second row t21, 22. - Modify stretchfactor, shearfactor and tiltangle according to the - given matrix. - - Examples (for a Turtle instance named turtle): - >>> turtle.shape("square") - >>> turtle.shapesize(4,2) - >>> turtle.shearfactor(-0.5) - >>> turtle.shapetransform() - (4.0, -1.0, -0.0, 2.0) + """Return or set the transformation matrix of the turtle shape. + + The matrix combines the stretch factors, the shear factor and the + tilt angle of the turtle shape into four numbers: t11 and t12 form + its first row, t21 and t22 its second. If none of them are given, + return the current matrix as a tuple of four numbers. Otherwise + set the given elements and update stretch factors, shear factor and + tilt angle accordingly. t11 * t22 - t12 * t21 must not be zero, + otherwise an error is raised. + + Arguments: + t11 -- a number or None + t12 -- a number or None + t21 -- a number or None + t22 -- a number or None + + Example (for a Turtle instance named turtle): + >>> turtle.shape("square") + >>> turtle.shapesize(4,2) + >>> turtle.shearfactor(-0.5) + >>> turtle.shapetransform() + (4.0, -1.0, -0.0, 2.0) """ if t11 is t12 is t21 is t22 is None: return self._shapetrafo @@ -3069,21 +3068,21 @@ def _polytrafo(self, poly): for (x, y) in poly] def get_shapepoly(self): - """Return the current shape polygon as tuple of coordinate pairs. - - No argument. + """Return the points that form the outline of the current turtle shape + as a tuple of (x, y) coordinate pairs. - Examples (for a Turtle instance named turtle): - >>> turtle.shape("square") - >>> turtle.shapetransform(4, -1, 0, 2) - >>> turtle.get_shapepoly() - ((50, -20), (30, 20), (-50, 20), (-30, -20)) + This can be used to define a new shape or components of a compound + shape. + Example (for a Turtle instance named turtle): + >>> turtle.shape("square") + >>> turtle.shapetransform(4, -1, 0, 2) + >>> turtle.get_shapepoly() + ((50, -20), (30, 20), (-50, 20), (-30, -20)) """ shape = self.screen._shapes[self.turtle.shapeIndex] if shape._type == "polygon": return self._getshapepoly(shape._data, shape._type == "compound") - # else return None def _getshapepoly(self, polygon, compound=False): """Calculate transformed shape polygon according to resizemode @@ -3099,8 +3098,9 @@ def _getshapepoly(self, polygon, compound=False): return tuple((t11*x + t12*y, t21*x + t22*y) for (x, y) in polygon) def _drawturtle(self): - """Manages the correct rendering of the turtle with respect to - its shape, resizemode, stretch and tilt etc.""" + """Draw the turtle according to its shape, resizemode, stretch + factors and tilt, or hide it if it is not shown. + """ screen = self.screen shape = screen._shapes[self.turtle.shapeIndex] ttype = shape._type @@ -3139,19 +3139,15 @@ def _drawturtle(self): ############################## stamp stuff ############################### def stamp(self): - """Stamp a copy of the turtleshape onto the canvas and return its id. + """Stamp a copy of the turtle shape onto the canvas and return its id. - No argument. - - Stamp a copy of the turtle shape onto the canvas at the current - turtle position. Return a stamp_id for that stamp, which can be - used to delete it by calling clearstamp(stamp_id). + The stamp is placed at the current turtle position. The returned + id can be used to delete the stamp with clearstamp(). Example (for a Turtle instance named turtle): - >>> turtle.color("blue") - >>> turtle.stamp() - 13 - >>> turtle.fd(50) + >>> turtle.color("blue") + >>> stamp_id = turtle.stamp() + >>> turtle.fd(50) """ screen = self.screen shape = screen._shapes[self.turtle.shapeIndex] @@ -3159,9 +3155,12 @@ def stamp(self): tshape = shape._data if ttype == "polygon": stitem = screen._createpoly() - if self._resizemode == "noresize": w = 1 - elif self._resizemode == "auto": w = self._pensize - else: w =self._outlinewidth + if self._resizemode == "noresize": + w = 1 + elif self._resizemode == "auto": + w = self._pensize + else: + w =self._outlinewidth shape = self._polytrafo(self._getshapepoly(tshape)) fc, oc = self._fillcolor, self._pencolor screen._drawpoly(stitem, shape, fill=fc, outline=oc, @@ -3199,37 +3198,40 @@ def _clearstamp(self, stampid): self.undobuffer.remove(("stamp", stampid)) def clearstamp(self, stampid): - """Delete stamp with given stampid + """Delete the stamp with the given stampid. + + stampid must be the return value of a previous stamp() call. Argument: - stampid - an integer, must be return value of previous stamp() call. + stampid -- an integer Example (for a Turtle instance named turtle): - >>> turtle.color("blue") - >>> astamp = turtle.stamp() - >>> turtle.fd(50) - >>> turtle.clearstamp(astamp) + >>> turtle.color("blue") + >>> astamp = turtle.stamp() + >>> turtle.fd(50) + >>> turtle.clearstamp(astamp) """ self._clearstamp(stampid) self._update() def clearstamps(self, n=None): - """Delete all or first/last n of turtle's stamps. + """Delete all or the first or last n of the turtle's stamps. - Optional argument: - n -- an integer + If n is not given, delete all stamps. If n is positive, delete the + first n stamps. If n is negative, delete the last n stamps, so -2 + deletes the last two. - If n is None, delete all of pen's stamps, - else if n > 0 delete first n stamps - else if n < 0 delete last n stamps. + Argument: + n -- an integer or None Example (for a Turtle instance named turtle): - >>> for i in range(8): - ... turtle.stamp(); turtle.fd(30) - ... - >>> turtle.clearstamps(2) - >>> turtle.clearstamps(-2) - >>> turtle.clearstamps() + >>> for i in range(8): + ... unused_stamp_id = turtle.stamp() + ... turtle.fd(30) + ... + >>> turtle.clearstamps(2) + >>> turtle.clearstamps(-2) + >>> turtle.clearstamps() """ if n is None: toDelete = self.stampItems[:] @@ -3301,6 +3303,7 @@ def _undogoto(self, entry): cLI, cL, pl, items = coodata screen = self.screen if abs(self._position - new) > 0.5: + # XXX(Stan): Figure out what on earth this is. print ("undogoto: HALLO-DA-STIMMT-WAS-NICHT!") # restore former situation self.currentLineItem = cLI @@ -3406,15 +3409,19 @@ def filling(self): @contextmanager def fill(self): - """A context manager for filling a shape. - - Implicitly ensures the code block is wrapped with - begin_fill() and end_fill(). + """Fill the shape drawn in the with block. Example (for a Turtle instance named turtle): - >>> turtle.color("black", "red") - >>> with turtle.fill(): - ... turtle.circle(60) + >>> turtle.color("black", "red") + >>> with turtle.fill(): + ... turtle.circle(80) + + fill() is a context manager, so using it is the same as calling + begin_fill() before the block and end_fill() after it: + >>> turtle.color("black", "red") + >>> turtle.begin_fill() + >>> turtle.circle(80) + >>> turtle.end_fill() """ self.begin_fill() try: @@ -3423,15 +3430,15 @@ def fill(self): self.end_fill() def begin_fill(self): - """Called just before drawing a shape to be filled. + """Call just before drawing a shape to be filled. - No argument. + The shape is filled by the next call to end_fill(). Example (for a Turtle instance named turtle): - >>> turtle.color("black", "red") - >>> turtle.begin_fill() - >>> turtle.circle(60) - >>> turtle.end_fill() + >>> turtle.color("black", "red") + >>> turtle.begin_fill() + >>> turtle.circle(80) + >>> turtle.end_fill() """ if not self.filling(): self._fillitem = self.screen._createpoly() @@ -3443,15 +3450,17 @@ def begin_fill(self): self._update() def end_fill(self): - """Fill the shape drawn after the call begin_fill(). + """Fill the shape drawn after the last call to begin_fill(). - No argument. + If the shape crosses itself or several shapes overlap, whether the + overlapping parts are filled depends on the operating system, the + type of overlap and how many there are. Example (for a Turtle instance named turtle): - >>> turtle.color("black", "red") - >>> turtle.begin_fill() - >>> turtle.circle(60) - >>> turtle.end_fill() + >>> turtle.color("black", "red") + >>> turtle.begin_fill() + >>> turtle.circle(80) + >>> turtle.end_fill() """ if self.filling(): if len(self._fillpath) > 2: @@ -3503,8 +3512,7 @@ def dot(self, size=None, *color): self.pen(pen) def _write(self, txt, align, font): - """Performs the writing for write() - """ + """Performs the writing for write().""" item, end = self.screen._write(self._position, txt, align, font, self._pencolor) self._update() @@ -3516,21 +3524,21 @@ def _write(self, txt, align, font): def write(self, arg, move=False, align="left", font=("Arial", 8, "normal")): """Write text at the current turtle position. - Arguments: - arg -- info, which is to be written to the TurtleScreen - move (optional) -- True/False - align (optional) -- one of the strings "left", "center" or right" - font (optional) -- a triple (fontname, fontsize, fonttype) + The text is the string representation of arg. align is one of + the strings "left", "center" or "right" and says how the text is + placed relative to the turtle position. font is a triple of font + name, font size and font type. If move is True, the pen is moved + to the bottom-right corner of the text. - Write text - the string representation of arg - at the current - turtle position according to align ("left", "center" or right") - and with the given font. - If move is True, the pen is moved to the bottom-right corner - of the text. By default, move is False. + Arguments: + arg -- an object + move -- a boolean + align -- a string + font -- a tuple of a string, an integer and a string Example (for a Turtle instance named turtle): - >>> turtle.write('Home = ', True, align="center") - >>> turtle.write((0,0), True) + >>> turtle.write("Home = ", True, align="center") + >>> turtle.write((0,0), True) """ with self._undo_sequence(): end = self._write(str(arg), align.lower(), font) @@ -3540,18 +3548,23 @@ def write(self, arg, move=False, align="left", font=("Arial", 8, "normal")): @contextmanager def poly(self): - """A context manager for recording the vertices of a polygon. - - Implicitly ensures that the code block is wrapped with - begin_poly() and end_poly() - - Example (for a Turtle instance named turtle) where we create a - triangle as the polygon and move the turtle 100 steps forward: - >>> with turtle.poly(): - ... for side in range(3) - ... turtle.forward(50) - ... turtle.right(60) - >>> turtle.forward(100) + """Record the vertices of a polygon drawn in the with block. + + The first and last vertices will be connected. + + Example (for a Turtle instance named turtle): + >>> with turtle.poly(): + ... turtle.forward(100) + ... turtle.right(60) + ... turtle.forward(100) + + poly() is a context manager, so using it is the same as calling + begin_poly() before the block and end_poly() after it: + >>> turtle.begin_poly() + >>> turtle.forward(100) + >>> turtle.right(60) + >>> turtle.forward(100) + >>> turtle.end_poly() """ self.begin_poly() try: @@ -3890,10 +3903,6 @@ def exitonclick(self): This is a method of the Screen-class and not available for TurtleScreen instances. - - Example (for a Screen instance named screen): - >>> screen.exitonclick() - """ def exitGracefully(x, y): """Screen.bye() with two dummy-parameters""" From d138fd9c613ecd82e58653e9ab544fbed95ac239 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Fri, 6 Nov 2026 12:33:29 +0000 Subject: [PATCH 005/229] Fix miscellaneous issues in the 'Configure Python' docs (#158679) --- Doc/tools/extensions/pyspecific.py | 9 ++ Doc/using/configure.rst | 183 +++++++++++++---------------- Makefile.pre.in | 1 + 3 files changed, 94 insertions(+), 99 deletions(-) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 9b335f2976a2fa3..ac956d66efd423a 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -11,6 +11,7 @@ import re +from docutils import nodes from sphinx import addnodes from sphinx.domains.python import PyFunction, PyMethod @@ -78,6 +79,13 @@ def parse_pdb_command(env, sig, signode): return fullname +def parse_make_target(env, sig, signode): + """Transform a make target signature into RST nodes.""" + signode += addnodes.desc_name(sig, sig) + signode['ids'].append(nodes.make_id(sig)) + return sig + + def parse_monitoring_event(env, sig, signode): """Transform a monitoring event signature into RST nodes.""" signode += addnodes.desc_addname('sys.monitoring.events.', 'sys.monitoring.events.') @@ -113,6 +121,7 @@ def setup(app): app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature) app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command) app.add_object_type('monitoring-event', 'monitoring-event', '%s (monitoring event)', parse_monitoring_event) + app.add_object_type('make-target', 'make-target', '%s (make target)', parse_make_target) app.add_directive_to_domain('py', 'awaitablefunction', PyAwaitableFunction) app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.connect('env-check-consistency', patch_pairindextypes) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 602cf9fb6f082de..ea608394b8eb101 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -7,7 +7,7 @@ Configure Python .. _build-requirements: -Build Requirements +Build requirements ================== To build CPython, you will need: @@ -171,6 +171,8 @@ platform-specific modules like :mod:`winreg` are not listed here. SQLite 3.15.2 is now required for :mod:`sqlite3`. +.. _generated-files: + Generated files =============== @@ -181,9 +183,11 @@ files. Commands to regenerate all generated files:: make regen-stdlib-module-names make regen-limited-abi make regen-configure + make regen-sbom + make regen-unicodedata -The ``Makefile.pre.in`` file documents generated files, their inputs, and tools used -to regenerate them. Search for ``regen-*`` make targets. +The :source:`Makefile.pre.in` file documents generated files, their inputs, and +tools used to regenerate them. Search for ``regen-*`` make targets. configure script ---------------- @@ -214,7 +218,7 @@ and `pkg-config `_ 1.8.1. .. _configure-options: -Configure Options +Configure options ================= List all :file:`configure` script options using:: @@ -223,7 +227,7 @@ List all :file:`configure` script options using:: See also the :file:`Misc/SpecialBuilds.txt` in the Python source distribution. -General Options +General options --------------- .. option:: --enable-loadable-sqlite-extensions @@ -628,7 +632,7 @@ Options for third-party dependencies overriding ``pkg-config``. -WebAssembly Options +WebAssembly options ------------------- .. option:: --enable-wasm-dynamic-linking @@ -647,7 +651,7 @@ WebAssembly Options .. versionadded:: 3.11 -Install Options +Install options --------------- .. option:: --prefix=PREFIX @@ -934,7 +938,7 @@ also be used to improve performance. .. _debug-build: -Python Debug Build +Python debug build ------------------ A debug build is Python built with the :option:`--with-pydebug` configure @@ -1248,7 +1252,7 @@ Libraries options .. versionadded:: 3.10 -Security Options +Security options ---------------- .. option:: --with-hash-algorithm=[fnv|siphash13|siphash24] @@ -1326,7 +1330,7 @@ Security Options .. versionadded:: 3.14 -macOS Options +macOS options ------------- See :source:`Mac/README.rst`. @@ -1381,7 +1385,7 @@ See :source:`Mac/README.rst`. .. versionadded:: 3.13 -iOS Options +iOS options ----------- See :source:`Platforms/Apple/iOS/README.md`. @@ -1406,7 +1410,7 @@ rejects :option:`--enable-shared`. See iOS builds may be configured without a framework. -Cross Compiling Options +Cross compiling options ----------------------- Cross compiling, also known as cross building, can be used to build Python @@ -1456,7 +1460,7 @@ Cross compiling example:: --with-build-python=../x86_64/python -Python Build System +Python build system =================== Main files of the build system @@ -1480,107 +1484,88 @@ Main build steps Main Makefile targets --------------------- -make -^^^^ - -For the most part, when rebuilding after editing some code or -refreshing your checkout from upstream, all you need to do is execute -``make``, which (per Make's semantics) builds the default target, the -first one defined in the Makefile. By tradition (including in the -CPython project) this is usually the ``all`` target. The -``configure`` script expands an ``autoconf`` variable, -``@DEF_MAKE_ALL_RULE@`` to describe precisely which targets ``make -all`` will build. The three choices are: - -* ``profile-opt`` (configured with ``--enable-optimizations``) -* ``build_wasm`` (chosen if the host platform matches ``wasm32-wasi*`` or - ``wasm32-emscripten``) -* ``build_all`` (configured without explicitly using either of the others) - -Depending on the most recent source file changes, Make will rebuild -any targets (object files and executables) deemed out-of-date, -including running ``configure`` again if necessary. Source/target -dependencies are many and maintained manually however, so Make -sometimes doesn't have all the information necessary to correctly -detect all targets which need to be rebuilt. Depending on which -targets aren't rebuilt, you might experience a number of problems. If -you have build or test problems which you can't otherwise explain, -``make clean && make`` should work around most dependency problems, at -the expense of longer build times. - - -make platform -^^^^^^^^^^^^^ +.. make-target:: make -Build the ``python`` program, but don't build the standard library -extension modules. This generates a file named ``platform`` which -contains a single line describing the details of the build platform, -e.g., ``macosx-14.3-arm64-3.12`` or ``linux-x86_64-3.13``. + For the most part, when rebuilding after editing some code or + refreshing your checkout from upstream, all you need to do is execute + ``make``, which (per Make's semantics) builds the default target, the + first one defined in the Makefile. By tradition (including in the + CPython project) this is usually the ``all`` target. The + ``configure`` script expands an ``autoconf`` variable, + ``@DEF_MAKE_ALL_RULE@`` to describe precisely which targets ``make + all`` will build. The choices are: + * ``profile-opt`` (configured with ``--enable-optimizations``) + * ``bolt-opt`` (configured with ``--enable-bolt``) + * ``build_emscripten`` (chosen if the host platform matches + ``wasm32-emscripten``) + * ``build_wasm`` (chosen if the host platform matches ``wasm32-wasi*``) + * ``build_all`` (configured without explicitly using any of the others) -make profile-opt -^^^^^^^^^^^^^^^^ + Depending on the most recent source file changes, Make will rebuild + any targets (object files and executables) deemed out-of-date, + including running ``configure`` again if necessary. Source/target + dependencies are many and maintained manually however, so Make + sometimes doesn't have all the information necessary to correctly + detect all targets which need to be rebuilt. Depending on which + targets aren't rebuilt, you might experience a number of problems. If + you have build or test problems which you can't otherwise explain, + ``make clean && make`` should work around most dependency problems, at + the expense of longer build times. -Build Python using profile-guided optimization (PGO). You can use the -configure :option:`--enable-optimizations` option to make this the -default target of the ``make`` command (``make all`` or just -``make``). +.. make-target:: make platform + Build the ``python`` program, but don't build the standard library + extension modules. This generates a file named ``platform`` which + contains a single line describing the details of the build platform, + for example, ``macosx-14.3-arm64-3.12`` or ``linux-x86_64-3.13``. +.. make-target:: make profile-opt -make clean -^^^^^^^^^^ + Build Python using profile-guided optimization (PGO). You can use the + configure :option:`--enable-optimizations` option to make this the + default target of the ``make`` command (``make all`` or just + ``make``). -Remove built files. +.. make-target:: make clean + Remove built files. -make distclean -^^^^^^^^^^^^^^ +.. make-target:: make distclean -In addition to the work done by ``make clean``, remove files -created by the configure script. ``configure`` will have to be run -before building again. [#]_ + In addition to the work done by ``make clean``, remove files + created by the configure script. ``configure`` will have to be run + before building again. [#]_ +.. make-target:: make install -make install -^^^^^^^^^^^^ + Build the ``all`` target and install Python. -Build the ``all`` target and install Python. +.. make-target:: make test + Build the ``all`` target and run the Python test suite with the + ``--fast-ci`` option without GUI tests. Variables: -make test -^^^^^^^^^ + * ``TESTOPTS``: additional regrtest command-line options. + * ``TESTPYTHONOPTS``: additional Python command-line options. + * ``TESTTIMEOUT``: timeout in seconds (default: 10 minutes). -Build the ``all`` target and run the Python test suite with the -``--fast-ci`` option without GUI tests. Variables: +.. make-target:: make ci -* ``TESTOPTS``: additional regrtest command-line options. -* ``TESTPYTHONOPTS``: additional Python command-line options. -* ``TESTTIMEOUT``: timeout in seconds (default: 10 minutes). + This is similar to ``make test``, but uses the ``-ugui`` to also run GUI tests. + .. versionadded:: 3.14 -make ci -^^^^^^^ - -This is similar to ``make test``, but uses the ``-ugui`` to also run GUI tests. - -.. versionadded:: 3.14 - - -make buildbottest -^^^^^^^^^^^^^^^^^ - -This is similar to ``make test``, but uses the ``--slow-ci`` -option and default timeout of 20 minutes, instead of ``--fast-ci`` option. +.. make-target:: make buildbottest + This is similar to ``make test``, but uses the ``--slow-ci`` + option and default timeout of 20 minutes, instead of ``--fast-ci`` option. -make regen-all -^^^^^^^^^^^^^^ +.. make-target:: make regen-all -Regenerate (almost) all generated files. These include (but are not -limited to) bytecode cases, and parser generator file. -``make regen-stdlib-module-names`` and ``autoconf`` must be run -separately for the remaining `generated files <#generated-files>`_. + Regenerate (almost) all generated files. These include (but are not + limited to) bytecode cases, and parser generator files. See + :ref:`generated-files` for the targets that must be run separately. C extensions @@ -1600,23 +1585,23 @@ Built-in modules have no ``__file__`` attribute: File "", line 1, in AttributeError: module 'sys' has no attribute '__file__' -Other C extensions are built as dynamic libraries, like the ``_asyncio`` module. +Other C extensions are built as dynamic libraries, like the ``_zoneinfo`` module. They are built with the ``Py_BUILD_CORE_MODULE`` macro defined. -Example on Linux x86-64: +For example on Linux x86-64: .. code-block:: pycon - >>> import _asyncio - >>> _asyncio - - >>> _asyncio.__file__ - '/usr/lib64/python3.9/lib-dynload/_asyncio.cpython-39-x86_64-linux-gnu.so' + >>> import _zoneinfo + >>> _zoneinfo + + >>> _zoneinfo.__file__ + '/.../lib.linux-x86_64-3.16/_zoneinfo.cpython-316d-x86_64-linux-gnu.so' :file:`Modules/Setup` is used to generate Makefile targets to build C extensions. At the beginning of the files, C extensions are built as built-in modules. Extensions defined after the ``*shared*`` marker are built as dynamic libraries. -The :c:macro:`!PyAPI_FUNC()`, :c:macro:`!PyAPI_DATA()` and +The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_DATA()` and :c:macro:`PyMODINIT_FUNC` macros of :file:`Include/exports.h` are defined differently depending if the ``Py_BUILD_CORE_MODULE`` macro is defined: diff --git a/Makefile.pre.in b/Makefile.pre.in index e74deee353488fe..f1b5eb3bba34fd1 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1915,6 +1915,7 @@ regen-unicodedata: # Regenerate all generated files # "clinic" is regenerated implicitly via "regen-global-objects". +# Keep the list of targets to run manually in sync with Doc/using/configure.rst. .PHONY: regen-all regen-all: regen-cases regen-slots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ From 3c92a06e0d08f482f1e368a41670404ef5d247da Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 6 Nov 2026 17:22:52 +0300 Subject: [PATCH 006/229] gh-161490: Fix leak in `test_any_str_deprecated` in `test_typing` (#161491) --- Lib/test/test_typing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b1775ba7d81e521..3879c49c714dd9b 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -11300,6 +11300,7 @@ def test_all_exported_names(self): def test_any_str_deprecated(self): # gh-105578 fresh = import_helper.import_fresh_module('typing') + self.addCleanup(atexit.unregister, fresh._clear_caches) with self.assertWarnsRegex(DeprecationWarning, r'typing\.AnyStr'): fresh.AnyStr From dd1d4f0c4965ea9643d0e5e32bf7c0cea4e7b0c1 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Fri, 6 Nov 2026 16:34:22 +0000 Subject: [PATCH 007/229] gh-131725: Generate GNU hash table in `msgfmt.py` (#131727) Co-authored-by: Serhiy Storchaka --- Lib/test/test_tools/msgfmt_data/fuzzy.mo | Bin 28 -> 40 bytes Lib/test/test_tools/msgfmt_data/general.mo | Bin 671 -> 723 bytes Lib/test/test_tools/test_msgfmt.py | 27 ++++++- ...03-25-18-00-00.gh-issue-131725.qwfh321.rst | 2 + Tools/i18n/msgfmt.py | 70 ++++++++++++++++-- 5 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-03-25-18-00-00.gh-issue-131725.qwfh321.rst diff --git a/Lib/test/test_tools/msgfmt_data/fuzzy.mo b/Lib/test/test_tools/msgfmt_data/fuzzy.mo index 4b144831cf5f75c62f6e9446b91ce164bb3c8dee..273edbbbd7cbd799fe1bf1497157bacf5f3a271e 100644 GIT binary patch delta 25 Ucmb2qm>?p;%)r1P1H{Mx02ti?8~^|S delta 13 OcmdOpnIOW!00jUFr2z2& diff --git a/Lib/test/test_tools/msgfmt_data/general.mo b/Lib/test/test_tools/msgfmt_data/general.mo index ee905cbb3ec58db04c2f2fca5f67e15ce7a852c9..3107f6711bdd10bbdde590bddaf308b78a58914c 100644 GIT binary patch delta 214 zcmbQwdYN^CNIfqD1H&3127w1a%mc&^ftVAB-vKc{5dQ#Tc_8LsWMEJR(i%XT9Z35C zX=WfD0_8^l>90V30+8kd(qT*x^HPDd2#{X{q(SDiL+RyA4BlV^jspci2Azh|=YcfP g46qoGV&DK`kb^*$0;Rx!1xT_&*=%5XwU}Rum2GR;Z z8pKxt(qDnJ4v^*p(n?Ga^9+Hs2#{|Fq(SBcL+N5B263> 24 + hval ^= g + return hval & 0xFFFFFFFF + + +def _next_prime(start): + def is_prime(num): + divn = 3 + sq = divn * divn + while sq < num and num % divn != 0: + divn += 1 + sq += 4 * divn + divn += 1 + + return num % divn != 0 + + candidate = start | 1 + while not is_prime(candidate): + candidate += 2 + return candidate + if __name__ == '__main__': main() From 32f1ea3f3af377963bab9e7c1195ce1e3db436b6 Mon Sep 17 00:00:00 2001 From: Aleksandr Burlakov Date: Fri, 6 Nov 2026 19:41:15 +0300 Subject: [PATCH 008/229] gh-161492: Fix assert in `test_str.test_compare` (#161493) --- Lib/test/test_str.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index bef0e62fb42bccc..b4b4572256c8f23 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -2605,7 +2605,7 @@ def test_compare(self): self.assertFalse(copy1 != copy2) self.assertTrue(copy1 <= copy2) - self.assertTrue(copy2 >= copy2) + self.assertTrue(copy1 >= copy2) self.assertTrue(ascii < ascii2) self.assertTrue(ascii < latin) From 17a64027984f1aa417e451f6d56afa88d070906d Mon Sep 17 00:00:00 2001 From: Saugat Pandey <39925335+iamsaugatpandey@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:41:33 -0500 Subject: [PATCH 009/229] gh-158500: Reject trailing characters in `datetime.date.fromisoformat()` (#160737) Co-authored-by: Stan Ulbrych --- Lib/_pydatetime.py | 10 +++++++++- Lib/test/datetimetester.py | 9 +++++++++ .../2026-10-26-21-37-42.gh-issue-158500.zxyuq5.rst | 4 ++++ Modules/_datetimemodule.c | 7 +++++++ 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-26-21-37-42.gh-issue-158500.zxyuq5.rst diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index 5bbb6d433ee2ad6..5a8fec43e668a53 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -396,6 +396,10 @@ def _parse_isoformat_date(dtstr): pos += has_sep dayno = _read_isoformat_component(dtstr[pos:pos + 1], 1) + pos += 1 + + if pos != len(dtstr): + raise ValueError("Invalid isoformat string") return list(_isoweek_to_gregorian(year, weekno, dayno)) else: @@ -406,6 +410,10 @@ def _parse_isoformat_date(dtstr): pos += has_sep day = _read_isoformat_component(dtstr[pos:pos + 2], 2) + pos += 2 + + if pos != len(dtstr): + raise ValueError("Invalid isoformat string") return [year, month, day] @@ -1931,7 +1939,7 @@ def fromisoformat(cls, date_string): raise ValueError( f'Invalid isoformat string: {date_string!r}') from None - if tstr: + if separator_location < len(date_string): try: (time_components, became_next_day, diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 92607e381db20c7..7d242f663824b39 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2243,6 +2243,10 @@ def test_fromisoformat_fails(self): '2020-W 5', # space in the week number '2020061', # 7 chars: day slice reads a 1-character tail '2020-W2', # 1-digit week number + # gh-158500: trailing characters must not be ignored + '2020010112', # YYYYMMDD followed by two characters + '2020W011xx', # YYYYWwwD followed by two characters + '20200101\u00e9', # 9 characters, but 10 bytes in UTF-8 '٢025-03-09', # Unicode characters '2009\ud80002\ud80028', # Separators are surrogate codepoints ] @@ -3984,6 +3988,11 @@ def test_fromisoformat_fails_datetime(self): '2009-04-19T12:30:45.-05:00', # Empty fraction before offset '2009-04-19T12:30:45.Z', # Empty fraction before Z '2009-04-19T12:30:45,+05:00', # Empty fraction (comma) before offset + # gh-158500: a separator must be followed by a time + '2009-04-19T', # Separator without a time + '20090419T', # Separator without a time + '2009-W16-7T', # Separator without a time + '2009W167T', # Separator without a time ] for bad_str in bad_strs: diff --git a/Misc/NEWS.d/next/Library/2026-10-26-21-37-42.gh-issue-158500.zxyuq5.rst b/Misc/NEWS.d/next/Library/2026-10-26-21-37-42.gh-issue-158500.zxyuq5.rst new file mode 100644 index 000000000000000..98588df2f8ce83a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-26-21-37-42.gh-issue-158500.zxyuq5.rst @@ -0,0 +1,4 @@ +:meth:`datetime.date.fromisoformat` now rejects strings with trailing +characters after a basic-format date, such as ``'2020010112'``. The pure-Python +:meth:`datetime.datetime.fromisoformat` now rejects a date followed by a bare +separator, such as ``'2020-01-01T'``, matching the C implementation. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 959abcf5e421cf5..380d8b435e73d6c 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -977,6 +977,10 @@ parse_isoformat_date(const char *dtstr, const size_t len, int *year, int *month, iso_day = 1; } + if ((size_t)(p - dtstr) != len) { + return -1; + } + int rv = iso_to_ymd(*year, iso_week, iso_day, year, month, day); if (rv) { return -3 + rv; @@ -997,6 +1001,9 @@ parse_isoformat_date(const char *dtstr, const size_t len, int *year, int *month, if (p == NULL) { return -1; } + if ((size_t)(p - dtstr) != len) { + return -1; + } return 0; } From 0e885d9b53038cc2513baed0d179d6246ba6e2c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Sat, 7 Nov 2026 00:53:27 +0800 Subject: [PATCH 010/229] gh-142782: Compare `zoneinfo` strong cache keys by string contents (#161463) Co-authored-by: Stan Ulbrych --- Lib/test/test_zoneinfo/test_zoneinfo.py | 71 +++++++++++++++++++ ...-11-06-00-00-00.gh-issue-142782.qU7nE4.rst | 3 + Modules/_zoneinfo.c | 8 +-- 3 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-06-00-00-00.gh-issue-142782.qU7nE4.rst diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index d4b3cd36e6d3293..6656d7c1dc2cd67 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -1723,6 +1723,13 @@ def __eq__(self, other): except CustomError: pass + def test_clear_cache_non_string_keys(self): + la0 = self.klass("America/Los_Angeles") + self.klass.clear_cache(only_keys=[None, 1]) + la1 = self.klass("America/Los_Angeles") + + self.assertIs(la0, la1) + def test_weak_cache_descriptor_use_after_free(self): class BombDescriptor: def __get__(self, obj, owner): @@ -1747,6 +1754,70 @@ class EvilZoneInfo(self.klass): class CZoneInfoCacheTest(ZoneInfoCacheTest): module = c_zoneinfo + def test_strong_cache_path_like_key(self): + # A non-str key is matched by identity, and comparing it against a + # str key must not raise TypeError. + class Key: + def __fspath__(self): + return "UTC" + + def __len__(self): + return 3 + + def split(self, sep): + return ["UTC"] + + key = Key() + original = self.klass(key) + self.assertIs(self.klass(key), original) + self.assertIsNot(self.klass("UTC"), original) + self.klass.clear_cache(only_keys=(key,)) + self.assertIsNot(self.klass(key), original) + + def test_strong_cache_str_subclass(self): + class Stringy(str): + def __eq__(self, other): + raise CustomError + + __hash__ = str.__hash__ + + for cached_type, lookup_type in ((str, Stringy), (Stringy, str), + (Stringy, Stringy)): + with self.subTest(cached_type=cached_type, lookup_type=lookup_type): + self.klass.clear_cache() + original = self.klass(cached_type("America/Los_Angeles")) + self.assertIs( + self.klass(lookup_type("America/Los_Angeles")), original + ) + + def test_strong_cache_reentrant_clear(self): + self._test_strong_cache_reentrant_clear("lookup") + + def test_strong_cache_reentrant_clear_only_keys(self): + self._test_strong_cache_reentrant_clear("clear") + + def _test_strong_cache_reentrant_clear(self, operation): + klass = self.klass + + class Stringy(str): + cleared = False + + def __eq__(self, other): + if not self.cleared: + self.cleared = True + klass.clear_cache(only_keys=(self,)) + return super().__eq__(other) + + __hash__ = str.__hash__ + + original = klass(Stringy("America/Los_Angeles")) + key = Stringy("America/Los_Angeles") + if operation == "lookup": + self.assertIs(klass(key), original) + else: + klass.clear_cache(only_keys=(key,)) + self.assertIsNot(klass("America/Los_Angeles"), original) + def test_inconsistent_weak_cache_get(self): class Cache: def get(self, key, default=None): diff --git a/Misc/NEWS.d/next/Library/2026-11-06-00-00-00.gh-issue-142782.qU7nE4.rst b/Misc/NEWS.d/next/Library/2026-11-06-00-00-00.gh-issue-142782.qU7nE4.rst new file mode 100644 index 000000000000000..4bccc120ab8d33f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-06-00-00-00.gh-issue-142782.qU7nE4.rst @@ -0,0 +1,3 @@ +Fix a use-after-free in :class:`zoneinfo.ZoneInfo` when a key's ``__eq__`` +clears the cache during a lookup. Strong cache keys are now compared by +string contents. diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 45f65b9d30b6374..3a87440c63e8d24 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -2438,11 +2438,9 @@ find_in_strong_cache(const StrongCacheNode *const root, PyObject *const key) { const StrongCacheNode *node = root; while (node != NULL) { - int rv = PyObject_RichCompareBool(key, node->key, Py_EQ); - if (rv < 0) { - return NULL; - } - if (rv) { + if (key == node->key || + (PyUnicode_Check(key) && PyUnicode_Check(node->key) && + PyUnicode_Equal(key, node->key))) { return (StrongCacheNode *)node; } From 2781ffe1f29b0ecf64b011c927ad71df2321b6af Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Nov 2026 18:19:36 +0100 Subject: [PATCH 011/229] gh-161353: Fix handle leak in test_tcl (#161368) Fix handle leaks in test_tcl when the test is run with -R 3:3. Call explicitly Tcl_FinalizeThread() in test_returning_deeply_nested() to clean up explicitly per-thread state. The test is run in a thread because of the @run_with_limited_c_stack() decorator. On Windows, Tcl_CreateInterp() creates an event in the Tcl notifier implementation. The event is stored in per-thread state which is only cleaned up at exit. Add the private _tkinter._finalize_thread() function. --- Lib/test/test_tcl.py | 5 +++++ Modules/_tkinter.c | 15 +++++++++++++++ Modules/clinic/_tkinter.c.h | 20 +++++++++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 1d90ac10ad77ade..31ef1457ef3944e 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -595,6 +595,11 @@ def test_returning_deeply_nested(self): interp.eval('set x a; for {set i 0} {$i < %d} {incr i} {set x [list $x]}' % support.exceeds_recursion_limit()) self.assertRaises(RecursionError, interp.getvar, 'x') + interp = None + + # gh-161353: Clean up explicitly per-thread state to avoid leaking an + # event handle on Windows when test_tcl is run with -R 3:3 option. + _tkinter._finalize_thread() def test_user_command(self): result = None diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 9a09687e55b9817..9540839b9682bca 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -3584,6 +3584,20 @@ _tkinter_getbusywaitinterval_impl(PyObject *module) return Tkinter_busywaitinterval; } +/*[clinic input] +_tkinter._finalize_thread + +Clean up per-thread state. +[clinic start generated code]*/ + +static PyObject * +_tkinter__finalize_thread_impl(PyObject *module) +/*[clinic end generated code: output=4ed0d3be4d1f8de8 input=ea68096ee9b83d23]*/ +{ + Tcl_FinalizeThread(); + Py_RETURN_NONE; +} + #include "clinic/_tkinter.c.h" static PyMethodDef Tktt_methods[] = @@ -3681,6 +3695,7 @@ static PyMethodDef moduleMethods[] = _TKINTER_CREATE_METHODDEF _TKINTER_SETBUSYWAITINTERVAL_METHODDEF _TKINTER_GETBUSYWAITINTERVAL_METHODDEF + _TKINTER__FINALIZE_THREAD_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 1e197167409a033..22529aaf7bee449 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -956,6 +956,24 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(_tkinter__finalize_thread__doc__, +"_finalize_thread($module, /)\n" +"--\n" +"\n" +"Clean up per-thread state."); + +#define _TKINTER__FINALIZE_THREAD_METHODDEF \ + {"_finalize_thread", (PyCFunction)_tkinter__finalize_thread, METH_NOARGS, _tkinter__finalize_thread__doc__}, + +static PyObject * +_tkinter__finalize_thread_impl(PyObject *module); + +static PyObject * +_tkinter__finalize_thread(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _tkinter__finalize_thread_impl(module); +} + #ifndef _TKINTER_TKAPP_CREATEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_CREATEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_CREATEFILEHANDLER_METHODDEF) */ @@ -963,4 +981,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=229ae8ed00ee4e40 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c94e363a6f9de67d input=a9049054013a1b77]*/ From d900c8f9aa7e71cca78536845340b19f9706e871 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Fri, 6 Nov 2026 11:05:52 -0800 Subject: [PATCH 012/229] gh-71199: Name posixmodule types `os.` on every platform (#161461) Pickles name posixmodule types and functions either through os or through the platform module (posix or nt). Both spellings must keep loading. times_result, uname_result, waitid_result, sched_param, DirEntry and ScandirIterator were named after the platform module (posix or nt), so pickles referenced posix.times_result and could not be loaded on Windows. Name them os. like stat_result (gh-63408). Older pickles keep loading through the platform module. Co-authored-by: Claude Opus 5.5 Co-authored-by: Stan Ulbrych --- Lib/test/test_os/test_os.py | 21 +++++++++++++++---- ...6-11-06-05-42-11.gh-issue-71199.qJN4cV.rst | 3 +++ Modules/posixmodule.c | 12 +++++------ 3 files changed, 26 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-06-05-42-11.gh-issue-71199.qJN4cV.rst diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 662b57702fe26cd..6e7201d0b056c21 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -178,15 +178,28 @@ def test_type_fqdn(self): def fqdn(obj): return (obj.__module__, obj.__qualname__) - native = os.name self.assertEqual(fqdn(os.stat_result), ("os", "stat_result")) - self.assertEqual(fqdn(os.times_result), (native, "times_result")) + self.assertEqual(fqdn(os.times_result), ("os", "times_result")) + self.assertEqual(fqdn(os.DirEntry), ("os", "DirEntry")) if hasattr(os, "statvfs_result"): self.assertEqual(fqdn(os.statvfs_result), ("os", "statvfs_result")) + self.assertEqual(fqdn(os.uname_result), ("os", "uname_result")) if hasattr(os, "sched_param"): - self.assertEqual(fqdn(os.sched_param), (native, "sched_param")) + self.assertEqual(fqdn(os.sched_param), ("os", "sched_param")) if hasattr(os, "waitid_result"): - self.assertEqual(fqdn(os.waitid_result), (native, "waitid_result")) + self.assertEqual(fqdn(os.waitid_result), ("os", "waitid_result")) + + def test_pickle_global_names(self): + # Existing pickles name these via os or the platform module (posix + # or nt); both must keep loading, as must new pickles. + for name in ('stat', 'stat_result', 'statvfs_result', 'times_result', + 'uname_result'): + with self.subTest(name): + obj = getattr(os, name) + for module in ('os', os.name): + p = f'c{module}\n{name}\n.'.encode() + self.assertIs(pickle.loads(p), obj) + self.assertIs(pickle.loads(pickle.dumps(obj)), obj) # Tests creating TESTFN diff --git a/Misc/NEWS.d/next/Library/2026-11-06-05-42-11.gh-issue-71199.qJN4cV.rst b/Misc/NEWS.d/next/Library/2026-11-06-05-42-11.gh-issue-71199.qJN4cV.rst new file mode 100644 index 000000000000000..fde88d5facc9511 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-06-05-42-11.gh-issue-71199.qJN4cV.rst @@ -0,0 +1,3 @@ +Change :mod:`os` module types such as :class:`~os.DirEntry` to report ``os`` +as their module on every platform. :ref:`Pickled ` objects +using the platform-specific qualified name, such as ``nt.times_result``, still load. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9709c4e1c9454b2..67461fdb3730860 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2561,7 +2561,7 @@ static PyStructSequence_Field waitid_result_fields[] = { }; static PyStructSequence_Desc waitid_result_desc = { - MODNAME ".waitid_result", /* name */ + "os.waitid_result", /* name */ waitid_result__doc__, /* doc */ waitid_result_fields, 5 @@ -6768,7 +6768,7 @@ or via the attributes sysname, nodename, release, version, and machine.\n\ See os.uname for more information."); static PyStructSequence_Desc uname_result_desc = { - MODNAME ".uname_result", /* name */ + "os.uname_result", /* name */ uname_result__doc__, /* doc */ uname_result_fields, 5 @@ -8818,7 +8818,7 @@ static PyStructSequence_Field sched_param_fields[] = { }; static PyStructSequence_Desc sched_param_desc = { - MODNAME ".sched_param", /* name */ + "os.sched_param", /* name */ os_sched_param__doc__, /* doc */ sched_param_fields, 1 @@ -11376,7 +11376,7 @@ and elapsed.\n\ See os.times for more information."); static PyStructSequence_Desc times_result_desc = { - MODNAME ".times_result", /* name */ + "os.times_result", /* name */ times_result__doc__, /* doc */ times_result_fields, 5 @@ -16835,7 +16835,7 @@ static PyType_Slot DirEntryType_slots[] = { }; static PyType_Spec DirEntryType_spec = { - .name = MODNAME ".DirEntry", + .name = "os.DirEntry", .basicsize = sizeof(DirEntry), .flags = ( Py_TPFLAGS_DEFAULT @@ -17392,7 +17392,7 @@ static PyType_Slot ScandirIteratorType_slots[] = { }; static PyType_Spec ScandirIteratorType_spec = { - .name = MODNAME ".ScandirIterator", + .name = "os.ScandirIterator", .basicsize = sizeof(ScandirIterator), // bpo-40549: Py_TPFLAGS_BASETYPE should not be used, since // PyType_GetModule(Py_TYPE(self)) doesn't work on a subclass instance. From 149562e46f161b9da1dec0d20a87fefdaf0f0eea Mon Sep 17 00:00:00 2001 From: PavlNekrasov <95914807+PavlNekrasov@users.noreply.github.com> Date: Sat, 7 Nov 2026 02:00:04 +0300 Subject: [PATCH 013/229] gh-161477: Fix single-element tuple in idlelib redirector `__repr__` (#161481) Result is mangled repr(). Extremely unlikely to be seen by IDLE users, but idlelib maintainers might if internal error. --- Lib/idlelib/idle_test/test_redirector.py | 6 ++++-- Lib/idlelib/redirector.py | 4 ++-- .../IDLE/2026-11-06-17-12-40.gh-issue-161477.smXIKC.rst | 3 +++ 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2026-11-06-17-12-40.gh-issue-161477.smXIKC.rst diff --git a/Lib/idlelib/idle_test/test_redirector.py b/Lib/idlelib/idle_test/test_redirector.py index bd486d7da660108..58e7775071c19c1 100644 --- a/Lib/idlelib/idle_test/test_redirector.py +++ b/Lib/idlelib/idle_test/test_redirector.py @@ -64,8 +64,10 @@ def tearDown(self): self.redir.close() def test_repr(self): # partly for 100% coverage - self.assertIn('Redirector', repr(self.redir)) - self.assertIn('Original', repr(self.orig_insert)) + self.assertEqual(repr(self.redir), + f"WidgetRedirector(Text<{self.text._w}>)") + self.assertEqual(repr(self.orig_insert), + f"OriginalCommand({self.redir!r}, 'insert')") def test_register(self): self.assertEqual(self.text.get('1.0', 'end'), '\n') diff --git a/Lib/idlelib/redirector.py b/Lib/idlelib/redirector.py index 8e2ba68d3815bf0..821a45ee7982a47 100644 --- a/Lib/idlelib/redirector.py +++ b/Lib/idlelib/redirector.py @@ -48,7 +48,7 @@ def __init__(self, widget): def __repr__(self): w = self.widget - return f"{self.__class__.__name__,}({w.__class__.__name__}<{w._w}>)" + return f"{self.__class__.__name__}({w.__class__.__name__}<{w._w}>)" def close(self): "Unregister operations and revert redirection created by .__init__." @@ -143,7 +143,7 @@ def __init__(self, redir, operation): self.orig_and_operation = (redir.orig, operation) def __repr__(self): - return f"{self.__class__.__name__,}({self.redir!r}, {self.operation!r})" + return f"{self.__class__.__name__}({self.redir!r}, {self.operation!r})" def __call__(self, *args): return self.tk_call(self.orig_and_operation + args) diff --git a/Misc/NEWS.d/next/IDLE/2026-11-06-17-12-40.gh-issue-161477.smXIKC.rst b/Misc/NEWS.d/next/IDLE/2026-11-06-17-12-40.gh-issue-161477.smXIKC.rst new file mode 100644 index 000000000000000..ca955e608bc8936 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-11-06-17-12-40.gh-issue-161477.smXIKC.rst @@ -0,0 +1,3 @@ +Fix ``repr()`` of ``idlelib.redirector.WidgetRedirector`` and +``idlelib.redirector.OriginalCommand``, which displayed a spurious +one-element tuple. From 5106b9195fefa8eb954741661c7b3362e20f0953 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Fri, 6 Nov 2026 15:31:21 -0800 Subject: [PATCH 014/229] gh-161458: Move test_winconsoleio and test_file_eintr into test_io (#161460) * gh-161458: Move test_file_eintr into test_io * gh-161458: Move test_winconsoleio into test_io * gh-161458: Move test_free_threading/test_io into test_io as test_free_threading * gh-161458: Skip test_io.test_free_threading on GIL builds The test_free_threading package skipped these on GIL builds. * gh-161458: Remove the test module list from the test_io docstring It goes stale whenever a module is added or moved. * gh-161458: Point to I/O tests outside test_io in its docstring Co-authored-by: Claude Opus 5.5 --- Lib/test/test_io/__init__.py | 18 ++++++++---------- Lib/test/{ => test_io}/test_file_eintr.py | 0 .../test_free_threading.py} | 7 +++++-- Lib/test/{ => test_io}/test_winconsoleio.py | 0 4 files changed, 13 insertions(+), 12 deletions(-) rename Lib/test/{ => test_io}/test_file_eintr.py (100%) rename Lib/test/{test_free_threading/test_io.py => test_io/test_free_threading.py} (98%) rename Lib/test/{ => test_io}/test_winconsoleio.py (100%) diff --git a/Lib/test/test_io/__init__.py b/Lib/test/test_io/__init__.py index c94fad3e779381d..ab735e1538185fa 100644 --- a/Lib/test/test_io/__init__.py +++ b/Lib/test/test_io/__init__.py @@ -3,16 +3,14 @@ Tests are split across multiple files to increase parallelism and focus on specific implementation pieces. -* test_io - * test_bufferedio - tests file buffering - * test_memoryio - tests BytesIO and StringIO - * test_fileio - tests FileIO - * test_file - tests the file interface - * test_general - tests everything else in the io module - * test_univnewlines - tests universal newline support - * test_largefile - tests operations on a file greater than 2**32 bytes - (only enabled with -ulargefile) -* test_free_threading/test_io - tests thread safety of io objects +Tests for code that uses io live with that code, for example: + +* test_capi.test_file - the PyFile_* C API +* test_builtin, test_print - open(), input() and print() +* test_cmd_line, test_subprocess - stdio buffering and newlines at startup + and through pipes +* test_utf8_mode - UTF-8 mode encodings, including with _pyio +* test_os, test_eintr - fd-level I/O and EINTR retries .. attention:: When writing tests for io, it's important to test both the C and Python diff --git a/Lib/test/test_file_eintr.py b/Lib/test/test_io/test_file_eintr.py similarity index 100% rename from Lib/test/test_file_eintr.py rename to Lib/test/test_io/test_file_eintr.py diff --git a/Lib/test/test_free_threading/test_io.py b/Lib/test/test_io/test_free_threading.py similarity index 98% rename from Lib/test/test_free_threading/test_io.py rename to Lib/test/test_io/test_free_threading.py index a2b1ec8eb72bb1a..d4cbfe446d92285 100644 --- a/Lib/test/test_free_threading/test_io.py +++ b/Lib/test/test_io/test_free_threading.py @@ -2,12 +2,15 @@ import io import _pyio as pyio import threading -from unittest import TestCase -from test.support import threading_helper +from unittest import SkipTest, TestCase +from test.support import Py_GIL_DISABLED, threading_helper from test.support.threading_helper import run_concurrently from random import randint from sys import getsizeof +if not Py_GIL_DISABLED: + raise SkipTest("GIL enabled") + threading_helper.requires_working_threading(module=True) diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_io/test_winconsoleio.py similarity index 100% rename from Lib/test/test_winconsoleio.py rename to Lib/test/test_io/test_winconsoleio.py From ce4ac8c0459eba99459f03824cf1ac195c1aa7a1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 7 Nov 2026 01:23:58 +0100 Subject: [PATCH 015/229] gh-161517: Fix test_gdb on GHA Hypothesis tests (#161519) Fix test_gdb when Python is built out-of-tree and Python is run from a virtual environment. Previously, test_gdb was just skipped. Fix CHECKOUT_HOOK_PATH in test_gdb.util: get python-gdb.py from the build directory, rather than using the sys.executable directory. --- Lib/test/test_gdb/util.py | 11 +++++++++-- .../2026-11-06-23-15-23.gh-issue-161517.LkAlDt.rst | 3 +++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-11-06-23-15-23.gh-issue-161517.LkAlDt.rst diff --git a/Lib/test/test_gdb/util.py b/Lib/test/test_gdb/util.py index 29c7e9ee8f05498..ffe92137d844b71 100644 --- a/Lib/test/test_gdb/util.py +++ b/Lib/test/test_gdb/util.py @@ -11,9 +11,16 @@ GDB_PROGRAM = shutil.which('gdb') or 'gdb' +# First, try to use the build directory to support out-of-tree build +for key in ('abs_builddir', 'abs_srcdir', 'srcdir'): + SRC_DIR = sysconfig.get_config_var(key) + if SRC_DIR: + break +else: + raise Exception("unable to get the Python source directory") + # Location of custom hooks file in a repository checkout. -CHECKOUT_HOOK_PATH = os.path.join(os.path.dirname(sys.executable), - 'python-gdb.py') +CHECKOUT_HOOK_PATH = os.path.join(SRC_DIR, 'python-gdb.py') SAMPLE_SCRIPT = os.path.join(os.path.dirname(__file__), 'gdb_sample.py') BREAKPOINT_FN = 'builtin_id' diff --git a/Misc/NEWS.d/next/Tests/2026-11-06-23-15-23.gh-issue-161517.LkAlDt.rst b/Misc/NEWS.d/next/Tests/2026-11-06-23-15-23.gh-issue-161517.LkAlDt.rst new file mode 100644 index 000000000000000..fc76d235d207f0e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-11-06-23-15-23.gh-issue-161517.LkAlDt.rst @@ -0,0 +1,3 @@ +Fix test_gdb when Python is built out-of-tree and Python is run from a +virtual environment. Previously, test_gdb was just skipped. Patch by Victor +Stinner. From 6c5596c7199792ea1e8cf26f65da8cd0023c0473 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Sat, 7 Nov 2026 11:23:41 +0500 Subject: [PATCH 016/229] GH-159101: Fix UAF when tp_finalize changes the base class (#159888) Co-authored-by: Peter Bierma --- .../2026-10-20-20-38-38.gh-issue-159101.vFwrZY.rst | 2 ++ Objects/object.c | 8 ++++---- Objects/typeobject.c | 4 ++++ 3 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-20-38-38.gh-issue-159101.vFwrZY.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-20-38-38.gh-issue-159101.vFwrZY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-20-38-38.gh-issue-159101.vFwrZY.rst new file mode 100644 index 000000000000000..33beb8d571d7eca --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-20-38-38.gh-issue-159101.vFwrZY.rst @@ -0,0 +1,2 @@ +Fix use-after-free when :meth:`~object.__del__` changes the base class. Patch by +Sergey Miryanov. diff --git a/Objects/object.c b/Objects/object.c index 757e54aa1d487d7..0695cc090c979c4 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -583,13 +583,13 @@ PyObject_CallFinalizer(PyObject *self) if (tp->tp_finalize == NULL) return; /* tp_finalize should only be called once. */ - if (_PyType_IS_GC(tp) && _PyGC_FINALIZED(self)) - return; - - tp->tp_finalize(self); if (_PyType_IS_GC(tp)) { + if (_PyGC_FINALIZED(self)) + return; + _PyGC_SET_FINALIZED(self); } + tp->tp_finalize(self); } int diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 66c7bc7f8902fc3..9833c338679d33f 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2827,6 +2827,10 @@ subtype_dealloc(PyObject *self) } _PyObject_GC_UNTRACK(self); } + + /* Extract the type again; tp_finalize may have changed it */ + type = Py_TYPE(self); + /* If we added a weaklist, we clear it. Do this *before* calling tp_del, clearing slots, or clearing the instance dict. From b8dd06f1d6ee3b172c58d688f122b29bb2582f9a Mon Sep 17 00:00:00 2001 From: Maik Schulze Date: Sat, 7 Nov 2026 08:21:14 +0100 Subject: [PATCH 017/229] gh-161420: Process mimalloc's delayed frees on every generic malloc again (gh-161533) --- .../2026-11-07-09-30-00.gh-issue-161420.mIdFre.rst | 2 ++ Objects/mimalloc/page.c | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-07-09-30-00.gh-issue-161420.mIdFre.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-07-09-30-00.gh-issue-161420.mIdFre.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-07-09-30-00.gh-issue-161420.mIdFre.rst new file mode 100644 index 000000000000000..296a2267f397be0 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-07-09-30-00.gh-issue-161420.mIdFre.rst @@ -0,0 +1,2 @@ +Restore ``mimalloc``'s previous behavior of processing delayed frees from other +threads on every generic malloc call, reverting the change introduced in v2.2.2. diff --git a/Objects/mimalloc/page.c b/Objects/mimalloc/page.c index 4bce8ce76044cbe..2f26d04aaaa4d87 100644 --- a/Objects/mimalloc/page.c +++ b/Objects/mimalloc/page.c @@ -1071,6 +1071,12 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_al } mi_assert_internal(mi_heap_is_initialized(heap)); + // Process delayed frees from other threads (but skip contended ones). + // CPython: gh-161420 showed that calling _mi_heap_delayed_free_partial only + // once every 100 generic malloc calls led to high memory usage. + // To avoid this, we now process delayed frees on every generic malloc call again. + _mi_heap_delayed_free_partial(heap); + // do administrative tasks every N generic mallocs if mi_unlikely(++heap->generic_count >= 100) { heap->generic_collect_count += heap->generic_count; @@ -1078,9 +1084,6 @@ void* _mi_malloc_generic(mi_heap_t* heap, size_t size, bool zero, size_t huge_al // call potential deferred free routines _mi_deferred_free(heap, false); - // free delayed frees from other threads (but skip contended ones) - _mi_heap_delayed_free_partial(heap); - // collect every once in a while (10000 by default) const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L); if (heap->generic_collect_count >= generic_collect) { From 301643a9e331679d50fba596a32d3119f99dc1bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Delfino?= Date: Sat, 7 Nov 2026 05:28:27 -0300 Subject: [PATCH 018/229] gh-161415: Remove nonexistent related help topics in `pydoc` (#161417) The TYPES topic listed FILES, which was removed from Helper.topics in Python 3.4, and the UNICODE topic listed unicode, the Python 2 built-in type. help() finds no entry for either of them. --- Lib/pydoc.py | 4 ++-- Lib/test/test_pydoc/test_pydoc.py | 13 +++++++++++++ .../2026-11-05-12-00-24.gh-issue-161415.NJhrAR.rst | 3 +++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-05-12-00-24.gh-issue-161415.NJhrAR.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index a586d35866b038f..e1efe8fc0a1b631 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -2028,12 +2028,12 @@ class Helper: topics = { 'TYPES': ('types', 'STRINGS UNICODE NUMBERS SEQUENCES MAPPINGS ' - 'FUNCTIONS CLASSES MODULES FILES inspect'), + 'FUNCTIONS CLASSES MODULES inspect'), 'STRINGS': ('strings', 'str UNICODE SEQUENCES STRINGMETHODS ' 'FORMATTING TYPES'), 'STRINGMETHODS': ('string-methods', 'STRINGS FORMATTING'), 'FORMATTING': ('formatstrings', 'OPERATORS'), - 'UNICODE': ('strings', 'encodings unicode SEQUENCES STRINGMETHODS ' + 'UNICODE': ('strings', 'encodings SEQUENCES STRINGMETHODS ' 'FORMATTING TYPES'), 'NUMBERS': ('numbers', 'INTEGER FLOAT COMPLEX TYPES'), 'INTEGER': ('integers', 'int range'), diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index a63530c29058275..552b0c7d0759498 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -2294,6 +2294,19 @@ def test_keywords(self): self.assertEqual(sorted(pydoc.Helper.keywords), sorted(keyword.kwlist + ['case', 'match', 'lazy'])) + def test_related_topics(self): + # Every related help topic and alias must be something help() can + # document. + for name, target in (*pydoc.Helper.topics.items(), + *pydoc.Helper.keywords.items()): + if isinstance(target, str): + target = ('', target) + for xref in target[1].split(): + with self.subTest(topic=name, xref=xref): + buf = StringIO() + pydoc.Helper(output=buf).help(xref) + self.assertNotIn('No help entry found', buf.getvalue()) + def test_symbols(self): # gh-161414 for symbol in pydoc.Helper.symbols: diff --git a/Misc/NEWS.d/next/Library/2026-11-05-12-00-24.gh-issue-161415.NJhrAR.rst b/Misc/NEWS.d/next/Library/2026-11-05-12-00-24.gh-issue-161415.NJhrAR.rst new file mode 100644 index 000000000000000..22cc4665c253dde --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-05-12-00-24.gh-issue-161415.NJhrAR.rst @@ -0,0 +1,3 @@ +Remove ``FILES`` and ``unicode`` from the related help topics listed by +:func:`help` for the ``TYPES`` and ``UNICODE`` topics. Neither exists in +Python 3, so asking for help on them failed. From cbaba022cf2778c5f02445ae231a8af1f1db53a1 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 7 Nov 2026 17:47:11 +0900 Subject: [PATCH 019/229] docs: Fix typos and obvious errors (GH-161386) * docs: Fix typos in API references and examples * docs: Revert turtle example change * docs: Fix unresolved ensure_value references in optparse * docs: Match tracemalloc example traceback order to its output * Apply batched suggestions from code review --------- Co-authored-by: Stan Ulbrych --- Doc/c-api/module.rst | 6 +++--- Doc/c-api/typeobj.rst | 2 +- Doc/extending/embedding.rst | 19 ++++++++++--------- Doc/extending/extending.rst | 2 +- Doc/howto/descriptor.rst | 2 +- Doc/howto/instrumentation.rst | 4 ++-- Doc/howto/isolating-extensions.rst | 4 ++-- Doc/howto/logging-cookbook.rst | 2 +- Doc/library/annotationlib.rst | 2 +- Doc/library/argparse.rst | 6 +++--- Doc/library/asyncio-eventloop.rst | 2 +- Doc/library/binascii.rst | 10 +++++----- Doc/library/calendar.rst | 8 ++++---- Doc/library/cgi.rst | 4 ++-- Doc/library/cgitb.rst | 4 ++-- Doc/library/codecs.rst | 2 +- Doc/library/doctest.rst | 2 +- Doc/library/ftplib.rst | 4 ++-- Doc/library/http.client.rst | 2 +- Doc/library/http.cookies.rst | 2 +- Doc/library/io.rst | 2 +- Doc/library/logging.rst | 6 +++--- Doc/library/multiprocessing.rst | 2 +- Doc/library/optparse.rst | 12 ++++++------ Doc/library/os.rst | 4 ++-- Doc/library/pdb.rst | 2 +- Doc/library/profile.rst | 4 ++-- Doc/library/profiling.sampling.rst | 2 +- Doc/library/ssl.rst | 4 ++-- Doc/library/tarfile.rst | 4 ++-- Doc/library/threading.rst | 4 ++-- Doc/library/timeit.rst | 2 +- Doc/library/tracemalloc.rst | 2 +- Doc/library/xml.dom.rst | 2 +- Doc/whatsnew/2.3.rst | 4 ++-- Doc/whatsnew/2.4.rst | 4 ++-- Doc/whatsnew/2.5.rst | 4 ++-- Doc/whatsnew/2.7.rst | 2 +- Doc/whatsnew/3.1.rst | 2 +- Doc/whatsnew/3.11.rst | 4 ++-- Doc/whatsnew/3.12.rst | 13 +++++++------ Doc/whatsnew/3.15.rst | 2 +- Doc/whatsnew/3.4.rst | 4 ++-- Doc/whatsnew/3.6.rst | 10 +++++----- Doc/whatsnew/3.7.rst | 4 ++-- Doc/whatsnew/3.8.rst | 2 +- Doc/whatsnew/3.9.rst | 4 ++-- 47 files changed, 101 insertions(+), 99 deletions(-) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 6e4e3c48c084295..9ec1407c88d01be 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -552,19 +552,19 @@ This means that if you have a module object, but you are not sure if it .. code-block:: c - PyObject *module = + PyObject *module = ; void *module_token; if (PyModule_GetToken(module, &module_token) < 0) { return NULL; } if (module_token != your_token) { - PyErr_SetString(PyExc_ValueError, "unexpected module") + PyErr_SetString(PyExc_ValueError, "unexpected module"); return NULL; } // This module's state has the expected memory layout; it's safe to cast - struct my_state state = (struct my_state*)PyModule_GetState(module) + struct my_state *state = (struct my_state*)PyModule_GetState(module); A module's token -- and the *your_token* value to use in the above code -- is: diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 56dd0bd32470cbc..c2b9124c9201ba0 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -3343,7 +3343,7 @@ A type that supports weakrefs, instance dicts, and hashing:: .tp_new = myobj_new, .tp_traverse = (traverseproc)myobj_traverse, .tp_clear = (inquiry)myobj_clear, - .tp_alloc = PyType_GenericNew, + .tp_alloc = PyType_GenericAlloc, .tp_dealloc = (destructor)myobj_dealloc, .tp_repr = (reprfunc)myobj_repr, .tp_hash = (hashfunc)myobj_hash, diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index cb41889437c8b0d..8a87d7cb1c3b4ca 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -95,15 +95,16 @@ perform some operation on a file. :: See :ref:`arg-parsing-string-and-buffers` for a description of this macro. Setting :c:member:`PyConfig.program_name` should be called before -:c:func:`Py_InitializeFromConfig` to inform the interpreter about paths to Python run-time -libraries. Next, the Python interpreter is initialized with -:c:func:`Py_Initialize`, followed by the execution of a hard-coded Python script -that prints the date and time. Afterwards, the :c:func:`Py_FinalizeEx` call shuts -the interpreter down, followed by the end of the program. In a real program, -you may want to get the Python script from another source, perhaps a text-editor -routine, a file, or a database. Getting the Python code from a file can better -be done by using the :c:func:`PyRun_SimpleFile` function, which saves you the -trouble of allocating memory space and loading the file contents. +:c:func:`Py_InitializeFromConfig` to inform the interpreter about paths to +Python run-time libraries. Next, the Python interpreter is initialized with +:c:func:`Py_InitializeFromConfig`, followed by the execution of a hard-coded +Python script that prints the date and time. Afterwards, the +:c:func:`Py_FinalizeEx` call shuts the interpreter down, followed by the end of +the program. In a real program, you may want to get the Python script from +another source, perhaps a text-editor routine, a file, or a database. Getting +the Python code from a file can better be done by using the +:c:func:`PyRun_SimpleFile` function, which saves you the trouble of allocating +memory space and loading the file contents. .. _lower-level-embedding: diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 6bf27ce33c31826..98ec3f7f993efba 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -1040,7 +1040,7 @@ exporting module, not a client module. Finally, the module's :c:data:`mod_exec } Note that ``PySpam_API`` is declared ``static``; otherwise the pointer -array would disappear when :c:func:`!PyInit_spam` terminates! +array would disappear when :c:func:`!spam_module_exec` terminates! The bulk of the work is in the header file :file:`spammodule.h`, which looks like this:: diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 801d76dd80aabfd..3962b3e3467a45b 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1678,7 +1678,7 @@ variables: def __new__(mcls, clsname, bases, mapping, **kwargs): 'Emulate type_new() in Objects/typeobject.c' - # type_new() calls PyTypeReady() which calls add_methods() + # type_new() calls PyType_Ready() which calls add_methods() slot_names = mapping.get('slot_names', []) for offset, name in enumerate(slot_names): mapping[name] = Member(name, clsname, offset) diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index c7ce5785047a6d5..e0d08b19db7a88f 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -430,7 +430,7 @@ Here is a tapset file, based on a non-shared build of CPython: /* Provide a higher-level wrapping around the function__entry and function__return markers: - \*/ + */ probe python.function.entry = process("python").mark("function__entry") { filename = user_string($arg1); @@ -498,7 +498,7 @@ frames, each second, across the whole system: } probe timer.ms(1000) { - printf("\033[2J\033[1;1H") /* clear screen \*/ + printf("\033[2J\033[1;1H") /* clear screen */ printf("%6s %80s %6s %30s %6s\n", "PID", "FILENAME", "LINE", "FUNCTION", "CALLS") foreach ([pid, filename, funcname, lineno] in fn_calls- limit 20) { diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 302c15528d5ef31..fcf2abb8157fa46 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -557,9 +557,9 @@ For example:: {"example_method", (PyCFunction)(void(*)(void))example_method, METH_METHOD|METH_FASTCALL|METH_KEYWORDS, - example_method_doc} + example_method_doc}, {NULL}, - } + }; Module State Access from Slot Methods, Getters and Setters diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 687a0c516f67173..c105620fc3c04aa 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1085,7 +1085,7 @@ each request is handled by a thread: # A dummy set of requests which will be used in the simulation - we'll just pick # from this list randomly. Note that all GET requests are from 192.168.2.XXX - # addresses, whereas POST requests are from 192.16.3.XXX addresses. Three users + # addresses, whereas POST requests are from 192.168.3.XXX addresses. Three users # are represented in the sample requests. REQUESTS = [ diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 5b22353ea08e647..1755a6582bf9573 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -581,7 +581,7 @@ Or used as the annotate function for an object: >>> class C: ... pass >>> C.__annotate__ = Annotate() - >>> get_annotations(Annotate(), format=Format.STRING) + >>> get_annotations(C, format=Format.STRING) {'x': 'MyType'} diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 02d6940ad06d2c0..ac4f12de6fb663a 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -746,11 +746,11 @@ the upper-cased name. For example:: >>> parser.parse_args(['--foo-bar', 'FOO-BAR']) Namespace(foo_bar='FOO-BAR') >>> parser.print_help() - usage: [-h] [--foo-bar FOO-BAR] + usage: [-h] [--foo-bar FOO_BAR] optional arguments: -h, --help show this help message and exit - --foo-bar FOO-BAR + --foo-bar FOO_BAR .. _action: @@ -1375,7 +1375,7 @@ behavior:: >>> parser.parse_args('-f 1 -q 2 -x 3'.split()) Namespace(foo_bar='1', quz='2', x='3') >>> parser.parse_args('--foo 1 -quz 2 -y 3'.split()) - Namespace(foo_bar='1', quz='2', x='2') + Namespace(foo_bar='1', quz='2', x='3') ``dest`` allows a custom attribute name to be provided:: diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index b847d73ae09b4dc..10feca3203aec55 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -2029,7 +2029,7 @@ Display the current date with call_later() An example of a callback displaying the current date every second. The callback uses the :meth:`loop.call_later` method to reschedule itself -after 5 seconds, and then stops the event loop:: +after 1 second, and eventually stops the event loop:: import asyncio import datetime as dt diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index be9ad897bcf2ee3..60d2b878aa455af 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -301,11 +301,11 @@ The :mod:`!binascii` module defines the following functions: encoding. The return value is the converted line(s). If the optional argument *quotetabs* is present and true, all tabs and spaces will be encoded. If the optional argument *istext* is present and true, newlines are not encoded but - trailing whitespace will be encoded. If the optional argument *header* is - present and true, spaces will be encoded as underscores per :rfc:`1522`. If the - optional argument *header* is present and false, newline characters will be - encoded as well; otherwise linefeed conversion might corrupt the binary data - stream. + trailing whitespace will be encoded. If the optional argument *istext* is + present and false, newline characters will be encoded as well; otherwise + linefeed conversion might corrupt the binary data stream. If the optional + argument *header* is present and true, spaces will be encoded as underscores + per :rfc:`1522`. .. function:: crc_hqx(data, value) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 18e0319d91fad95..69578e4a4846ced 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -162,10 +162,10 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is final newline. - .. method:: formatday(theday, weekday, width) + .. method:: formatday(day, weekday, width) Return a string representing a single day formatted with the given *width*. - If *theday* is ``0``, return a string of spaces of + If *day* is ``0``, return a string of spaces of the specified width, representing an empty day. The *weekday* parameter is unused. @@ -177,10 +177,10 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :meth:`setfirstweekday` method. - .. method:: formatweekday(weekday, width) + .. method:: formatweekday(day, width) Return a string representing the name of a single weekday formatted to - the specified *width*. The *weekday* parameter is an integer representing + the specified *width*. The *day* parameter is an integer representing the day of the week, where ``0`` is Monday and ``6`` is Sunday. diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index f9108fa954a906f..6e4d277fcefe754 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -12,8 +12,8 @@ It was :ref:`removed in Python 3.13 ` after being deprecated in Python 3.11. The removal was decided in :pep:`594`. A fork of the module on PyPI can be used instead: :pypi:`legacy-cgi`. -This is a copy of the cgi module, no longer maintained or supported by the core -Python team. +This package contains copies of the :mod:`!cgi` and :mod:`!cgitb` modules, +which are no longer maintained or supported by the core Python team. The last version of Python that provided the :mod:`!cgi` module was `Python 3.12 `_. diff --git a/Doc/library/cgitb.rst b/Doc/library/cgitb.rst index fc646aa4c48acd0..081a4f561bd3667 100644 --- a/Doc/library/cgitb.rst +++ b/Doc/library/cgitb.rst @@ -12,8 +12,8 @@ It was :ref:`removed in Python 3.13 ` after being deprecated in Python 3.11. The removal was decided in :pep:`594`. A fork of the module on PyPI can now be used instead: :pypi:`legacy-cgi`. -This is a copy of the cgi module, no longer maintained or supported by the core -Python team. +This package contains copies of the :mod:`!cgi` and :mod:`!cgitb` modules, +which are no longer maintained or supported by the core Python team. The last version of Python that provided the :mod:`!cgitb` module was `Python 3.12 `_. diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index ca4f5f6dfea4f4a..f19407818528e20 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1642,7 +1642,7 @@ This module implements the following exception: :synopsis: Internationalized Domain Names implementation This module implements :rfc:`3490` (Internationalized Domain Names in -Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for +Applications) and :rfc:`3491` (Nameprep: A Stringprep Profile for Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding and :mod:`stringprep`. diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 894929ac99634a1..de4ac60101a2733 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1642,7 +1642,7 @@ DocTestRunner objects verbosity is not specified, then the :class:`DocTestRunner`'s verbosity is used. - :class:`DocTestParser` has the following attributes: + :class:`DocTestRunner` has the following attributes: .. attribute:: tries diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index a0a59868a5ace99..15041a125ba92ad 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -92,7 +92,7 @@ FTP objects :param str user: |param_doc_user| - If given, :code:`login(host, passwd, acct)` is implicitly called + If given, :code:`login(user, passwd, acct)` is implicitly called by the constructor. :param str passwd: @@ -464,7 +464,7 @@ FTP_TLS objects :param str user: |param_doc_user| - If given, :code:`login(host, passwd, acct)` is implicitly called + If given, :code:`login(user, passwd, acct)` is implicitly called by the constructor. :param str passwd: diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index ab557f1805e1049..055971ec21aee37 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -663,7 +663,7 @@ method attribute. Here is an example session that uses the ``PUT`` method:: >>> conn.request("PUT", "/file", BODY) >>> response = conn.getresponse() >>> print(response.status, response.reason) - 200, OK + 200 OK .. _httpmessage-objects: diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index 4965c5fc3ba1d86..4a8d461213f4e57 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -163,7 +163,7 @@ Morsel Objects that the cookie was first set in. For this to be accepted by the user agent, you **must** also set ``Secure``. - In addition, it is recommended to use the ``__Host`` prefix when setting + In addition, it is recommended to use the ``__Host-`` prefix when setting partitioned cookies to make them bound to the hostname and not the registrable domain. Read `CHIPS (Cookies Having Independent Partitioned State)`_ diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 09e9e42565a99ef..495505ec63f8d13 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -205,7 +205,7 @@ High-level module interface def read_text(path, encoding=None): encoding = io.text_encoding(encoding) # stacklevel=2 - with open(path, encoding) as f: + with open(path, encoding=encoding) as f: return f.read() In this example, an :class:`EncodingWarning` is emitted for the caller of diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index a3d117c10702414..123cebf7189041f 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -747,14 +747,14 @@ Formatter Objects .. versionchanged:: 3.3 Previously, the default format was hard-coded as in this example: ``2010-09-06 22:38:15,292`` where the part before the comma is - handled by a strptime format string (``'%Y-%m-%d %H:%M:%S'``), and the - part after the comma is a millisecond value. Because strptime does not + handled by a strftime format string (``'%Y-%m-%d %H:%M:%S'``), and the + part after the comma is a millisecond value. Because strftime does not have a format placeholder for milliseconds, the millisecond value is appended using another format string, ``'%s,%03d'`` --- and both of these format strings have been hardcoded into this method. With the change, these strings are defined as class-level attributes which can be overridden at the instance level when desired. The names of the - attributes are ``default_time_format`` (for the strptime format string) + attributes are ``default_time_format`` (for the strftime format string) and ``default_msec_format`` (for appending the millisecond value). .. versionchanged:: 3.9 diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index cd376bd49f43eda..dc45f8401541f45 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -2750,7 +2750,7 @@ multiple connections at the same time. assumed to be the fastest available. See :ref:`multiprocessing-address-formats`. Note that if *family* is ``'AF_UNIX'`` and address is ``None`` then the socket will be created in a - private temporary directory created using :func:`tempfile.mkstemp`. + private temporary directory created using :func:`tempfile.mkdtemp`. If the listener object uses a socket then *backlog* (1 by default) is passed to the :meth:`~socket.socket.listen` method of the socket once it has been diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 4271e65e142cad3..3486c83a6576435 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -2003,7 +2003,7 @@ To add new types, you need to define your own subclass of :mod:`!optparse`'s (e.g., ``-f``), and ``value`` is the string from the command line that must be checked and converted to your desired type. ``check_mytype()`` should return an object of the hypothetical type ``mytype``. The value returned by - a type-checking function will wind up in the OptionValues instance returned + a type-checking function will wind up in the Values instance returned by :meth:`OptionParser.parse_args`, or be passed to a callback as the ``value`` parameter. @@ -2070,7 +2070,7 @@ Adding new actions is a bit trickier, because you have to understand that "store" actions actions that result in :mod:`!optparse` storing a value to an attribute of the - current OptionValues instance; these options require a :attr:`~Option.dest` + current Values instance; these options require a :attr:`~Option.dest` attribute to be supplied to the Option constructor. "typed" actions @@ -2152,8 +2152,8 @@ Features of note: control back to :meth:`Option.take_action` for the standard :mod:`!optparse` actions. -* ``values`` is an instance of the optparse_parser.Values class, which provides - the very useful :meth:`ensure_value` method. :meth:`ensure_value` is +* ``values`` is an instance of the optparse.Values class, which provides + the very useful :meth:`!ensure_value` method. :meth:`!ensure_value` is essentially :func:`getattr` with a safety valve; it is called as :: values.ensure_value(attr, value) @@ -2163,9 +2163,9 @@ Features of note: very handy for actions like ``"extend"``, ``"append"``, and ``"count"``, all of which accumulate data in a variable and expect that variable to be of a certain type (a list for the first two, an integer for the latter). Using - :meth:`ensure_value` means that scripts using your action don't have to worry + :meth:`!ensure_value` means that scripts using your action don't have to worry about setting a default value for the option destinations in question; they - can just leave the default as ``None`` and :meth:`ensure_value` will take care of + can just leave the default as ``None`` and :meth:`!ensure_value` will take care of getting it right when it's needed. Exceptions diff --git a/Doc/library/os.rst b/Doc/library/os.rst index ca8b60d57ead764..76403d80955b476 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3374,7 +3374,7 @@ features: .. versionchanged:: 3.12 ``st_ctime_ns`` is deprecated on Windows. Use ``st_birthtime_ns`` - for the file creation time. In the future, ``st_ctime`` will contain + for the file creation time. In the future, ``st_ctime_ns`` will contain the time of the most recent metadata change, as for other platforms. .. attribute:: st_birthtime @@ -5456,7 +5456,7 @@ written in Python, such as a mail server's external command delivery program. parent's signal mask. This argument corresponds to the C library :c:macro:`!POSIX_SPAWN_SETSIGMASK` flag. - The *sigdef* argument will reset the disposition of all signals in the set + The *setsigdef* argument will reset the disposition of all signals in the set specified. This argument corresponds to the C library :c:macro:`!POSIX_SPAWN_SETSIGDEF` flag. diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 995a9756a4ef53b..df5de52a412efac 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -812,7 +812,7 @@ can be overridden by the local file. List or jump between chained exceptions. - When using ``pdb.pm()`` or ``Pdb.post_mortem(...)`` with a chained exception + When using ``pdb.pm()`` or ``pdb.post_mortem(...)`` with a chained exception instead of a traceback, it allows the user to move between the chained exceptions using ``exceptions`` command to list exceptions, and ``exceptions `` to switch to that exception. diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index f7e85d1598727f9..324f69ef9d31c5a 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -92,7 +92,7 @@ following functions: the *timer* argument. This must be a function that returns a single number representing the current time. If the number is an integer, the *timeunit* specifies a multiplier that specifies the duration of each unit of time. For - example, if the timer returns times measured in thousands of seconds, the + example, if the timer returns times measured in thousandths of seconds, the time unit would be ``.001``. Directly using the :class:`Profile` class allows formatting profile results @@ -345,7 +345,7 @@ you are using :class:`profile.Profile` or :class:`profiling.tracing.Profile`, ``your_time_func`` should return a single number. If it returns integers, you can also invoke the class constructor with a second argument specifying the real duration of one unit of time. For example, if - ``your_integer_time_func`` returns times measured in thousands of seconds, + ``your_integer_time_func`` returns times measured in thousandths of seconds, you would construct the :class:`Profile` instance as follows:: pr = profiling.tracing.Profile(your_integer_time_func, 0.001) diff --git a/Doc/library/profiling.sampling.rst b/Doc/library/profiling.sampling.rst index 7bb0e966583bcbb..ef88583d8c2dd1b 100644 --- a/Doc/library/profiling.sampling.rst +++ b/Doc/library/profiling.sampling.rst @@ -1051,7 +1051,7 @@ an interactive flame graph visualization:: `Try the interactive example <../_static/tachyon-example-flamegraph.html>`__! If no output file is specified, the profiler generates a filename based on -the process ID (for example, ``flamegraph.12345.html``). +the process ID (for example, ``flamegraph_12345.html``). The generated HTML file requires no external dependencies and can be opened directly in a web browser. The visualization displays call stacks as nested diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 139dc2c24eab764..d177abd48fa9cd4 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2860,7 +2860,7 @@ Visual inspection shows that the certificate does identify the desired service Now the SSL channel is established and the certificate verified, you can proceed to talk with the server:: - >>> conn.sendall(b"HEAD / HTTP/1.0\r\nHost: linuxfr.org\r\n\r\n") + >>> conn.sendall(b"HEAD / HTTP/1.0\r\nHost: www.python.org\r\n\r\n") >>> pprint.pprint(conn.recv(1024).split(b"\r\n")) [b'HTTP/1.1 200 OK', b'Date: Sat, 18 Oct 2014 18:27:20 GMT', @@ -3072,7 +3072,7 @@ provided. When compared to :class:`SSLSocket`, this object lacks the following features: - - Any form of network IO; ``recv()`` and ``send()`` read and write only to + - Any form of network IO; ``read()`` and ``write()`` read and write only to the underlying :class:`MemoryBIO` buffers. - There is no *do_handshake_on_connect* machinery. You must always manually diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 98d1ebfd7f603e2..092332812296ccf 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -781,7 +781,7 @@ call the :meth:`~TarInfo.replace` method to create a modified copy in one step. Several attributes can be set to ``None`` to indicate that a piece of metadata is unused or unknown. -Different :class:`TarInfo` methods handle ``None`` differently: +Different :class:`TarFile` methods handle ``None`` differently: - The :meth:`~TarFile.extract` or :meth:`~TarFile.extractall` methods will ignore the corresponding metadata, leaving it set to a default. @@ -1476,7 +1476,7 @@ The same example using the :keyword:`with` statement:: How to create and write an archive to stdout using :data:`sys.stdout.buffer ` in the *fileobj* parameter -in :meth:`TarFile.add`:: +in :func:`~tarfile.open`:: import sys import tarfile diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 5d9a7b6314b1668..5e80c30953955da 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -821,7 +821,7 @@ locks. In the locked state, some thread owns the lock; in the unlocked state, no thread owns it. Threads call a lock's :meth:`~RLock.acquire` method to lock it, -and its :meth:`~Lock.release` method to unlock it. +and its :meth:`~RLock.release` method to unlock it. .. note:: @@ -832,7 +832,7 @@ and its :meth:`~Lock.release` method to unlock it. RLock's :meth:`~RLock.acquire`/:meth:`~RLock.release` call pairs may be nested, unlike Lock's :meth:`~Lock.acquire`/:meth:`~Lock.release`. Only the final -:meth:`~RLock.release` (the :meth:`~Lock.release` of the outermost pair) resets +:meth:`~RLock.release` (the :meth:`~RLock.release` of the outermost pair) resets the lock to an unlocked state and allows another thread blocked in :meth:`~RLock.acquire` to proceed. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index dd574fce09f5fc8..09da385bfdf79a1 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -148,7 +148,7 @@ The module defines three convenience functions and a public class: Automatically determine how many times to call :meth:`.timeit`. This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= *Timer.target_time* seconds, returning the eventual + so that the total time >= *target_time* seconds, returning the eventual (number of loops, time taken for that number of loops). It calls :meth:`.timeit` with increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the time taken is at least *target_time* seconds. diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 4a19acb430b0d13..6ce677b15a51b4a 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -750,7 +750,7 @@ Traceback Example:: print("Traceback (most recent call first):") - for line in traceback: + for line in traceback.format(most_recent_first=True): print(line) Output:: diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index e5b5147720802e3..208eb4ad414f706 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -515,7 +515,7 @@ NodeList Objects A :class:`NodeList` represents a sequence of nodes. These objects are used in two ways in the DOM Core recommendation: an :class:`Element` object provides one as its list of child nodes, and the :meth:`~Element.getElementsByTagName` -and :meth:`~Element.getElementsByTagNameNS` methods of :class:`Node` return +and :meth:`~Element.getElementsByTagNameNS` methods of :class:`Element` return objects with this interface to represent query results. :class:`NodeList` does *not* inherit from :class:`Node`. diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index f2d372440717517..984f9d363eb7187 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1357,7 +1357,7 @@ complete list of changes, or look through the CVS logs for all the details. other than ``e`` and ``10``. (Contributed by Raymond Hettinger.) * Several new POSIX functions (:func:`!getpgid`, :func:`!killpg`, :func:`!lchown`, - :func:`!loadavg`, :func:`!major`, :func:`!makedev`, :func:`!minor`, and + :func:`!getloadavg`, :func:`!major`, :func:`!makedev`, :func:`!minor`, and :func:`!mknod`) were added to the :mod:`posix` module that underlies the :mod:`os` module. (Contributed by Gustavo Niemeyer, Geert Jansen, and Denis S. Otkidach.) @@ -1658,7 +1658,7 @@ complete list of changes, or look through the CVS logs for all the details. has been added. The "idna" encoding can be used to convert between a Unicode domain name and the ASCII-compatible encoding (ACE) of that name. :: - >{}>{}> u"www.Alliancefrançaise.nu".encode("idna") + >>> u"www.Alliancefrançaise.nu".encode("idna") 'www.xn--alliancefranaise-npb.nu' The :mod:`socket` module has also been extended to transparently convert diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index ae3e9ed4646eeda..a3c75ec7f8e39ad 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -1072,7 +1072,7 @@ complete list of changes, or look through the CVS logs for all the details. 3.0 version of the package uses a new incremental parser for MIME messages, available in the :mod:`!email.FeedParser` module. The new parser doesn't require reading the entire message into memory, and doesn't raise exceptions if a - message is malformed; instead it records any problems in the :attr:`!defect` + message is malformed; instead it records any problems in the :attr:`!defects` attribute of the message. (Developed by Anthony Baxter, Barry Warsaw, Thomas Wouters, and others.) @@ -1177,7 +1177,7 @@ complete list of changes, or look through the CVS logs for all the details. import logging logging.basicConfig(filename='/var/log/application.log', level=0, # Log all messages - format='%(levelname):%(process):%(thread):%(message)') + format='%(levelname)s:%(process)d:%(thread)d:%(message)s') Other additions to the :mod:`logging` package include a ``log(level, msg)`` convenience method, as well as a :class:`~logging.handlers.TimedRotatingFileHandler` class that diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 33d8b741ba17668..1e0c2bec9fb98ef 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -1738,7 +1738,7 @@ constructor that will create a :c:expr:`PyObject *` pointer. A simple usage:: d = {} ctypes.pythonapi.PyObject_SetItem(ctypes.py_object(d), ctypes.py_object("abc"), ctypes.py_object(1)) - # d is now {'abc', 1}. + # d is now {'abc': 1}. Don't forget to use :func:`~ctypes.py_object`; if it's omitted you end up with a segmentation fault. @@ -2182,7 +2182,7 @@ Changes to Python's build process and to the C API include: specified *type* and uses a fast-calling qualifier. :c:macro:`Py_LOCAL_INLINE` does the same thing and also requests the function be inlined. If macro :c:macro:`!PY_LOCAL_AGGRESSIVE` is defined before - :file:`python.h` is included, a set of more aggressive optimizations are enabled + :file:`Python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these optimizations actually make the code faster. (Contributed by Fredrik Lundh at the NeedForSpeed sprint.) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index de90c37a38d38ff..bc766e37e380e82 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2298,7 +2298,7 @@ and getting the pointer requires providing the name: void *vtable; - if (!PyCapsule_IsValid(capsule, "mymodule.CAPI") { + if (!PyCapsule_IsValid(capsule, "mymodule.CAPI")) { PyErr_SetString(PyExc_ValueError, "argument type invalid"); return NULL; } diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index 9dc63c7976c5123..6c170abd402b871 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -310,7 +310,7 @@ New, Improved, and Deprecated Modules CSV header, SQL field list, or user input:: >>> query = input() - SELECT region, dept, count(*) FROM main GROUPBY region, dept + SELECT region, dept, count(*) FROM main GROUP BY region, dept >>> cursor.execute(query) >>> query_fields = [desc[0] for desc in cursor.description] diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 9f30141c10aa4ce..605de1570b2b961 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1912,8 +1912,8 @@ C APIs pending removal are * :meth:`!importlib.machinery.PathFinder.find_module` * :meth:`!importlib.machinery.WindowsRegistryFinder.find_module` * :func:`!importlib.util.module_for_loader` -* :func:`!importlib.util.set_loader_wrapper` -* :func:`!importlib.util.set_package_wrapper` +* :func:`!importlib.util.set_loader` +* :func:`!importlib.util.set_package` * :class:`!pkgutil.ImpImporter` * :class:`!pkgutil.ImpLoader` * :meth:`!pathlib.Path.link_to` diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index d822d83164db6ff..ddaf07384a6b8c4 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -133,7 +133,7 @@ Security improvements: C API improvements: -* :ref:`PEP 697 `, unstable C API tier +* :ref:`PEP 689 `, unstable C API tier * :ref:`PEP 683 `, immortal objects CPython implementation improvements: @@ -1035,13 +1035,13 @@ unittest Add a ``--durations`` command line option, showing the N slowest test cases:: - python3 -m unittest --durations=3 lib.tests.test_threading + python3 -m unittest --durations=3 test.test_threading ..... Slowest test durations ---------------------------------------------------------------------- - 1.210s test_timeout (Lib.test.test_threading.BarrierTests) - 1.003s test_default_timeout (Lib.test.test_threading.BarrierTests) - 0.518s test_timeout (Lib.test.test_threading.EventTests) + 1.210s test_timeout (test.test_threading.BarrierTests) + 1.003s test_default_timeout (test.test_threading.BarrierTests) + 0.518s test_timeout (test.test_threading.EventTests) (0.000 durations hidden. Use -v to show these durations.) ---------------------------------------------------------------------- @@ -1862,8 +1862,9 @@ New Features ------------ .. _whatsnew312-pep697: +.. _whatsnew312-pep689: -* :pep:`697`: Introduce the :ref:`Unstable C API tier `, +* :pep:`689`: Introduce the :ref:`Unstable C API tier `, intended for low-level tools like debuggers and JIT compilers. This API may change in each minor release of CPython without deprecation warnings. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 87d14720fb78d64..19fc5801cbe8a91 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1555,7 +1555,7 @@ ssl key agreement, extending the previous :meth:`ssl.SSLContext.set_ecdh_curve` method. This new API provides the ability to list multiple groups and - supports fixed-field and post-quantum groups in addition to ECDH + supports finite-field and post-quantum groups in addition to ECDH curves. This method can also be used to control what key shares are sent in the TLS handshake. * :meth:`ssl.SSLSocket.group` returns the group selected for doing key diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 273cbe7ff925a0b..32d1bbe6af7f83f 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2281,7 +2281,7 @@ Changes in the Python API * :meth:`!importlib.util.module_for_loader` now sets ``__loader__`` and ``__package__`` unconditionally to properly support reloading. If this is not desired then you will need to set these attributes manually. You can use - :func:`!importlib.util.module_to_load` for module management. + :func:`!importlib.util._module_to_load` for module management. * Import now resets relevant attributes (e.g. ``__name__``, ``__loader__``, ``__package__``, ``__file__``, ``__cached__``) unconditionally when reloading. @@ -2298,7 +2298,7 @@ Changes in the Python API * Frozen modules no longer define a ``__file__`` attribute. It's semantically incorrect for frozen modules to set the attribute as they are not loaded from any explicit location. If you must know that a module comes from frozen code - then you can see if the module's ``__spec__.location`` is set to ``'frozen'``, + then you can see if the module's ``__spec__.origin`` is set to ``'frozen'``, check if the loader is a subclass of :class:`importlib.machinery.FrozenImporter`, or if Python 2 compatibility is necessary you can use :func:`!imp.is_frozen`. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b0382dd8215bc00..d2daf052e43f824 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2111,11 +2111,11 @@ API and Feature Removals name for a given path. (Contributed by Yury Selivanov in :issue:`13248`.) -* ``traceback.Ignore`` class and ``traceback.usage``, ``traceback.modname``, - ``traceback.fullmodname``, ``traceback.find_lines_from_code``, - ``traceback.find_lines``, ``traceback.find_strings``, - ``traceback.find_executable_lines`` methods were removed from the - :mod:`traceback` module. They were undocumented methods deprecated since +* ``trace.Ignore`` class and ``trace.usage``, ``trace.modname``, + ``trace.fullmodname``, ``trace.find_lines_from_code``, + ``trace.find_lines``, ``trace.find_strings``, + ``trace.find_executable_linenos`` methods were removed from the + :mod:`trace` module. They were undocumented methods deprecated since Python 3.2 and equivalent functionality is available from private methods. * The ``tk_menuBar()`` and ``tk_bindForTraversal()`` dummy methods in diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 6900c97e9546c9a..671b83f87d34304 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -248,7 +248,7 @@ locales are available on the running system, it has the downside of having no effect on extension modules (such as GNU ``readline``), child processes running non-Python applications, and child processes running older versions of Python. To reduce the risk of corrupting text data when communicating with such -components, Python 3.7 also implements :ref:`whatsnew37-pep540`). +components, Python 3.7 also implements :ref:`whatsnew37-pep538`. The UTF-8 mode is enabled by default when the locale is ``C`` or ``POSIX``, and the :pep:`538` locale coercion feature fails to change it to a UTF-8 based @@ -1085,7 +1085,7 @@ itertools :func:`itertools.islice` now accepts :meth:`integer-like objects ` as start, stop, -and slice arguments. +and step arguments. (Contributed by Will Roberts in :issue:`30537`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 5f3fad1a1a51b50..37dfde666873878 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -2082,7 +2082,7 @@ Changes in the C API return NULL; #if PY_VERSION_HEX < 0x03080000 // Workaround for Python issue 35810; no longer necessary in Python 3.8 - PY_INCREF(type) + Py_INCREF(type); #endif return foo; } diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 49a52b7504bc951..e2731b6de15a3e0 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1031,8 +1031,8 @@ Removed :exc:`DeprecationWarning` since Python 3.8. (Contributed by Inada Naoki in :issue:`39377`) -* ``with (await asyncio.lock):`` and ``with (yield from asyncio.lock):`` statements are - not longer supported, use ``async with lock`` instead. The same is correct for +* ``with (await lock):`` and ``with (yield from lock):`` statements are + no longer supported, use ``async with lock`` instead. The same is correct for ``asyncio.Condition`` and ``asyncio.Semaphore``. (Contributed by Andrew Svetlov in :issue:`34793`.) From 6026791c977a1df2ff8a427aad294388992ae078 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sat, 7 Nov 2026 14:04:46 +0300 Subject: [PATCH 020/229] gh-161388: [tkinter] Fix crash on raising `__str__` method passed to `setvar` (#161397) --- Lib/test/test_tcl.py | 23 +++++++++++++++++++++++ Modules/_tkinter.c | 8 ++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 31ef1457ef3944e..9f749b4ec9c3459 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -601,6 +601,29 @@ def test_returning_deeply_nested(self): # event handle on Windows when test_tcl is run with -R 3:3 option. _tkinter._finalize_thread() + def test_setvar_evil(self): + # gh-161388 + class StrRaises: + def __str__(self): + raise RuntimeError("setvar evil") + + value = StrRaises() + tk = self.interp.tk + + with self.assertRaisesRegex(RuntimeError, "setvar evil"): + tk.setvar("local_name1", value) + self.assertRaises(TclError, tk.getvar, "local_name1") + with self.assertRaisesRegex(RuntimeError, "setvar evil"): + tk.setvar("local_name2", "local_name3", value) + self.assertRaises(TclError, tk.getvar, "local_name2", "local_name3") + + with self.assertRaisesRegex(RuntimeError, "setvar evil"): + tk.globalsetvar("global_name1", value) + self.assertRaises(TclError, tk.globalgetvar, "global_name1") + with self.assertRaisesRegex(RuntimeError, "setvar evil"): + tk.globalsetvar("global_name2", "global_name3", value) + self.assertRaises(TclError, tk.globalgetvar, "global_name2", "global_name3") + def test_user_command(self): result = None def testfunc(arg): diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 9540839b9682bca..d0a03953a80d92c 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -1234,8 +1234,9 @@ AsObjNested(PyObject *value, int depth) { PyObject *v = PyObject_Str(value); - if (!v) - return 0; + if (!v) { + return NULL; + } result = AsObjNested(v, depth); Py_DECREF(v); return result; @@ -2101,6 +2102,9 @@ SetVar(TkappObject *self, PyObject *args, int flags) /* XXX must hold tcl lock already??? */ newval = AsObj(newValue); + if (newval == NULL) { + return NULL; + } if (self->trace) { if (flags & TCL_GLOBAL_ONLY) { TRACE(self, ("((sssNO))", "uplevel", "#0", "set", From 91ba786ed777cfcd10c7e8c346ec8fe1f8278ae3 Mon Sep 17 00:00:00 2001 From: Stian Halseth <33956011+shalseth@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:30:26 +0100 Subject: [PATCH 021/229] gh-161475: Fix ctypes structures with float arrays returned on Linux SPARC (#161476) On Linux, GCC and clang pass arrays of floating-point type in a structure in integer registers; only Solaris follows the vendor ABI there (GCC PR target/114416). Since gh-110190, ctypes described such array elements to libffi as floats on all SPARC, so libffi read structures returned by value from the FP registers on Linux. --- .../2026-11-06-11-59-26.gh-issue-161475.82LWf4.rst | 2 ++ Modules/_ctypes/stgdict.c | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-06-11-59-26.gh-issue-161475.82LWf4.rst diff --git a/Misc/NEWS.d/next/Library/2026-11-06-11-59-26.gh-issue-161475.82LWf4.rst b/Misc/NEWS.d/next/Library/2026-11-06-11-59-26.gh-issue-161475.82LWf4.rst new file mode 100644 index 000000000000000..3033bc7543b9016 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-06-11-59-26.gh-issue-161475.82LWf4.rst @@ -0,0 +1,2 @@ +Fix :mod:`ctypes` structures containing arrays of floating-point type +returned by value on 64-bit Linux SPARC. diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index fc47844104876ea..7920c87ebe3cef3 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -683,10 +683,22 @@ _replace_array_elements(ctypes_state *st, PyObject *layout_fields, structs[struct_index].type = FFI_TYPE_STRUCT; structs[struct_index].elements = &dummy_types[dummy_index]; ++struct_index; + ffi_type *etype = &einfo->ffi_type_pointer; +#if defined(__sparc__) && defined(__arch64__) && !defined(__sun) + /* On Linux, GCC and clang pass arrays of floating-point type + in a structure in integer registers; only Solaris follows + the vendor ABI here (GCC PR target/114416). */ + if (etype->type == FFI_TYPE_FLOAT) { + etype = &ffi_type_uint32; + } + else if (etype->type == FFI_TYPE_DOUBLE) { + etype = &ffi_type_uint64; + } +#endif /* Copy over the element's type, length times. */ while (length > 0) { assert(dummy_index < (num_ffi_type_pointers)); - dummy_types[dummy_index++] = &einfo->ffi_type_pointer; + dummy_types[dummy_index++] = etype; length--; } assert(dummy_index < (num_ffi_type_pointers)); From 3de6d3a100d75929864ef235cce81e8b8e80b07a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 7 Nov 2026 15:50:40 +0100 Subject: [PATCH 022/229] gh-161430: Omit tests expected to be skipped in regrtest (#161432) No longer list tests expected to be skipped in the final report. Strip also "test." prefix in TestResults to make the output shorter. --- Lib/test/libregrtest/results.py | 22 +- Lib/test/libregrtest/skipped.py | 329 ++++++++++++++++++ Lib/test/support/__init__.py | 7 + Lib/test/test_perf_profiler.py | 9 +- Lib/test/test_perfmaps.py | 10 +- Lib/test/test_samply_profiler.py | 10 +- ...-11-05-16-20-23.gh-issue-161430.M7b8N6.rst | 2 + 7 files changed, 363 insertions(+), 26 deletions(-) create mode 100644 Lib/test/libregrtest/skipped.py create mode 100644 Misc/NEWS.d/next/Tests/2026-11-05-16-20-23.gh-issue-161430.M7b8N6.rst diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 3475f645729182c..4096bfc7191b99b 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -5,6 +5,7 @@ from .runtests import RunTests from .result import State, TestResult, TestStats, Location +from .skipped import expected_to_be_skipped from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, printlist, count, format_duration) @@ -98,6 +99,8 @@ def get_exitcode(self, fail_env_changed: bool, fail_rerun: bool) -> int: def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: test_name = result.test_name + # Remove "test." prefix added by some test packages + test_name = test_name.removeprefix('test.') rerun = runtests.rerun fail_env_changed = runtests.fail_env_changed @@ -204,6 +207,16 @@ def write_junit(self, filename: StrPath) -> None: for s in ET.tostringlist(root): f.write(s) + def split_expected_skip(self, tests): + skipped = [] + expected = [] + for name in tests: + if expected_to_be_skipped(name): + expected.append(name) + else: + skipped.append(name) + return (skipped, expected) + def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None: ansi = get_colors() green = ansi.GREEN @@ -221,13 +234,20 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> all_tests = [] omitted = set(tests) - self.get_executed() + # Filter tests expected to be skipped + skipped, expected_skip = self.split_expected_skip(self.skipped) + if expected_skip: + print() + count_text = count(len(expected_skip), "test") + print(f"{count_text} expected to be skipped (not listed)") + # less important all_tests.append( (sorted(omitted), "test", f"{yellow}{{}} omitted:{reset}") ) if not quiet: all_tests.append( - (self.skipped, "test", f"{yellow}{{}} skipped:{reset}") + (skipped, "test", f"{yellow}{{}} skipped:{reset}") ) all_tests.append( ( diff --git a/Lib/test/libregrtest/skipped.py b/Lib/test/libregrtest/skipped.py new file mode 100644 index 000000000000000..1116929a502f84f --- /dev/null +++ b/Lib/test/libregrtest/skipped.py @@ -0,0 +1,329 @@ +""" +Ruleset to check if a test is expected to be skipped. +Tests are mostly filtered using sys.platform, except a few exceptions. +""" + +import os +import sys +import sysconfig +from test import support + + +# On a WASI CI, the main regrtest process is run on the build Python and test +# worker processes are run on the host Python. Detect WASI using +# _PYTHON_HOST_PLATFORM environment variable. Below, IS_WASI should be checked +# explicitly since test.support constants (such as support.has_socket_support) +# are set by the build Python which is different. +HOST_PLATFORM = os.environ.get('_PYTHON_HOST_PLATFORM', '') +IS_WASI = support.is_wasi or HOST_PLATFORM.startswith('wasi') + + +NEED_MULTIPROCESSING = { + 'test_multiprocessing_fork', + 'test_multiprocessing_forkserver', + 'test_multiprocessing_main_handling', + 'test_multiprocessing_spawn', +} +SELECT_TESTS = { + 'test_devpoll', + 'test_epoll', + 'test_kqueue', + 'test_poll', +} +NEED_TESTMULTIPHASE = { + # Lib/test/test_importlib/util.py requires _testmultiphase + 'test_compileall', + 'test_import', + 'test_pkgutil', + 'test_pyclbr', + 'test_runpy', +} +NEED_WORKING_THREADING = { + 'test_concurrent_futures', + 'test_queue', + 'test_thread', + 'test_threadedtempfile', + 'test_threading', + 'test_threading_local', + 'test_webbrowser', +} +NEED_TKINTER = { + 'test_idle', + 'test_tcl', + 'test_tkinter', + 'test_ttk', + 'test_ttk_textonly', + 'test_turtle', +} + + +# List of tests expected to be skipped on sys.platform +EXPECTED_SKIP = set() +if support.Py_GIL_DISABLED: + # Need GIL build + EXPECTED_SKIP.add('test_interpreters') +else: + # Need Free Threading build + EXPECTED_SKIP.add('test_free_threading') + +if not support.has_socket_support or IS_WASI: + # Need socket support + EXPECTED_SKIP |= { + 'test_asyncgen', + 'test_asyncio', + 'test_docxmlrpc', + 'test_ftplib', + 'test_httplib', + 'test_httpservers', + 'test_imaplib', + 'test_mailbox', + 'test_poplib', + 'test_select', + 'test_selectors', + 'test_smtplib', + 'test_socket', + 'test_urllib', + 'test_urllib2', + 'test_urllib2_localnet', + 'test_urllib_response', + 'test_wsgiref', + 'test_xml_dom_xmlbuilder', + 'test_xmlrpc', + } + EXPECTED_SKIP |= SELECT_TESTS +if not support.has_subprocess_support or IS_WASI: + # Need subprocess support + EXPECTED_SKIP |= { + 'test_argparse', + 'test_asyncio.test_subprocess', + 'test_c_stack_unwind', + 'test_cext', + 'test_clinic', + 'test_cmd_line', + 'test_embed', + 'test_faulthandler', + 'test_file_eintr', + 'test_gdb', + 'test_generated_cases', + 'test_getopt', + 'test_optparse', + 'test_perf_profiler', + 'test_regrtest', + 'test_repl', + 'test_samply_profiler', + 'test_subprocess', + 'test_tools', + } +if not support.has_fork_support or IS_WASI: + # Need fork + EXPECTED_SKIP |= { + 'test_fork1', + 'test_multiprocessing_forkserver', + 'test_wait3', + 'test_wait4', + } +if not(sys.platform in ("linux", "darwin") or support.MS_WINDOWS): + # Need Py_SUPPORTS_REMOTE_DEBUG macro + EXPECTED_SKIP.add('test_remote_pdb') +if not sysconfig.is_python_build(): + # Skip tests on installed Python. + # Tests only relevant for from-source Python builds + EXPECTED_SKIP |= { + # Parser/asdl.py is not installed + 'test_asdl_parser', + # Tools/clinic/ is not installed + 'test_clinic', + # python-gdb.py is not installed + 'test_gdb', + # Tools/cases_generator is not installed + 'test_generated_cases', + } +if not sysconfig.get_config_var('WITH_DTRACE'): + # Need ./configure --with-dtrace + EXPECTED_SKIP.add('test_dtrace') + + +if support.MS_WINDOWS: + # Tests specific to non-Windows platforms + EXPECTED_SKIP |= { + 'test_asyncio.test_unix_events', # UNIX only + 'test_dbm_gnu', # Need dbm.gnu module + 'test_dbm_ndbm', # Need dbm.ndbm module + 'test_fcntl', # Need fcntl module + 'test_free_threading.test_dbm_gnu', # Need dbm.gnu module + 'test_free_threading.test_grp', # Need grp module + 'test_free_threading.test_pwd', # Need pwd module + 'test_free_threading.test_readline', # Need readline module + 'test_free_threading.test_resource', # Need resource module + 'test_free_threading.test_syslog', # Need syslog module + 'test_gdb', # test_gdb doesn't work on Windows + 'test_grp', # Need grp module + 'test_ioctl', # Need fcntl module + 'test_openpty', # Need os.openpty() + 'test_pty', # Need termios module + 'test_pwd', # Need pwd module + 'test_readline', # Need readline module + 'test_resource', # Need resource module + 'test_syslog', # Need syslog module + 'test_termios', # Need termios module + 'test_threadsignals', # Can't test signal on Windows + 'test_tty', # Need termios and tty modules + 'test_wait3', # Need os.wait3() + 'test_wait4', # Need os.wait4() + } +else: + EXPECTED_SKIP |= { + # Tests specific to Windows + 'test_asyncio.test_windows_events', + 'test_asyncio.test_windows_utils', + 'test_msvcrt', + 'test_os.test_windows', + 'test_startfile', # Need os.startfile() + 'test_winapi', + 'test_winconsoleio', + 'test_winreg', + 'test_winsound', + 'test_wmi', + } +if sys.platform in ("win32", 'darwin'): + EXPECTED_SKIP.add('test_multiprocessing_fork') +if sys.platform in ("win32", "cygwin"): + EXPECTED_SKIP.add('test_multiprocessing_forkserver') + +if not support.is_android: + # Test specific to Android + EXPECTED_SKIP.add('test_android') + + +if not support.is_apple: + # Test specific to Apple + EXPECTED_SKIP.add('test_apple') + + +if sys.platform == "darwin": + # On macOS, running gdb fails badly: + # https://github.com/python/cpython/pull/161520 + EXPECTED_SKIP.add('test_gdb') + + +if support.is_apple_mobile: + # configure disables multiple extension modules on Apple mobile + EXPECTED_SKIP |= { + 'test_curses', # need _curses + 'test_dbm_gnu', # need _gdbm + 'test_grp', # need grp + 'test_pwd', # need pwd + 'test_readline', # need readline + 'test_syslog', # need syslog + } + # configure disables _multiprocessing on Apple mobile + EXPECTED_SKIP |= NEED_MULTIPROCESSING + # configure disables _tkinter on Apple mobile + EXPECTED_SKIP |= NEED_TKINTER + + +if support.is_emscripten: + # configure disables multiple extension modules on Emscripten + EXPECTED_SKIP |= { + 'test__interpchannels', # need _interpchannels + 'test__interpreters', # need _interpreters + 'test_crossinterp', # need _interpreters + 'test_curses', # need _curses + 'test_dbm_gnu', # need _gdbm + 'test_dbm_ndbm', # need _dbm + 'test_grp', # need grp + 'test_pwd', # need pwd + 'test_readline', # need readline + 'test_resource', # need resource + 'test_syslog', # need syslog + } + # configure disables _multiprocessing on Emscripten + EXPECTED_SKIP |= NEED_MULTIPROCESSING + + # Missing os.openpty() function on Emscripten + EXPECTED_SKIP.add('test_openpty') + + EXPECTED_SKIP |= { + # Incompatible with Emscripten's mmap emulation + 'test_mmap', + # Cannot create socketpair on Emscripten/WASI + 'test_selectors', + } + + +if support.is_android or support.is_apple_mobile or support.is_wasm32 or IS_WASI: + EXPECTED_SKIP |= { + # pty is not available on these platforms + 'test_pty', + # venv is not available on these platforms + 'test_venv', + } + + +if IS_WASI: + # configure disables multiple extension modules on WASI + EXPECTED_SKIP |= { + 'test__interpchannels', # need _interpchannels + 'test__interpreters', # need _interpreters + 'test_crossinterp', # need _interpreters + 'test_curses', # need _curses + 'test_dbm_gnu', # need _gdbm + 'test_dbm_ndbm', # need _dbm + 'test_external_inspection', # need _remote_debugging + 'test_fcntl', # need fcntl + 'test_gc_stats', # need _remote_debugging + 'test_grp', # need grp + 'test_ioctl', # need fcntl + 'test_mmap', # need mmap + 'test_pwd', # need pwd + 'test_pyrepl', # need termios + 'test_remote_pdb', # need _remote_debugging + 'test_resource', # need resource + 'test_syslog', # need syslog + 'test_termios', # need termios + 'test_tty', # need termios + 'test_xxlimited', # need xxlimited + } + # configure disables _multiprocessing and _testmultiphase + # extension modules on WASI + EXPECTED_SKIP |= NEED_MULTIPROCESSING + EXPECTED_SKIP |= NEED_TESTMULTIPHASE + + # Missing os.openpty() function on WASI + EXPECTED_SKIP.add('test_openpty') + + +def expected_to_be_skipped(test_name): + # Test EXPECTED_SKIP + test_name = test_name.removeprefix('test.') + if test_name in EXPECTED_SKIP: + return True + package = test_name.split('.')[0] + if package in EXPECTED_SKIP: + return True + + # Custom ruleset + if (test_name in NEED_WORKING_THREADING + or package in NEED_WORKING_THREADING): + # Detect whether Python can start new threads + # - wasm32-wasi does not have threading. + # - wasm32-emscripten can be compiled with or without pthread + # support (-s USE_PTHREADS / __EMSCRIPTEN_PTHREADS__). + if IS_WASI: + return True + from test.support import threading_helper + return not threading_helper.can_start_thread + + if test_name in SELECT_TESTS: + # For example, test_epoll needs select.epoll() + import select + name = test_name.removeprefix('test_') + return not hasattr(select, name) + + if test_name in ('test_perf_profiler', 'test_perfmaps', 'test_samply_profiler'): + # Need perf trampoline profiling: configure defines the + # PY_HAVE_PERF_TRAMPOLINE macro on Linux (x86-64 and AArch64) + # and on macOS. + return not support.supports_perf_trampoline_profiling() + + return False diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index c14c50dae5bedcb..df3ae0890c0f126 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3614,3 +3614,10 @@ def inject_memory_error_cm(start=0, stop=0): yield finally: _testcapi.remove_mem_hooks() + + +def supports_perf_trampoline_profiling(): + perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") + if not perf_trampoline: + return False + return int(perf_trampoline) == 1 diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 0ebc0792cfc8222..8a2c4889d9471c4 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -23,14 +23,7 @@ raise unittest.SkipTest("test crash randomly on ASAN/MSAN build") -def supports_trampoline_profiling(): - perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") - if not perf_trampoline: - return False - return int(perf_trampoline) == 1 - - -if not supports_trampoline_profiling(): +if not support.supports_perf_trampoline_profiling(): raise unittest.SkipTest("perf trampoline profiling not supported") diff --git a/Lib/test/test_perfmaps.py b/Lib/test/test_perfmaps.py index ee4eb50033c4704..e6e5985233d1a8f 100644 --- a/Lib/test/test_perfmaps.py +++ b/Lib/test/test_perfmaps.py @@ -1,20 +1,14 @@ import os import sys -import sysconfig import unittest +from test import support try: from _testinternalcapi import perf_map_state_teardown, write_perf_map_entry except ImportError: raise unittest.SkipTest("requires _testinternalcapi") -def supports_trampoline_profiling(): - perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") - if not perf_trampoline: - return False - return int(perf_trampoline) == 1 - -if not supports_trampoline_profiling(): +if not support.supports_perf_trampoline_profiling(): raise unittest.SkipTest("perf trampoline profiling not supported") class TestPerfMapWriting(unittest.TestCase): diff --git a/Lib/test/test_samply_profiler.py b/Lib/test/test_samply_profiler.py index b9ac6586f4f7f55..ae3f047b15194a8 100644 --- a/Lib/test/test_samply_profiler.py +++ b/Lib/test/test_samply_profiler.py @@ -1,7 +1,6 @@ import unittest import subprocess import sys -import sysconfig import os import pathlib from test import support @@ -20,14 +19,7 @@ raise unittest.SkipTest("test crash randomly on ASAN/MSAN build") -def supports_trampoline_profiling(): - perf_trampoline = sysconfig.get_config_var("PY_HAVE_PERF_TRAMPOLINE") - if not perf_trampoline: - return False - return int(perf_trampoline) == 1 - - -if not supports_trampoline_profiling(): +if not support.supports_perf_trampoline_profiling(): raise unittest.SkipTest("perf trampoline profiling not supported") diff --git a/Misc/NEWS.d/next/Tests/2026-11-05-16-20-23.gh-issue-161430.M7b8N6.rst b/Misc/NEWS.d/next/Tests/2026-11-05-16-20-23.gh-issue-161430.M7b8N6.rst new file mode 100644 index 000000000000000..b53d2f01d5ce141 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-11-05-16-20-23.gh-issue-161430.M7b8N6.rst @@ -0,0 +1,2 @@ +No longer list tests expected to be skipped in the final report. Patch by +Victor Stinner. From bd83de61433a8f6e5ae49be6ec54dc9f7d9e6aaf Mon Sep 17 00:00:00 2001 From: Ivan Zorin Date: Sat, 7 Nov 2026 18:07:27 +0300 Subject: [PATCH 023/229] gh-161433: Identify the failing item when hashing tuples (#161496) --- Lib/test/test_tuple.py | 39 +++++++++++++++++++ Misc/ACKS | 1 + ...-11-06-15-15-00.gh-issue-161433.A9b3Kq.rst | 2 + Objects/tupleobject.c | 18 +++++++++ 4 files changed, 60 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-15-15-00.gh-issue-161433.A9b3Kq.rst diff --git a/Lib/test/test_tuple.py b/Lib/test/test_tuple.py index 5650dec09adef43..a3bd60eca4f6b15 100644 --- a/Lib/test/test_tuple.py +++ b/Lib/test/test_tuple.py @@ -450,6 +450,45 @@ def test_lexicographic_ordering(self): self.assertLess(a, b) self.assertLess(b, c) + def test_hash_unhashable_note(self): + with self.assertRaisesRegex(TypeError, "unhashable type: 'list'") as cm: + hash(("alice", [])) + self.assertEqual(cm.exception.__notes__, + ["when hashing tuple item 1"]) + + with self.assertRaisesRegex(TypeError, "unhashable type: 'list'") as cm: + hash((("alice", []), 42)) + self.assertEqual(cm.exception.__notes__, + ["when hashing tuple item 1", + "when hashing tuple item 0"]) + + class BrokenHash: + def __hash__(self): + raise TypeError("custom failure") + + with self.assertRaisesRegex(TypeError, "custom failure") as cm: + hash((BrokenHash(),)) + self.assertEqual(cm.exception.__notes__, ["when hashing tuple item 0"]) + + class BrokenHashNote: + def __hash__(self): + exc = TypeError("custom failure") + exc.add_note("from custom hash") + raise exc + + with self.assertRaisesRegex(TypeError, "custom failure") as cm: + hash((BrokenHashNote(),)) + self.assertEqual(cm.exception.__notes__, + ["from custom hash", "when hashing tuple item 0"]) + + class BrokenHashValueError: + def __hash__(self): + raise ValueError("custom failure") + + with self.assertRaisesRegex(ValueError, "custom failure") as cm: + hash((BrokenHashValueError(),)) + self.assertFalse(hasattr(cm.exception, "__notes__")) + # Notes on testing hash codes. The primary thing is that Python doesn't # care about "random" hash codes. To the contrary, we like them to be # very regular when possible, so that the low-order bits are as evenly diff --git a/Misc/ACKS b/Misc/ACKS index 65f6f423eb7c428..4178ae3108128aa 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -2048,6 +2048,7 @@ Sjoerd de Vries Jonas Wagner Daniel Wagner-Hall Niki W. Waibel +WakaTaira Wojtek Walczak Charles Waldman Richard Walker diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-15-15-00.gh-issue-161433.A9b3Kq.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-15-15-00.gh-issue-161433.A9b3Kq.rst new file mode 100644 index 000000000000000..251837ac6d632a4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-15-15-00.gh-issue-161433.A9b3Kq.rst @@ -0,0 +1,2 @@ +Add context notes to :exc:`TypeError` raised while hashing a +:class:`tuple`, identifying the index of the item whose hash failed. diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 68ca2772507295e..992699756ac770f 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -8,6 +8,7 @@ #include "pycore_list.h" // _Py_memory_repeat() #include "pycore_modsupport.h" // _PyArg_NoKwnames() #include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_pyerrors.h" // _PyException_AddNote() #include "pycore_stackref.h" // PyStackRef_AsPyObjectSteal() #include "pycore_tuple.h" // _PyTupleIterObject @@ -392,6 +393,23 @@ tuple_hash(PyObject *op) for (Py_ssize_t i = 0; i < len; i++) { Py_uhash_t lane = PyObject_Hash(item[i]); if (lane == (Py_uhash_t)-1) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + /* Preserve the original error while providing context for + the element that could not be hashed. */ + PyObject *exc = PyErr_GetRaisedException(); + PyObject *note = PyUnicode_FromFormat( + "when hashing tuple item %zd", i); + if (note != NULL) { + if (_PyException_AddNote(exc, note) < 0) { + PyErr_Clear(); + } + Py_DECREF(note); + } + else { + PyErr_Clear(); + } + PyErr_SetRaisedException(exc); + } _Py_LeaveRecursiveCall(); return -1; } From d7684fbef52e563a89c88a23efc68de2de9b3503 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 7 Nov 2026 18:27:01 +0100 Subject: [PATCH 024/229] gh-161430: Cleanup tests name in regrtest TestResults.display_result() (#161567) Add utils.cleanup_test_name() function. --- Lib/test/libregrtest/results.py | 8 ++++---- Lib/test/libregrtest/skipped.py | 3 ++- Lib/test/libregrtest/utils.py | 4 ++++ Lib/test/test_regrtest.py | 15 +++++++++++++++ 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 4096bfc7191b99b..eac8f4c40ebca17 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -8,7 +8,7 @@ from .skipped import expected_to_be_skipped from .utils import ( StrPath, TestName, TestTuple, TestList, FilterDict, - printlist, count, format_duration) + printlist, count, format_duration, cleanup_test_name) # Python uses exit code 1 when an exception is not caught @@ -98,9 +98,7 @@ def get_exitcode(self, fail_env_changed: bool, fail_rerun: bool) -> int: return exitcode def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: - test_name = result.test_name - # Remove "test." prefix added by some test packages - test_name = test_name.removeprefix('test.') + test_name = cleanup_test_name(result.test_name) rerun = runtests.rerun fail_env_changed = runtests.fail_env_changed @@ -218,6 +216,8 @@ def split_expected_skip(self, tests): return (skipped, expected) def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None: + tests = tuple(cleanup_test_name(name) for name in tests) + ansi = get_colors() green = ansi.GREEN red = ansi.BOLD_RED diff --git a/Lib/test/libregrtest/skipped.py b/Lib/test/libregrtest/skipped.py index 1116929a502f84f..8553b46f10d37e1 100644 --- a/Lib/test/libregrtest/skipped.py +++ b/Lib/test/libregrtest/skipped.py @@ -7,6 +7,7 @@ import sys import sysconfig from test import support +from .utils import cleanup_test_name # On a WASI CI, the main regrtest process is run on the build Python and test @@ -295,7 +296,7 @@ def expected_to_be_skipped(test_name): # Test EXPECTED_SKIP - test_name = test_name.removeprefix('test.') + test_name = cleanup_test_name(test_name) if test_name in EXPECTED_SKIP: return True package = test_name.split('.')[0] diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index b34f106a83ca854..0b8f7f51395a643 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -67,6 +67,10 @@ FilterDict = dict[TestName, FilterTuple] +def cleanup_test_name(name: TestName) -> TestName: + return name.removeprefix('test.') + + def format_duration(seconds: float) -> str: ms = math.ceil(seconds * 1e3) seconds, ms = divmod(ms, 1000) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e8211526ae847c5..cd2cd8a21e8ea7f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -2981,6 +2981,21 @@ def test_sanitize_xml(self): self.assertEqual(sanitize_xml('valid t\xe9xt \u20ac'), 'valid t\xe9xt \u20ac') + def test_cleanup_test_name(self): + cleanup_test_name = utils.cleanup_test_name + + # Strip "test." prefix + self.assertEqual(cleanup_test_name('test.test_asyncio.test_graph'), + 'test_asyncio.test_graph') + self.assertEqual(cleanup_test_name('test.test_multiprocessing_spawn.test_manager'), + 'test_multiprocessing_spawn.test_manager') + + # No change + self.assertEqual(cleanup_test_name('test_cmath'), + 'test_cmath') + self.assertEqual(cleanup_test_name('test_asyncio.test_graph'), + 'test_asyncio.test_graph') + class GitHubActionsReporterTestCase(unittest.TestCase): def setUp(self): From 8dde46eaa6baf5f027a7234845b20d50b509624e Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Sat, 7 Nov 2026 14:07:53 -0500 Subject: [PATCH 025/229] gh-161503: Check hostname in `ssl.get_server_certificate` with custom `ca_certs` (#161507) --- Doc/library/ssl.rst | 8 +++++++- Lib/ssl.py | 4 +++- Lib/test/test_ssl.py | 8 ++++++-- .../2026-11-06-16-22-47.gh-issue-161503.FkIl87.rst | 4 ++++ 4 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-11-06-16-22-47.gh-issue-161503.FkIl87.rst diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index d177abd48fa9cd4..020a8ab7301b6b7 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -432,7 +432,8 @@ Certificate handling same format as used for the *cafile* parameter in :meth:`SSLContext.load_verify_locations`. The call will attempt to validate the server certificate against that set of root certificates, and will fail - if the validation attempt fails. A timeout can be specified with the + if the validation attempt fails or the certificate hostname does not match + the server hostname. A timeout can be specified with the ``timeout`` parameter. .. versionchanged:: 3.3 @@ -445,6 +446,11 @@ Certificate handling .. versionchanged:: 3.10 The *timeout* parameter was added. + .. versionchanged:: next + The function now raises :exc:`SSLCertVerificationError` if *ca_certs* is + specified but the certificate hostname does not match the server + hostname. + .. function:: DER_cert_to_PEM_cert(der_cert_bytes) Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded diff --git a/Lib/ssl.py b/Lib/ssl.py index df2eece13f2b138..49eec9cde0b6317 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1638,7 +1638,8 @@ def get_server_certificate(addr, ssl_version=PROTOCOL_TLS_CLIENT, ca_certs=None, timeout=_GLOBAL_DEFAULT_TIMEOUT): """Retrieve the certificate from the server at the specified address, and return it as a PEM-encoded string. - If 'ca_certs' is specified, validate the server cert against it. + If 'ca_certs' is specified, validate the server cert against it and + check that it matches the hostname. If 'ssl_version' is specified, use it in the connection attempt. If 'timeout' is specified, use it in the connection attempt. """ @@ -1650,6 +1651,7 @@ def get_server_certificate(addr, ssl_version=PROTOCOL_TLS_CLIENT, cert_reqs = CERT_NONE context = _create_stdlib_context(ssl_version, cert_reqs=cert_reqs, + check_hostname=ca_certs is not None, cafile=ca_certs) with create_connection(addr, timeout=timeout) as sock: with context.wrap_socket(sock, server_hostname=host) as sslsock: diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 5b4e4f080607726..b0e3cbaa548e8d5 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -3112,6 +3112,10 @@ def test_get_server_certificate_fail(self): # independent test method _test_get_server_certificate_fail(self, *self.server_addr) + def test_get_server_certificate_fail_by_hostname(self): + _, port = self.server_addr + _test_get_server_certificate_fail(self, socket_helper.HOSTv4, port, SIGNING_CA) + def test_get_server_certificate_timeout(self): def servername_cb(ssl_sock, server_name, initial_context): time.sleep(0.2) @@ -3364,10 +3368,10 @@ def _test_get_server_certificate(test, host, port, cert=None): if support.verbose: sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) -def _test_get_server_certificate_fail(test, host, port): +def _test_get_server_certificate_fail(test, host, port, ca_certs=CERTFILE): with warnings_helper.check_no_resource_warning(test): try: - pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + pem = ssl.get_server_certificate((host, port), ca_certs=ca_certs) except ssl.SSLError as x: #should fail if support.verbose: diff --git a/Misc/NEWS.d/next/Security/2026-11-06-16-22-47.gh-issue-161503.FkIl87.rst b/Misc/NEWS.d/next/Security/2026-11-06-16-22-47.gh-issue-161503.FkIl87.rst new file mode 100644 index 000000000000000..308ba0c83f3efe4 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-11-06-16-22-47.gh-issue-161503.FkIl87.rst @@ -0,0 +1,4 @@ +.. cve: CVE-2026-19552 + +:func:`ssl.get_server_certificate` now verifies that the server hostname matches +its certificate when the caller specifies *ca_certs*. From 63e7b1889beec3e67db0448ae055c5b68e7b64f3 Mon Sep 17 00:00:00 2001 From: emirplus Date: Sat, 7 Nov 2026 22:20:02 +0300 Subject: [PATCH 026/229] gh-160050: Use asymptotically faster divmod in `l_mod` for large integers (#160443) --- Lib/test/test_int.py | 9 +++++++++ .../2026-10-23-19-37-00.gh-issue-160050.aB3x9Q.rst | 1 + Objects/longobject.c | 8 ++++++++ 3 files changed, 18 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-37-00.gh-issue-160050.aB3x9Q.rst diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index fa675f626cbe109..40c636d586c550f 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -778,6 +778,14 @@ def test_pylong_int_divmod(self): a, b = divmod(n*3 + 1, n) assert a == 3 and b == 1 + def test_pylong_int_mod(self): + n = (1 << 100_000) + m = (1 << 50_000) + self.assertEqual((n * 3 + 1) % m, 1) + self.assertEqual((-n * 3 - 1) % m, m - 1) + self.assertEqual((n * 3 + 1) % -m, 1 - m) + self.assertEqual((-n * 3 - 1) % -m, -1) + @support.cpython_only # tests implementation details of CPython. @unittest.skipUnless(_pylong, "_pylong module required") def test_pylong_int_divmod_crash(self): @@ -789,6 +797,7 @@ def test_pylong_int_divmod_crash(self): with mock.patch.object(_pylong, "int_divmod", wraps=bad_int_divmod): msg = r"tuple of length 2 is required from int_divmod\(\)" self.assertRaisesRegex(ValueError, msg, divmod, a, b) + self.assertRaisesRegex(ValueError, msg, lambda: a % b) def test_pylong_str_to_int(self): v1 = 1 << 100_000 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-37-00.gh-issue-160050.aB3x9Q.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-37-00.gh-issue-160050.aB3x9Q.rst new file mode 100644 index 000000000000000..9089e4bc4dbb14b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-37-00.gh-issue-160050.aB3x9Q.rst @@ -0,0 +1 @@ +Use asymptotically faster :func:`divmod` implementation for integer modulo of large integers. diff --git a/Objects/longobject.c b/Objects/longobject.c index c96ed6a215b5a1d..9d02793fdd63665 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -4564,6 +4564,14 @@ l_mod(PyLongObject *v, PyLongObject *w, PyLongObject **pmod) *pmod = (PyLongObject *)fast_mod(v, w); return -(*pmod == NULL); } +#if WITH_PYLONG_MODULE + Py_ssize_t size_v = _PyLong_DigitCount(v); /* digits in numerator */ + Py_ssize_t size_w = _PyLong_DigitCount(w); /* digits in denominator */ + if (size_w > 300 && (size_v - size_w) > 150) { + /* Switch to _pylong.int_divmod(). See comments in l_divmod(). */ + return pylong_int_divmod(v, w, NULL, pmod); + } +#endif if (long_rem(v, w, &mod) < 0) return -1; if ((_PyLong_IsNegative(mod) && _PyLong_IsPositive(w)) || From 8eb3d2c1c41ca7ce628554413981c1eb763a1494 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 7 Nov 2026 21:53:13 +0200 Subject: [PATCH 027/229] gh-161569: Check cookie policy before deleting expired cookies in `http.cookiejar` (#161570) gh-161569: Check cookie policy before deleting expired cookies in http.cookiejar An expired Set-Cookie header was allowed to delete a stored cookie using the Domain and Path attributes taken verbatim from the header, before CookiePolicy.set_ok() was consulted. A response from one host could therefore delete cookies belonging to any other domain in a shared jar. The policy is now consulted before an expired cookie may delete an existing one, as required by RFC 6265 section 5.3. Co-authored-by: Victor Stinner --- Lib/http/cookiejar.py | 45 +++++++++-------- Lib/test/test_http_cookiejar.py | 49 +++++++++++++++++++ ...11-07-01-43-49.gh-issue-161569.cookiej.rst | 7 +++ 3 files changed, 81 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-11-07-01-43-49.gh-issue-161569.cookiej.rst diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 21dd6a9a6c2f523..d87e9a90ecfcfe2 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1568,28 +1568,33 @@ def _cookie_from_cookie_tuple(self, tup, request): if expires is Absent: expires = None discard = True - elif expires <= self._now: - # Expiry date in past is request to delete cookie. This can't be - # in DefaultCookiePolicy, because can't delete cookies there. - try: - self.clear(domain, path, name) - except KeyError: - pass - _debug("Expiring cookie, domain='%s', path='%s', name='%s'", - domain, path, name) + + cookie = Cookie(version, + name, value, + port, port_specified, + domain, domain_specified, domain_initial_dot, + path, path_specified, + secure, + expires, + discard, + comment, + comment_url, + rest) + + if expires is not None and expires <= self._now: + # RFC 6265 section 5.3 ignores the cookie entirely if its Domain + # attribute does not match the request host (step 6) before an + # existing cookie is removed (steps 11-12) + if self._policy.set_ok(cookie, request): + try: + self.clear(domain, path, name) + except KeyError: + pass + _debug("Expiring cookie, domain='%s', path='%s', name='%s'", + domain, path, name) return None - return Cookie(version, - name, value, - port, port_specified, - domain, domain_specified, domain_initial_dot, - path, path_specified, - secure, - expires, - discard, - comment, - comment_url, - rest) + return cookie def _cookies_from_attrs_set(self, attrs_set, request): cookie_tuples = self._normalized_cookie_tuples(attrs_set) diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index da6f468b2f37188..d53e8c6b614d347 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -743,6 +743,55 @@ def test_expires(self): # XXX RFC 2965 expiry rules (some apply to V0 too) + def test_expired_cookie_from_other_domain_is_ignored(self): + c = CookieJar() + interact_netscape(c, "http://www.victim.test/", + 'session=secret; Domain=.victim.test; Path=/') + + for domain in ('.victim.test', 'victim.test'): + for expires in ( + 'Max-Age=0', + 'Expires=Thu, 01 Jan 1970 00:00:00 GMT', + ): + hdr = f'session=x; Domain={domain}; Path=/; {expires}' + with self.subTest(hdr=hdr): + interact_netscape(c, "http://www.attacker.test/", hdr) + self.assertEqual(len(c), 1) + self.assertIn( + 'session=secret', + interact_netscape(c, "http://www.victim.test/")) + + interact_netscape(c, "http://www.victim.test/", + 'session=x; Domain=.victim.test; Path=/; Max-Age=0') + self.assertEqual(len(c), 0) + + def test_expired_cookie_from_other_path_is_ignored(self): + pol = DefaultCookiePolicy(strict_ns_set_path=True) + c = CookieJar(pol) + interact_netscape(c, "http://www.acme.com/a/", + 'token=secret; Path=/a') + self.assertEqual(len(c), 1) + interact_netscape(c, "http://www.acme.com/b/", + 'token=x; Path=/a; Max-Age=0') + self.assertEqual(len(c), 1) + interact_netscape(c, "http://www.acme.com/a/", + 'token=x; Path=/a; Max-Age=0') + self.assertEqual(len(c), 0) + + def test_expired_cookie_for_blocked_domain_is_ignored(self): + c = CookieJar() + interact_netscape(c, "http://www.victim.test/", + 'session=secret; Domain=.victim.test; Path=/') + self.assertEqual(len(c), 1) + c.set_policy(DefaultCookiePolicy(blocked_domains=[".victim.test"])) + interact_netscape(c, "http://www.victim.test/", + 'session=x; Domain=.victim.test; Path=/; Max-Age=0') + self.assertEqual(len(c), 1) + c.set_policy(DefaultCookiePolicy()) + interact_netscape(c, "http://www.victim.test/", + 'session=x; Domain=.victim.test; Path=/; Max-Age=0') + self.assertEqual(len(c), 0) + def test_default_path(self): # RFC 2965 pol = DefaultCookiePolicy(rfc2965=True) diff --git a/Misc/NEWS.d/next/Security/2026-11-07-01-43-49.gh-issue-161569.cookiej.rst b/Misc/NEWS.d/next/Security/2026-11-07-01-43-49.gh-issue-161569.cookiej.rst new file mode 100644 index 000000000000000..b16f6caf90f1b06 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-11-07-01-43-49.gh-issue-161569.cookiej.rst @@ -0,0 +1,7 @@ +.. cve: CVE-2026-124249 + +Fix :class:`http.cookiejar.CookieJar` deleting a stored cookie when a +response from an unrelated host sets an already expired cookie with a +``Domain`` or ``Path`` attribute pointing at that cookie. The cookie policy +is now consulted before an expired cookie is allowed to delete an existing +one. From 0e98aea32593e39c647c2b3d7054ec4e36bf643a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Sun, 8 Nov 2026 04:00:57 +0800 Subject: [PATCH 028/229] gh-161555: Fix `test_heapq` under global lazy imports (#161556) Co-authored-by: Stan Ulbrych --- Lib/test/lazy_imports_all_exclude.txt | 1 - Lib/test/test_heapq.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/lazy_imports_all_exclude.txt b/Lib/test/lazy_imports_all_exclude.txt index 45796d1db9f0a58..8d303858139b9f7 100644 --- a/Lib/test/lazy_imports_all_exclude.txt +++ b/Lib/test/lazy_imports_all_exclude.txt @@ -24,7 +24,6 @@ test_clinic test_crossinterp test_datetime test_generated_cases -test_heapq test_import test_importlib test_json diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py index d6623fee9bb2b43..0d288caeabc8edb 100644 --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -39,7 +39,7 @@ def load_tests(loader, tests, ignore): class HeapqMergeDocTestFinder: def find(self, *args, **kwargs): dtf = doctest.DocTestFinder() - return dtf.find(py_heapq.merge) + return dtf.find(py_heapq.merge, globs=py_heapq.__dict__) tests.addTests(doctest.DocTestSuite(py_heapq, test_finder=HeapqMergeDocTestFinder())) From 989ece60ab880f5f5c774a81b358db02950bbe1d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 7 Nov 2026 22:35:26 +0100 Subject: [PATCH 029/229] gh-161430: Update skipped tests filter in regrtest (#161572) Filter test_io.test_free_threading (need Free Threading), test_io.test_winconsoleio (need Windows), and test_io.test_file_eintr (need subprocess). --- Lib/test/libregrtest/skipped.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/skipped.py b/Lib/test/libregrtest/skipped.py index 8553b46f10d37e1..45ecbb94d66f8d8 100644 --- a/Lib/test/libregrtest/skipped.py +++ b/Lib/test/libregrtest/skipped.py @@ -65,7 +65,10 @@ EXPECTED_SKIP.add('test_interpreters') else: # Need Free Threading build - EXPECTED_SKIP.add('test_free_threading') + EXPECTED_SKIP |= { + 'test_free_threading', + 'test_io.test_free_threading', + } if not support.has_socket_support or IS_WASI: # Need socket support @@ -103,10 +106,10 @@ 'test_cmd_line', 'test_embed', 'test_faulthandler', - 'test_file_eintr', 'test_gdb', 'test_generated_cases', 'test_getopt', + 'test_io.test_file_eintr', 'test_optparse', 'test_perf_profiler', 'test_regrtest', @@ -179,9 +182,9 @@ 'test_asyncio.test_windows_utils', 'test_msvcrt', 'test_os.test_windows', + 'test_io.test_winconsoleio', 'test_startfile', # Need os.startfile() 'test_winapi', - 'test_winconsoleio', 'test_winreg', 'test_winsound', 'test_wmi', From 1798d4fb3ca0e1cd8f9e6ab0ac99a0a9fb1f7527 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Sat, 7 Nov 2026 22:44:24 +0100 Subject: [PATCH 030/229] gh-148891: prevent crash in HAMT caused by GC during traverse (#148928) `_PyHamt_Assoc` allocates HAMT nodes, which can trigger GC. A weakref callback or finaliser fired during GC could re-enter `contextvar_set`, replacing `ctx->ctx_vars` via `Py_SETREF` and freeing the old HAMT while `_PyHamt_Assoc` was still traversing it, causing a use-after-free. Hold a strong reference to `ctx->ctx_vars` during the `_PyHamt_Assoc` call, so that a re-entrant free cannot affect the HAMT being traversed. --- .../2026-04-23-08-36-47.gh-issue-148891.AMMHow.rst | 1 + Python/context.c | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-04-23-08-36-47.gh-issue-148891.AMMHow.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-23-08-36-47.gh-issue-148891.AMMHow.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-23-08-36-47.gh-issue-148891.AMMHow.rst new file mode 100644 index 000000000000000..bae5c1366ca6a2c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-23-08-36-47.gh-issue-148891.AMMHow.rst @@ -0,0 +1 @@ +Fix a crash in :func:`contextvars.Context.run` where a GC invocation could free the current HAMT during traverse. diff --git a/Python/context.c b/Python/context.c index 29c483d2dc90d68..e4e58df0f96c98f 100644 --- a/Python/context.c +++ b/Python/context.c @@ -866,8 +866,13 @@ contextvar_set(PyContextVar *var, PyObject *val) return -1; } + // gh-148891: Make a strong reference so a GC during _PyHamt_Assoc will + // not free the HAMT during traversal. + PyHamtObject *old_vars = context_get_current_vars(ctx); + Py_INCREF(old_vars); PyHamtObject *new_vars = _PyHamt_Assoc( - context_get_current_vars(ctx), (PyObject *)var, val); + old_vars, (PyObject *)var, val); + Py_DECREF(old_vars); if (new_vars == NULL) { return -1; } @@ -894,8 +899,12 @@ contextvar_del(PyContextVar *var) return -1; } + // gh-148891: Make a strong reference so a GC during _PyHamt_Without will + // not free the HAMT during traversal. PyHamtObject *vars = context_get_current_vars(ctx); + Py_INCREF(vars); PyHamtObject *new_vars = _PyHamt_Without(vars, (PyObject *)var); + Py_DECREF(vars); if (new_vars == NULL) { return -1; } From 9ea60a5529fe728e681a0498cefb58eaa163424a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 7 Nov 2026 13:59:00 -0800 Subject: [PATCH 031/229] gh-87613: Pass NULL for an empty **kwargs in Argument Clinic (#161382) The parser allocated an empty dict for **kwargs when there were no keyword arguments. Pass NULL instead, like the caller does, and handle NULL in the impls that used the dict unconditionally. All **kwargs impls were audited: _csv and zoneinfo already accept NULL, and the rest (structseq and SimpleNamespace `__replace__`, _interpreters.new_config, atexit.register, SubElement) now check for it. atexit stores None, which the run loop already maps back to NULL. pyperf, PGO+LTO, `t = time.gmtime(0)`, `ns = SimpleNamespace(a=1)`: | Benchmark | main | this change | |---------------------------------|:------:|:---------------------:| | struct_time.`__replace__`() | 229 ns | 47.0 ns: 4.87x faster | | SimpleNamespace.`__replace__`() | 72.2 ns | 65.5 ns: 1.10x faster | Co-authored-by: Claude Opus 5.5 --- Lib/test/test_clinic.py | 14 +++---- Modules/_elementtree.c | 4 +- Modules/_interpretersmodule.c | 2 +- Modules/atexitmodule.c | 3 +- Modules/clinic/_csv.c.h | 42 +++----------------- Modules/clinic/_elementtree.c.h | 12 +----- Modules/clinic/_interpretersmodule.c.h | 12 +----- Modules/clinic/_testclinic_kwds.c.h | 53 +++----------------------- Modules/clinic/_zoneinfo.c.h | 12 +----- Modules/clinic/atexitmodule.c.h | 12 +----- Objects/clinic/namespaceobject.c.h | 12 +----- Objects/clinic/structseq.c.h | 12 +----- Objects/namespaceobject.c | 4 +- Objects/structseq.c | 9 +++-- Tools/clinic/libclinic/converters.py | 14 +------ 15 files changed, 45 insertions(+), 172 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index e57f4c4a9d556af..4e0b0137e9faa0b 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -5708,7 +5708,7 @@ def errmsg(name): def test_lone_kwds(self): with self.assertRaises(TypeError): ac_tester.lone_kwds(1, 2) - self.assertEqual(ac_tester.lone_kwds(), ({},)) + self.assertEqual(ac_tester.lone_kwds(), (None,)) self.assertEqual(ac_tester.lone_kwds(y='y'), ({'y': 'y'},)) kwds = {'y': 'y', 'z': 'z'} self.assertEqual(ac_tester.lone_kwds(y='y', z='z'), (kwds,)) @@ -5721,7 +5721,7 @@ def test_kwds_with_pos_only(self): ac_tester.kwds_with_pos_only(y='y') with self.assertRaises(TypeError): ac_tester.kwds_with_pos_only(1, y='y') - self.assertEqual(ac_tester.kwds_with_pos_only(1, 2), (1, 2, {})) + self.assertEqual(ac_tester.kwds_with_pos_only(1, 2), (1, 2, None)) self.assertEqual(ac_tester.kwds_with_pos_only(1, 2, y='y'), (1, 2, {'y': 'y'})) kwds = {'y': 'y', 'z': 'z'} self.assertEqual(ac_tester.kwds_with_pos_only(1, 2, y='y', z='z'), (1, 2, kwds)) @@ -5732,16 +5732,16 @@ def test_kwds_with_optional_pos_only(self): ac_tester.kwds_with_optional_pos_only() with self.assertRaises(TypeError): ac_tester.kwds_with_optional_pos_only(y='y') - self.assertEqual(ac_tester.kwds_with_optional_pos_only(1), (1, None, {})) - self.assertEqual(ac_tester.kwds_with_optional_pos_only(1, 2), (1, 2, {})) + self.assertEqual(ac_tester.kwds_with_optional_pos_only(1), (1, None, None)) + self.assertEqual(ac_tester.kwds_with_optional_pos_only(1, 2), (1, 2, None)) self.assertEqual(ac_tester.kwds_with_optional_pos_only(1, y='y'), (1, None, {'y': 'y'})) self.assertEqual(ac_tester.kwds_with_optional_pos_only(1, 2, y='y'), (1, 2, {'y': 'y'})) def test_kwds_with_stararg(self): - self.assertEqual(ac_tester.kwds_with_stararg(), ((), {})) - self.assertEqual(ac_tester.kwds_with_stararg(1, 2), ((1, 2), {})) + self.assertEqual(ac_tester.kwds_with_stararg(), ((), None)) + self.assertEqual(ac_tester.kwds_with_stararg(1, 2), ((1, 2), None)) self.assertEqual(ac_tester.kwds_with_stararg(y='y'), ((), {'y': 'y'})) args = (1, 2) kwds = {'y': 'y', 'z': 'z'} @@ -5753,7 +5753,7 @@ def test_kwds_with_pos_only_and_stararg(self): ac_tester.kwds_with_pos_only_and_stararg() with self.assertRaises(TypeError): ac_tester.kwds_with_pos_only_and_stararg(y='y') - self.assertEqual(ac_tester.kwds_with_pos_only_and_stararg(1, 2), (1, 2, (), {})) + self.assertEqual(ac_tester.kwds_with_pos_only_and_stararg(1, 2), (1, 2, (), None)) self.assertEqual(ac_tester.kwds_with_pos_only_and_stararg(1, 2, y='y'), (1, 2, (), {'y': 'y'})) args = ('lobster', 'thermidor') kwds = {'y': 'y', 'z': 'z'} diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f8f661a73418912..983a00593b85d89 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -640,11 +640,11 @@ _elementtree_SubElement_impl(PyObject *module, PyObject *parent, attrib = PyDict_Copy(attrib); if (!attrib) return NULL; - if (PyDict_Update(attrib, extra) < 0) { + if (extra && PyDict_Update(attrib, extra) < 0) { Py_DECREF(attrib); return NULL; } - } else if (PyDict_GET_SIZE(extra)) { + } else if (extra && PyDict_GET_SIZE(extra)) { /* have keyword args */ attrib = get_attrib_from_keywords(extra); if (!attrib) diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 6630d5f6fe2d33c..454db063d18b6a0 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -828,7 +828,7 @@ _interpreters_new_config_impl(PyObject *module, const char *name, return NULL; } - if (PyDict_GET_SIZE(overrides) > 0) { + if (overrides != NULL && PyDict_GET_SIZE(overrides) > 0) { if (_PyInterpreterConfig_UpdateFromDict(&config, overrides) < 0) { return NULL; } diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index b835fe670938cbc..d572351e80f6586 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -191,7 +191,8 @@ atexit_register_impl(PyObject *module, PyObject *func, PyObject *args, "the first argument must be callable"); return NULL; } - PyObject *callback = PyTuple_Pack(3, func, args, kwargs); + PyObject *callback = PyTuple_Pack(3, func, args, + kwargs ? kwargs : Py_None); if (callback == NULL) { return NULL; diff --git a/Modules/clinic/_csv.c.h b/Modules/clinic/_csv.c.h index df3bdbadf29aed6..09dbcbea7e057b1 100644 --- a/Modules/clinic/_csv.c.h +++ b/Modules/clinic/_csv.c.h @@ -70,15 +70,7 @@ _csv_Dialect___replace__(PyObject *self, PyObject *args, PyObject *kwargs) if (!_PyArg_NoPositional("__replace__", args)) { goto exit; } - if (kwargs == NULL) { - changes = PyDict_New(); - if (changes == NULL) { - goto exit; - } - } - else { - changes = Py_NewRef(kwargs); - } + changes = Py_XNewRef(kwargs); return_value = _csv_Dialect___replace___impl((DialectObj *)self, changes); exit: @@ -142,15 +134,7 @@ _csv_reader(PyObject *module, PyObject *args, PyObject *kwargs) } dialect = PyTuple_GET_ITEM(args, 1); skip_optional: - if (kwargs == NULL) { - fmtparams = PyDict_New(); - if (fmtparams == NULL) { - goto exit; - } - } - else { - fmtparams = Py_NewRef(kwargs); - } + fmtparams = Py_XNewRef(kwargs); return_value = _csv_reader_impl(module, iterable, dialect, fmtparams); exit: @@ -195,15 +179,7 @@ _csv_writer(PyObject *module, PyObject *args, PyObject *kwargs) } dialect = PyTuple_GET_ITEM(args, 1); skip_optional: - if (kwargs == NULL) { - fmtparams = PyDict_New(); - if (fmtparams == NULL) { - goto exit; - } - } - else { - fmtparams = Py_NewRef(kwargs); - } + fmtparams = Py_XNewRef(kwargs); return_value = _csv_writer_impl(module, output_file, dialect, fmtparams); exit: @@ -266,15 +242,7 @@ _csv_register_dialect(PyObject *module, PyObject *args, PyObject *kwargs) } dialect_obj = PyTuple_GET_ITEM(args, 1); skip_optional: - if (kwargs == NULL) { - fmtparams = PyDict_New(); - if (fmtparams == NULL) { - goto exit; - } - } - else { - fmtparams = Py_NewRef(kwargs); - } + fmtparams = Py_XNewRef(kwargs); return_value = _csv_register_dialect_impl(module, name_obj, dialect_obj, fmtparams); exit: @@ -465,4 +433,4 @@ _csv_field_size_limit(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=7e3a92d524d88023 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=30eac1c04ff9d38a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 9b0494f35fae9a8..afc9daa5288f2c5 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -51,15 +51,7 @@ _elementtree_SubElement(PyObject *module, PyObject *args, PyObject *kwargs) } attrib = PyTuple_GET_ITEM(args, 2); skip_optional: - if (kwargs == NULL) { - extra = PyDict_New(); - if (extra == NULL) { - goto exit; - } - } - else { - extra = Py_NewRef(kwargs); - } + extra = Py_XNewRef(kwargs); return_value = _elementtree_SubElement_impl(module, parent, tag, attrib, extra); exit: @@ -1482,4 +1474,4 @@ _elementtree_XMLParser__setevents(PyObject *self, PyObject *const *args, Py_ssiz exit: return return_value; } -/*[clinic end generated code: output=01693c4bcc6c63b2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d454d27e90eb399e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_interpretersmodule.c.h b/Modules/clinic/_interpretersmodule.c.h index f297280279c5cdf..4d087b152d2b7f7 100644 --- a/Modules/clinic/_interpretersmodule.c.h +++ b/Modules/clinic/_interpretersmodule.c.h @@ -50,15 +50,7 @@ _interpreters_new_config(PyObject *module, PyObject *args, PyObject *kwargs) goto exit; } skip_optional: - if (kwargs == NULL) { - overrides = PyDict_New(); - if (overrides == NULL) { - goto exit; - } - } - else { - overrides = Py_NewRef(kwargs); - } + overrides = Py_XNewRef(kwargs); return_value = _interpreters_new_config_impl(module, name, overrides); exit: @@ -1260,4 +1252,4 @@ _interpreters_capture_exception(PyObject *module, PyObject *const *args, Py_ssiz exit: return return_value; } -/*[clinic end generated code: output=fa37872d6ee729ef input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ad79d44c2b139a83 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic_kwds.c.h b/Modules/clinic/_testclinic_kwds.c.h index faae3f3d0c26584..729ba1b6fa7c3da 100644 --- a/Modules/clinic/_testclinic_kwds.c.h +++ b/Modules/clinic/_testclinic_kwds.c.h @@ -33,15 +33,7 @@ lone_kwds(PyObject *module, PyObject *args, PyObject *kwargs) if (!_PyArg_NoPositional("lone_kwds", args)) { goto exit; } - if (kwargs == NULL) { - __clinic_kwds = PyDict_New(); - if (__clinic_kwds == NULL) { - goto exit; - } - } - else { - __clinic_kwds = Py_NewRef(kwargs); - } + __clinic_kwds = Py_XNewRef(kwargs); return_value = lone_kwds_impl(module, __clinic_kwds); exit: @@ -76,15 +68,7 @@ kwds_with_pos_only(PyObject *module, PyObject *args, PyObject *kwargs) } a = PyTuple_GET_ITEM(args, 0); b = PyTuple_GET_ITEM(args, 1); - if (kwargs == NULL) { - __clinic_kwds = PyDict_New(); - if (__clinic_kwds == NULL) { - goto exit; - } - } - else { - __clinic_kwds = Py_NewRef(kwargs); - } + __clinic_kwds = Py_XNewRef(kwargs); return_value = kwds_with_pos_only_impl(module, a, b, __clinic_kwds); exit: @@ -123,15 +107,7 @@ kwds_with_optional_pos_only(PyObject *module, PyObject *args, PyObject *kwargs) } b = PyTuple_GET_ITEM(args, 1); skip_optional: - if (kwargs == NULL) { - __clinic_kwds = PyDict_New(); - if (__clinic_kwds == NULL) { - goto exit; - } - } - else { - __clinic_kwds = Py_NewRef(kwargs); - } + __clinic_kwds = Py_XNewRef(kwargs); return_value = kwds_with_optional_pos_only_impl(module, a, b, __clinic_kwds); exit: @@ -160,18 +136,9 @@ kwds_with_stararg(PyObject *module, PyObject *args, PyObject *kwargs) PyObject *__clinic_kwds = NULL; __clinic_args = Py_NewRef(args); - if (kwargs == NULL) { - __clinic_kwds = PyDict_New(); - if (__clinic_kwds == NULL) { - goto exit; - } - } - else { - __clinic_kwds = Py_NewRef(kwargs); - } + __clinic_kwds = Py_XNewRef(kwargs); return_value = kwds_with_stararg_impl(module, __clinic_args, __clinic_kwds); -exit: /* Cleanup for args */ Py_XDECREF(__clinic_args); /* Cleanup for kwds */ @@ -211,15 +178,7 @@ kwds_with_pos_only_and_stararg(PyObject *module, PyObject *args, PyObject *kwarg if (!__clinic_args) { goto exit; } - if (kwargs == NULL) { - __clinic_kwds = PyDict_New(); - if (__clinic_kwds == NULL) { - goto exit; - } - } - else { - __clinic_kwds = Py_NewRef(kwargs); - } + __clinic_kwds = Py_XNewRef(kwargs); return_value = kwds_with_pos_only_and_stararg_impl(module, a, b, __clinic_args, __clinic_kwds); exit: @@ -230,4 +189,4 @@ kwds_with_pos_only_and_stararg(PyObject *module, PyObject *args, PyObject *kwarg return return_value; } -/*[clinic end generated code: output=27a7ba9440b9a884 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ad46500147228faf input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_zoneinfo.c.h b/Modules/clinic/_zoneinfo.c.h index 5d13e5b745562c2..b3c5defcbd937d1 100644 --- a/Modules/clinic/_zoneinfo.c.h +++ b/Modules/clinic/_zoneinfo.c.h @@ -425,15 +425,7 @@ zoneinfo_ZoneInfo___init_subclass__(PyObject *type, PyObject *args, PyObject *kw if (!_PyArg_NoPositional("__init_subclass__", args)) { goto exit; } - if (kwargs == NULL) { - __clinic_kwargs = PyDict_New(); - if (__clinic_kwargs == NULL) { - goto exit; - } - } - else { - __clinic_kwargs = Py_NewRef(kwargs); - } + __clinic_kwargs = Py_XNewRef(kwargs); return_value = zoneinfo_ZoneInfo___init_subclass___impl((PyTypeObject *)type, __clinic_kwargs); exit: @@ -442,4 +434,4 @@ zoneinfo_ZoneInfo___init_subclass__(PyObject *type, PyObject *args, PyObject *kw return return_value; } -/*[clinic end generated code: output=d7aeeec4b950fc35 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=42fad9178a9395ea input=a9049054013a1b77]*/ diff --git a/Modules/clinic/atexitmodule.c.h b/Modules/clinic/atexitmodule.c.h index 9afa3a15a29614e..8297f03c0276b59 100644 --- a/Modules/clinic/atexitmodule.c.h +++ b/Modules/clinic/atexitmodule.c.h @@ -39,15 +39,7 @@ atexit_register(PyObject *module, PyObject *args, PyObject *kwargs) if (!__clinic_args) { goto exit; } - if (kwargs == NULL) { - __clinic_kwargs = PyDict_New(); - if (__clinic_kwargs == NULL) { - goto exit; - } - } - else { - __clinic_kwargs = Py_NewRef(kwargs); - } + __clinic_kwargs = Py_XNewRef(kwargs); return_value = atexit_register_impl(module, func, __clinic_args, __clinic_kwargs); exit: @@ -58,4 +50,4 @@ atexit_register(PyObject *module, PyObject *args, PyObject *kwargs) return return_value; } -/*[clinic end generated code: output=8fd8f691fdbc4e0a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ab4b375d516aa59b input=a9049054013a1b77]*/ diff --git a/Objects/clinic/namespaceobject.c.h b/Objects/clinic/namespaceobject.c.h index cdd9ef586c79ecd..9f6301592d2c84c 100644 --- a/Objects/clinic/namespaceobject.c.h +++ b/Objects/clinic/namespaceobject.c.h @@ -25,15 +25,7 @@ namespace___replace__(PyObject *self, PyObject *args, PyObject *kwargs) if (!_PyArg_NoPositional("__replace__", args)) { goto exit; } - if (kwargs == NULL) { - changes = PyDict_New(); - if (changes == NULL) { - goto exit; - } - } - else { - changes = Py_NewRef(kwargs); - } + changes = Py_XNewRef(kwargs); return_value = namespace___replace___impl((_PyNamespaceObject *)self, changes); exit: @@ -42,4 +34,4 @@ namespace___replace__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } -/*[clinic end generated code: output=67b5fcb6a6aee559 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1b1c015f14360d64 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/structseq.c.h b/Objects/clinic/structseq.c.h index f115f8ddcf490c6..ce5014311f4cca7 100644 --- a/Objects/clinic/structseq.c.h +++ b/Objects/clinic/structseq.c.h @@ -87,15 +87,7 @@ structseq___replace__(PyObject *self, PyObject *args, PyObject *kwargs) if (!_PyArg_NoPositional("__replace__", args)) { goto exit; } - if (kwargs == NULL) { - changes = PyDict_New(); - if (changes == NULL) { - goto exit; - } - } - else { - changes = Py_NewRef(kwargs); - } + changes = Py_XNewRef(kwargs); return_value = structseq___replace___impl((PyStructSequence *)self, changes); exit: @@ -104,4 +96,4 @@ structseq___replace__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } -/*[clinic end generated code: output=18fcf88e5df26774 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f0231faecca9248b input=a9049054013a1b77]*/ diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index e1ed2f6df9a6e21..6d5fced0ed73b35 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -261,7 +261,9 @@ namespace___replace___impl(_PyNamespaceObject *self, PyObject *changes) Py_DECREF(result); return NULL; } - if (PyDict_Update(((_PyNamespaceObject*)result)->ns_dict, changes) < 0) { + if (changes != NULL + && PyDict_Update(((_PyNamespaceObject*)result)->ns_dict, changes) < 0) + { Py_DECREF(result); return NULL; } diff --git a/Objects/structseq.c b/Objects/structseq.c index 36daf523d8004f1..949790e6f416e95 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -424,9 +424,10 @@ structseq___replace___impl(PyStructSequence *self, PyObject *changes) // We do not support types with unnamed fields, so we can iterate over // i >= n_visible_fields case without slicing with (i - n_unnamed_fields). for (i = 0; i < n_fields; ++i) { - PyObject *ob; - if (PyDict_PopString(changes, Py_TYPE(self)->tp_members[i].name, - &ob) < 0) { + PyObject *ob = NULL; + if (changes != NULL + && PyDict_PopString(changes, Py_TYPE(self)->tp_members[i].name, + &ob) < 0) { goto error; } if (ob == NULL) { @@ -435,7 +436,7 @@ structseq___replace___impl(PyStructSequence *self, PyObject *changes) result->ob_item[i] = ob; } // Check if there are any unexpected fields. - if (PyDict_GET_SIZE(changes) > 0) { + if (changes != NULL && PyDict_GET_SIZE(changes) > 0) { PyObject *names = PyDict_Keys(changes); if (names) { PyErr_Format(PyExc_TypeError, "Got unexpected field name(s): %R", names); diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py index c54dab302b66586..e3571eb597f24af 100644 --- a/Tools/clinic/libclinic/converters.py +++ b/Tools/clinic/libclinic/converters.py @@ -1449,15 +1449,5 @@ def cleanup(self) -> str: return f'Py_XDECREF({self.parser_name});\n' def parse_var_keyword(self) -> str: - param_name = self.parser_name - return f""" - if (kwargs == NULL) {{{{ - {param_name} = PyDict_New(); - if ({param_name} == NULL) {{{{ - goto exit; - }}}} - }}}} - else {{{{ - {param_name} = Py_NewRef(kwargs); - }}}} - """ + # NULL when there are no keywords: no call allocates an empty dict. + return f"{self.parser_name} = Py_XNewRef(kwargs);\n" From 5f46ac97490ddfd8f7a4d01b9f765cccd00d0e72 Mon Sep 17 00:00:00 2001 From: XiaohongGong Date: Sun, 8 Nov 2026 06:18:39 +0800 Subject: [PATCH 032/229] gh-160872: Keep PyLong "x_divrem" inner loop carry in 64 bits (#161156) * gh-160872: Keep PyLong "x_divrem" inner loop carry in 64 bits --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- .../2026-11-02-05-50-35.gh-issue-160872.x8NvzN.rst | 1 + Objects/longobject.c | 12 +++++------- 2 files changed, 6 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-02-05-50-35.gh-issue-160872.x8NvzN.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-02-05-50-35.gh-issue-160872.x8NvzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-02-05-50-35.gh-issue-160872.x8NvzN.rst new file mode 100644 index 000000000000000..2911abe2f166f1c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-02-05-50-35.gh-issue-160872.x8NvzN.rst @@ -0,0 +1 @@ +Optimize division and remainder operations on large :class:`int` values. diff --git a/Objects/longobject.c b/Objects/longobject.c index 9d02793fdd63665..09bfe4ad4641281 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3325,8 +3325,7 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) int d; digit wm1, wm2, carry, q, r, vtop, *v0, *vk, *w0, *ak; twodigits vv; - sdigit zhi; - stwodigits z; + stwodigits z, zhi; /* We follow Knuth [The Art of Computer Programming, Vol. 2 (3rd edn.), section 4.3.1, Algorithm D], except that we don't explicitly @@ -3416,16 +3415,15 @@ x_divrem(PyLongObject *v1, PyLongObject *w1, PyLongObject **prem) for (i = 0; i < size_w; ++i) { /* invariants: -PyLong_BASE <= -q <= zhi <= 0; -PyLong_BASE * q <= z < PyLong_BASE */ - z = (sdigit)vk[i] + zhi - + z = (stwodigits)vk[i] + zhi - (stwodigits)q * (stwodigits)w0[i]; vk[i] = (digit)z & PyLong_MASK; - zhi = (sdigit)Py_ARITHMETIC_RIGHT_SHIFT(stwodigits, - z, PyLong_SHIFT); + zhi = Py_ARITHMETIC_RIGHT_SHIFT(stwodigits, z, PyLong_SHIFT); } /* add w back if q was too large (this branch taken rarely) */ - assert((sdigit)vtop + zhi == -1 || (sdigit)vtop + zhi == 0); - if ((sdigit)vtop + zhi < 0) { + assert((stwodigits)vtop + zhi == -1 || (stwodigits)vtop + zhi == 0); + if ((stwodigits)vtop + zhi < 0) { carry = 0; for (i = 0; i < size_w; ++i) { carry += vk[i] + w0[i]; From f76742309542a255af560adccbd06197f350df5d Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 7 Nov 2026 14:48:59 -0800 Subject: [PATCH 033/229] gh-161242: Use AC vectorcall for AttributeError, ImportError, NameError (#161462) These exceptions and their subclasses had no vectorcall, and `__init__` parsed keywords through PyArg_ParseTupleAndKeywords with a temporary empty tuple on every call. Error messages are unchanged except when more keywords than parameters are given, which now names the unexpected keyword. The clinic inputs have no docstring, so docstrings are unchanged and these exceptions don't gain a `__text_signature__`. pyperf microbenchmark (PGO+LTO, run serially): | Benchmark | base | vectorcall | |----------------------------------|:-------:|:---------------------:| | AttributeError(s, name=s, obj=s) | 127 ns | 41.9 ns: 3.02x faster | | ImportError(s, name=s, path=s) | 129 ns | 43.4 ns: 2.96x faster | | NameError(s, name=s) | 88.3 ns | 39.3 ns: 2.24x faster | | ModuleNotFoundError(s, name=s) | 89.9 ns | 40.6 ns: 2.21x faster | | AttributeError(s) | 39.4 ns | 32.1 ns: 1.23x faster | | UnboundLocalError(s) | 38.6 ns | 31.8 ns: 1.22x faster | | NameError(s) | 38.5 ns | 32.2 ns: 1.19x faster | | ImportError(s) | 40.7 ns | 34.4 ns: 1.18x faster | | attribute_error_to_none() | 176 ns | 173 ns: 1.02x faster | Benchmark code: ```python import pyperf runner = pyperf.Runner() setup = """s = 'x' class O: pass o = O() def attribute_error_to_none(): try: o.x except AttributeError: pass """ for stmt in ["AttributeError(s)", "AttributeError(s, name=s, obj=s)", "NameError(s)", "NameError(s, name=s)", "ImportError(s)", "ImportError(s, name=s, path=s)", "ModuleNotFoundError(s, name=s)", "UnboundLocalError(s)", "attribute_error_to_none()"]: runner.timeit(stmt, stmt, setup) ``` Co-Authored-By: Claude Opus 5.5 --- Lib/test/test_call.py | 2 +- ...-11-06-21-21-04.gh-issue-161242.d4vNgC.rst | 3 + Objects/clinic/exceptions.c.h | 329 +++++++++++++++++- Objects/exceptions.c | 110 +++--- 4 files changed, 389 insertions(+), 55 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-21-21-04.gh-issue-161242.d4vNgC.rst diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 7115359156eb696..8d8b7f9bbc38fb3 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -208,7 +208,7 @@ def test_varargs14_kw(self): itertools.product, 0, repeat=1, foo=2) def test_varargs15_kw(self): - msg = r"^ImportError\(\) takes at most 3 keyword arguments \(4 given\)$" + msg = r"^ImportError\(\) got an unexpected keyword argument 'foo'$" self.assertRaisesRegex(TypeError, msg, ImportError, 0, name=1, path=2, name_from=3, foo=3) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-21-21-04.gh-issue-161242.d4vNgC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-21-21-04.gh-issue-161242.d4vNgC.rst new file mode 100644 index 000000000000000..3dbd521612feedb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-21-21-04.gh-issue-161242.d4vNgC.rst @@ -0,0 +1,3 @@ +Speed up :exc:`AttributeError`, :exc:`ImportError` and +:exc:`NameError` by enabling :ref:`vectorcall`. Unexpected +keyword arguments are now reported by name. diff --git a/Objects/clinic/exceptions.c.h b/Objects/clinic/exceptions.c.h index c06cdb0444ee971..b77606c3c7a8c76 100644 --- a/Objects/clinic/exceptions.c.h +++ b/Objects/clinic/exceptions.c.h @@ -2,8 +2,12 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +#endif #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_BadArgument() +#include "pycore_runtime.h" // _Py_SINGLETON() static PyObject * BaseException_str_impl(PyBaseExceptionObject *self); @@ -362,6 +366,329 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *matcher_value) return return_value; } + +static int +ImportError_init_impl(PyImportErrorObject *self, PyObject *args, + PyObject *name, PyObject *path, PyObject *name_from); + +static int +ImportError_init_helper(PyObject *self, PyObject *const *args, + Py_ssize_t nargs, Py_ssize_t nkw, PyObject *kwargs, PyObject *kwnames) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(name), &_Py_ID(path), &_Py_ID(name_from), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", "path", "name_from", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "ImportError", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject * const *fastargs; + Py_ssize_t noptargs = 0 + nkw - 0; + PyObject *__clinic_args = NULL; + PyObject *name = NULL; + PyObject *path = NULL; + PyObject *name_from = NULL; + + fastargs = _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (fastargs[0]) { + name = fastargs[0]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[1]) { + path = fastargs[1]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + name_from = fastargs[2]; +skip_optional_kwonly: + __clinic_args = PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } + return_value = ImportError_init_impl((PyImportErrorObject *)self, __clinic_args, name, path, name_from); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static int +ImportError_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + return ImportError_init_helper(self, _PyTuple_CAST(args)->ob_item, + PyTuple_GET_SIZE(args), + kwargs ? PyDict_GET_SIZE(kwargs) : 0, + kwargs, NULL); +} + +static PyObject * +ImportError_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; + int _result; + + assert(_PyType_CAST(type)->tp_init == ImportError_init); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + self = _PyType_CAST(type)->tp_new(_PyType_CAST(type), + (PyObject *)&_Py_SINGLETON(tuple_empty), NULL); + if (self == NULL) { + return NULL; + } + _result = ImportError_init_helper(self, args, nargs, + kwnames ? PyTuple_GET_SIZE(kwnames) : 0, + NULL, kwnames); + if (_result != 0) { + Py_DECREF(self); + return NULL; + } + return self; +} + +static int +NameError_init_impl(PyNameErrorObject *self, PyObject *args, PyObject *name); + +static int +NameError_init_helper(PyObject *self, PyObject *const *args, + Py_ssize_t nargs, Py_ssize_t nkw, PyObject *kwargs, PyObject *kwnames) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "NameError", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject * const *fastargs; + Py_ssize_t noptargs = 0 + nkw - 0; + PyObject *__clinic_args = NULL; + PyObject *name = NULL; + + fastargs = _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + name = fastargs[0]; +skip_optional_kwonly: + __clinic_args = PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } + return_value = NameError_init_impl((PyNameErrorObject *)self, __clinic_args, name); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static int +NameError_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + return NameError_init_helper(self, _PyTuple_CAST(args)->ob_item, + PyTuple_GET_SIZE(args), + kwargs ? PyDict_GET_SIZE(kwargs) : 0, + kwargs, NULL); +} + +static PyObject * +NameError_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; + int _result; + + assert(_PyType_CAST(type)->tp_init == NameError_init); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + self = _PyType_CAST(type)->tp_new(_PyType_CAST(type), + (PyObject *)&_Py_SINGLETON(tuple_empty), NULL); + if (self == NULL) { + return NULL; + } + _result = NameError_init_helper(self, args, nargs, + kwnames ? PyTuple_GET_SIZE(kwnames) : 0, + NULL, kwnames); + if (_result != 0) { + Py_DECREF(self); + return NULL; + } + return self; +} + +static int +AttributeError_init_impl(PyAttributeErrorObject *self, PyObject *args, + PyObject *name, PyObject *obj); + +static int +AttributeError_init_helper(PyObject *self, PyObject *const *args, + Py_ssize_t nargs, Py_ssize_t nkw, PyObject *kwargs, PyObject *kwnames) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(name), &_Py_ID(obj), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", "obj", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "AttributeError", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t noptargs = 0 + nkw - 0; + PyObject *__clinic_args = NULL; + PyObject *name = NULL; + PyObject *obj = NULL; + + fastargs = _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 0, /*varpos*/ 1, argsbuf); + if (!fastargs) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (fastargs[0]) { + name = fastargs[0]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + obj = fastargs[1]; +skip_optional_kwonly: + __clinic_args = PyTuple_FromArray(args, nargs); + if (__clinic_args == NULL) { + goto exit; + } + return_value = AttributeError_init_impl((PyAttributeErrorObject *)self, __clinic_args, name, obj); + +exit: + /* Cleanup for args */ + Py_XDECREF(__clinic_args); + + return return_value; +} + +static int +AttributeError_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + return AttributeError_init_helper(self, _PyTuple_CAST(args)->ob_item, + PyTuple_GET_SIZE(args), + kwargs ? PyDict_GET_SIZE(kwargs) : 0, + kwargs, NULL); +} + +static PyObject * +AttributeError_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; + int _result; + + assert(_PyType_CAST(type)->tp_init == AttributeError_init); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + self = _PyType_CAST(type)->tp_new(_PyType_CAST(type), + (PyObject *)&_Py_SINGLETON(tuple_empty), NULL); + if (self == NULL) { + return NULL; + } + _result = AttributeError_init_helper(self, args, nargs, + kwnames ? PyTuple_GET_SIZE(kwnames) : 0, + NULL, kwnames); + if (_result != 0) { + Py_DECREF(self); + return NULL; + } + return self; +} #define BASEEXCEPTION_ARGS_GETSETDEF {"args", (getter)BaseException_args_get, (setter)BaseException_args_set, NULL}, #define BASEEXCEPTION___TRACEBACK___GETSETDEF {"__traceback__", (getter)BaseException___traceback___get, (setter)BaseException___traceback___set, NULL}, @@ -370,4 +697,4 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *matcher_value) #define BASEEXCEPTION___CAUSE___GETSETDEF {"__cause__", (getter)BaseException___cause___get, (setter)BaseException___cause___set, NULL}, -/*[clinic end generated code: output=20829af0c376b1e9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c8eaffee13bc041d input=a9049054013a1b77]*/ diff --git a/Objects/exceptions.c b/Objects/exceptions.c index e2f757c5ee9c0be..dc0ba0d7e533dda 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -24,8 +24,11 @@ /*[clinic input] class BaseException "PyBaseExceptionObject *" "&PyExc_BaseException" class BaseExceptionGroup "PyBaseExceptionGroupObject *" "&PyExc_BaseExceptionGroup" +class ImportError "PyImportErrorObject *" "&PyExc_ImportError" +class NameError "PyNameErrorObject *" "&PyExc_NameError" +class AttributeError "PyAttributeErrorObject *" "&PyExc_AttributeError" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b7c45e78cff8edc3]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=fce4682940ab8948]*/ static struct _Py_exc_state* @@ -1778,29 +1781,27 @@ PyImportErrorObject_CAST(PyObject *self) return (PyImportErrorObject *)self; } +/*[clinic input] +@vectorcall +ImportError.__init__ as ImportError_init + + *args: tuple + name: object = NULL + path: object = NULL + name_from: object = NULL + +[clinic start generated code]*/ + static int -ImportError_init(PyObject *op, PyObject *args, PyObject *kwds) +ImportError_init_impl(PyImportErrorObject *self, PyObject *args, + PyObject *name, PyObject *path, PyObject *name_from) +/*[clinic end generated code: output=4917add0f9473bd2 input=4e94a3b530f9be97]*/ { - static char *kwlist[] = {"name", "path", "name_from", 0}; - PyObject *empty_tuple; PyObject *msg = NULL; - PyObject *name = NULL; - PyObject *path = NULL; - PyObject *name_from = NULL; - if (BaseException_init(op, args, NULL) == -1) - return -1; - - PyImportErrorObject *self = PyImportErrorObject_CAST(op); - empty_tuple = PyTuple_New(0); - if (!empty_tuple) - return -1; - if (!PyArg_ParseTupleAndKeywords(empty_tuple, kwds, "|$OOO:ImportError", kwlist, - &name, &path, &name_from)) { - Py_DECREF(empty_tuple); + if (BaseException_init((PyObject *)self, args, NULL) == -1) { return -1; } - Py_DECREF(empty_tuple); Py_XSETREF(self->name, Py_XNewRef(name)); Py_XSETREF(self->path, Py_XNewRef(path)); @@ -1814,6 +1815,7 @@ ImportError_init(PyObject *op, PyObject *args, PyObject *kwds) return 0; } + static int ImportError_clear(PyObject *op) { @@ -2579,33 +2581,29 @@ PyNameErrorObject_CAST(PyObject *self) return (PyNameErrorObject *)self; } -static int -NameError_init(PyObject *op, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"name", NULL}; - PyObject *name = NULL; +/*[clinic input] +@vectorcall +NameError.__init__ as NameError_init - if (BaseException_init(op, args, NULL) == -1) { - return -1; - } + *args: tuple + name: object = NULL - PyObject *empty_tuple = PyTuple_New(0); - if (!empty_tuple) { - return -1; - } - if (!PyArg_ParseTupleAndKeywords(empty_tuple, kwds, "|$O:NameError", kwlist, - &name)) { - Py_DECREF(empty_tuple); +[clinic start generated code]*/ + +static int +NameError_init_impl(PyNameErrorObject *self, PyObject *args, PyObject *name) +/*[clinic end generated code: output=f15305075d9aa272 input=2e2b7e781ae146b1]*/ +{ + if (BaseException_init((PyObject *)self, args, NULL) == -1) { return -1; } - Py_DECREF(empty_tuple); - PyNameErrorObject *self = PyNameErrorObject_CAST(op); Py_XSETREF(self->name, Py_XNewRef(name)); return 0; } + static int NameError_clear(PyObject *op) { @@ -2662,35 +2660,32 @@ PyAttributeErrorObject_CAST(PyObject *self) return (PyAttributeErrorObject *)self; } -static int -AttributeError_init(PyObject *op, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"name", "obj", NULL}; - PyObject *name = NULL; - PyObject *obj = NULL; +/*[clinic input] +@vectorcall +AttributeError.__init__ as AttributeError_init - if (BaseException_init(op, args, NULL) == -1) { - return -1; - } + *args: tuple + name: object = NULL + obj: object = NULL - PyObject *empty_tuple = PyTuple_New(0); - if (!empty_tuple) { - return -1; - } - if (!PyArg_ParseTupleAndKeywords(empty_tuple, kwds, "|$OO:AttributeError", kwlist, - &name, &obj)) { - Py_DECREF(empty_tuple); +[clinic start generated code]*/ + +static int +AttributeError_init_impl(PyAttributeErrorObject *self, PyObject *args, + PyObject *name, PyObject *obj) +/*[clinic end generated code: output=3d346f9a389f60fa input=bed43aafea47f1a0]*/ +{ + if (BaseException_init((PyObject *)self, args, NULL) == -1) { return -1; } - Py_DECREF(empty_tuple); - PyAttributeErrorObject *self = PyAttributeErrorObject_CAST(op); Py_XSETREF(self->name, Py_XNewRef(name)); Py_XSETREF(self->obj, Py_XNewRef(obj)); return 0; } + static int AttributeError_clear(PyObject *op) { @@ -4565,6 +4560,15 @@ _PyExc_InitTypes(PyInterpreterState *interp) { exc->tp_vectorcall = BaseException_vectorcall; } + else if (exc->tp_init == ImportError_init) { + exc->tp_vectorcall = ImportError_vectorcall; + } + else if (exc->tp_init == NameError_init) { + exc->tp_vectorcall = NameError_vectorcall; + } + else if (exc->tp_init == AttributeError_init) { + exc->tp_vectorcall = AttributeError_vectorcall; + } } return 0; } From 1955eb3ec065bda56ed74c7473f293b8f21a22b2 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 7 Nov 2026 17:06:25 -0800 Subject: [PATCH 034/229] gh-160725: Speed up print() by using string constants and doing faster binding (#160727) Pass the cached ' ' and '\n' strings for the default sep and end directly, instead of converting them from C strings with PyFile_WriteString(), which runs strlen() and the UTF-8 decoder on every call to get back the same cached string. Rebinding file.write during print() keeps working as before. GH-160803 significantly lowered binding cost. Could save ~5ns/call on x64 linux by binding only once, but that is left for work in https://discuss.python.org/t/make-print-use-a-single-write-call/109886 | Benchmark | main | this PR | |------------------------------------------------|:-------:|:---------------------:| | print(file=devnull) | 51.2 ns | 47.4 ns: 1.08x faster | | print('hello world', file=devnull) | 81.0 ns | 79.2 ns: 1.02x faster | | print('a', 1, 2.5, file=devnull) | 281 ns | 278 ns: 1.01x faster | | print('a', 'b', sep='-', end='', file=devnull) | 124 ns | 126 ns: 1.02x slower | Benchmark hidden because not significant (1): print(*ten_ints, file=devnull) Co-authored-by: Claude Opus 5.5 --- Python/bltinmodule.c | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c8e58f77d1d375f..72f2e356c34f7c8 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2240,7 +2240,7 @@ builtin_print_impl(PyObject *module, PyObject * const *objects, PyObject *file, int flush) /*[clinic end generated code: output=38d8def56c837bcc input=ff35cb3d59ee8115]*/ { - int i, err; + int i; if (file == Py_None) { file = PySys_GetAttr(&_Py_ID(stdout)); @@ -2259,7 +2259,7 @@ builtin_print_impl(PyObject *module, PyObject * const *objects, } if (sep == Py_None) { - sep = NULL; + sep = _Py_LATIN1_CHR(' '); } else if (sep && !PyUnicode_Check(sep)) { PyErr_Format(PyExc_TypeError, @@ -2269,7 +2269,7 @@ builtin_print_impl(PyObject *module, PyObject * const *objects, return NULL; } if (end == Py_None) { - end = NULL; + end = _Py_LATIN1_CHR('\n'); } else if (end && !PyUnicode_Check(end)) { PyErr_Format(PyExc_TypeError, @@ -2280,32 +2280,16 @@ builtin_print_impl(PyObject *module, PyObject * const *objects, } for (i = 0; i < objects_length; i++) { - if (i > 0) { - if (sep == NULL) { - err = PyFile_WriteString(" ", file); - } - else { - err = PyFile_WriteObject(sep, file, Py_PRINT_RAW); - } - if (err) { - Py_DECREF(file); - return NULL; - } + if (i > 0 && PyFile_WriteObject(sep, file, Py_PRINT_RAW) < 0) { + Py_DECREF(file); + return NULL; } - err = PyFile_WriteObject(objects[i], file, Py_PRINT_RAW); - if (err) { + if (PyFile_WriteObject(objects[i], file, Py_PRINT_RAW) < 0) { Py_DECREF(file); return NULL; } } - - if (end == NULL) { - err = PyFile_WriteString("\n", file); - } - else { - err = PyFile_WriteObject(end, file, Py_PRINT_RAW); - } - if (err) { + if (PyFile_WriteObject(end, file, Py_PRINT_RAW) < 0) { Py_DECREF(file); return NULL; } From a497a11c46c2bf096dd544e7a422cc457e806916 Mon Sep 17 00:00:00 2001 From: Vismay <89745751+vismaytiwari@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:16:48 +0530 Subject: [PATCH 035/229] gh-153285: Preserve LTO target names on macOS (#153532) With `--with-lto` on macOS, linker flags include Make's `$@` target variable. Immediate expansion in `PY_CORE_EXE_LDFLAGS` erased the target name, causing executable links to share an LTO object path. Parallel links could then produce incorrect executables and cause `test_embed` failures. Use recursive assignment (`=` instead of `:=`) so `$@` expands when each link recipe runs. --- Makefile.pre.in | 2 +- .../next/Build/2026-07-11-01-53-58.gh-issue-153285.YmSox4.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2026-07-11-01-53-58.gh-issue-153285.YmSox4.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index f1b5eb3bba34fd1..3955afae436e9c5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -135,7 +135,7 @@ PY_CORE_CFLAGS= $(PY_STDMODULE_CFLAGS) -DPy_BUILD_CORE # LDFLAGS used to build executables and shared targets. PY_CORE_LDFLAGS=$(PY_LDFLAGS) $(PY_LDFLAGS_NODIST) CONFIGURE_EXE_LDFLAGS=@EXE_LDFLAGS@ -PY_CORE_EXE_LDFLAGS:= $(if $(CONFIGURE_EXE_LDFLAGS), $(CONFIGURE_EXE_LDFLAGS) $(PY_LDFLAGS_NODIST), $(PY_CORE_LDFLAGS)) +PY_CORE_EXE_LDFLAGS= $(if $(CONFIGURE_EXE_LDFLAGS), $(CONFIGURE_EXE_LDFLAGS) $(PY_LDFLAGS_NODIST), $(PY_CORE_LDFLAGS)) # Strict or non-strict aliasing flags used to compile dtoa.c, see above CFLAGS_ALIASING=@CFLAGS_ALIASING@ # Compilation flags only for ceval.c. diff --git a/Misc/NEWS.d/next/Build/2026-07-11-01-53-58.gh-issue-153285.YmSox4.rst b/Misc/NEWS.d/next/Build/2026-07-11-01-53-58.gh-issue-153285.YmSox4.rst new file mode 100644 index 000000000000000..d672c8b9910d76d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-07-11-01-53-58.gh-issue-153285.YmSox4.rst @@ -0,0 +1,2 @@ +Fix incorrect executable linking on macOS with ``--with-lto`` caused by +premature expansion of Make's ``$@`` target variable. From c56b0138d9390fe32b1342120ae0628d6d902f66 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 8 Nov 2026 15:39:21 +0300 Subject: [PATCH 036/229] gh-161602: Enable `test_builtin` test with `PYTHON_LAZY_IMPORTS=all` (#161603) --- Lib/test/lazy_imports_all_exclude.txt | 1 - Lib/test/test_builtin.py | 10 ++++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/lazy_imports_all_exclude.txt b/Lib/test/lazy_imports_all_exclude.txt index 8d303858139b9f7..864d723fe56b11a 100644 --- a/Lib/test/lazy_imports_all_exclude.txt +++ b/Lib/test/lazy_imports_all_exclude.txt @@ -19,7 +19,6 @@ test.test_inspect.test_inspect test___all__ test__interpreters -test_builtin test_clinic test_crossinterp test_datetime diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 501f482570ac329..d21de4adc2a98d8 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1052,10 +1052,16 @@ def test_eval_builtins_mapping(self): eval, code, ns) def test_exec_builtins_mapping_import(self): + # Globally enabled lazy imports do use different import hooks: + if sys.get_lazy_imports() == "all": + hook = "__lazy_import__" + else: + hook = "__import__" + code = compile("import foo.bar", "test", "exec") ns = {'__builtins__': types.MappingProxyType({})} - self.assertRaisesRegex(ImportError, "__import__ not found", exec, code, ns) - ns = {'__builtins__': types.MappingProxyType({'__import__': lambda *args: args})} + self.assertRaisesRegex(ImportError, f"{hook} not found", exec, code, ns) + ns = {'__builtins__': types.MappingProxyType({hook: lambda *args: args})} exec(code, ns) self.assertEqual(ns['foo'], ('foo.bar', ns, ns, None, 0)) From 3ad9ded4635547c4b9ee6bf4b58c6f3bb890bc39 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 8 Nov 2026 15:26:45 +0100 Subject: [PATCH 037/229] gh-161609: Catch ProcessLookupError on os.getsid() in regrtest (#161610) --- Lib/test/libregrtest/run_workers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index ad02d8dba7c23e7..7118a67a2336c6f 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -153,6 +153,9 @@ def _kill(self) -> None: # On OpenBSD getsid() is only allowed for a process in the # same session, so the failure means that it is not. use_killpg = True + except ProcessLookupError: + # The process completed in the meanwhile + return if use_killpg: what = f"{self} process group" From 403f6dd31a70613d4c49a4c9954aacb33a0637dc Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 8 Nov 2026 07:28:43 -0800 Subject: [PATCH 038/229] gh-88037 - Move port validation logic to parsing time (#159122) * Move port validation logic to parsing time * Fix CI. Handle the special case of file://C:\\ parsed by url2pathname function where ":" should not be treated as port separator. --------- Co-authored-by: Miguel Mendes --- Lib/test/test_urllib.py | 24 +++++++-- Lib/test/test_urllib2.py | 4 +- Lib/test/test_urlparse.py | 13 ++--- Lib/urllib/parse.py | 53 ++++++++++++------- Lib/urllib/request.py | 20 ++++++- ...6-10-11-13-08-07.gh-issue-88037.CzKcxh.rst | 3 ++ 6 files changed, 82 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-11-13-08-07.gh-issue-88037.CzKcxh.rst diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index ab59727a8fd1820..3858b03546061e6 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -7,6 +7,7 @@ import email.message import io import unittest +import unittest.mock from test import support from test.support import os_helper from test.support import socket_helper @@ -400,11 +401,17 @@ def test_url_host_with_newline_header_injection_rejected(self): host = "localhost\r\nX-injected: header\r\n" schemeless_url = "//" + host + ":8080/test/?test=a" try: - InvalidURL = http.client.InvalidURL + # Once \r\n are stripped from the URL, the ':' in + # "X-injected:" is mistaken for the port separator, and + # urlsplit()/urlparse() now reject the resulting + # non-numeric "port" at parse time, before a Request is + # even built -- rejecting the injection earlier than + # http.client's own InvalidURL check ever gets a chance to. with self.assertRaisesRegex( - InvalidURL, r"contain control.*\\r"): + ValueError, "Port could not be cast to integer value"): urllib.request.urlopen(f"http:{schemeless_url}") - with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"): + with self.assertRaisesRegex( + ValueError, "Port could not be cast to integer value"): urllib.request.urlopen(f"https:{schemeless_url}") finally: self.unfakehttp() @@ -1672,6 +1679,17 @@ def test_url2pathname_resolve_host(self): self.assertEqual(fn(f'//{socket.gethostname()}/foo/bar'), f'{sep}foo{sep}bar') self.assertEqual(fn(f'//{socket.gethostname()}/foo/bar', resolve_host=True), f'{sep}foo{sep}bar') + def test_url2pathname_win_netloc_not_validated_as_port(self): + # The Windows branch of url2pathname() stuffs things like drive + # letters into the netloc (e.g. '//C:\\' -> authority 'C:\\'), + # which is not a real host[:port] pair. url2pathname() must not + # reject it as an invalid port; run the Windows branch directly + # so this is exercised on every platform, not just win32. + with unittest.mock.patch('os.name', 'nt'): + fn = urllib.request.url2pathname + self.assertEqual(fn('//C:\\'), 'C:\\') + self.assertEqual(fn('//C:80/'), 'C:80\\') + @unittest.skipUnless(sys.platform == 'win32', 'test specific to Windows pathnames.') def test_url2pathname_win(self): diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index a5b5a3d706dd409..f01b94c581e87ac 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -951,9 +951,9 @@ def test_file(self): parsed._replace(netloc='localhost:80').geturl(), "file:///file_does_not_exist.txt", "file://not-a-local-host.com//dir/file.txt", - "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), + "file://%s:80/%s/%s" % (socket.gethostbyname('localhost'), os.getcwd(), TESTFN), - "file://somerandomhost.ontheinternet.com%s/%s" % + "file://somerandomhost.ontheinternet.com/%s/%s" % (os.getcwd(), TESTFN), ]: try: diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index a5b7966c7780e9e..112cc139d597b6c 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -917,9 +917,8 @@ def test_urlsplit_attributes(self): # Verify an illegal port raises ValueError url = b"HTTP://WWW.PYTHON.ORG:65536/doc/#frag" - p = urllib.parse.urlsplit(url) with self.assertRaisesRegex(ValueError, "out of range"): - p.port + urllib.parse.urlsplit(url) def test_urlsplit_remove_unsafe_bytes(self): # Remove ASCII tabs and newlines from input @@ -1029,10 +1028,8 @@ def test_attributes_bad_port(self, bytes, parse, port): self.skipTest('non-ASCII bytes') netloc = str_encode(netloc) url = str_encode(url) - p = parse(url) - self.assertEqual(p.netloc, netloc) with self.assertRaises(ValueError): - p.port + parse(url) @support.subTests('bytes', (False, True)) @support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse)) @@ -1670,13 +1667,11 @@ def test_splitting_bracketed_hosts(self): def test_port_casting_failure_message(self): message = "Port could not be cast to integer value as 'oracle'" - p1 = urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') with self.assertRaisesRegex(ValueError, message): - p1.port + urllib.parse.urlparse('http://Server=sde; Service=sde:oracle') - p2 = urllib.parse.urlsplit('http://Server=sde; Service=sde:oracle') with self.assertRaisesRegex(ValueError, message): - p2.port + urllib.parse.urlsplit('http://Server=sde; Service=sde:oracle') def test_telurl_params(self): p1 = urllib.parse.urlparse('tel:123-4;phone-context=+1-650-516') diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 4247b9a4b07fa3f..ee0bceb2928c06f 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -140,6 +140,27 @@ def _coerce_args(*args): return args + (_noop,) return _decode_args(args) + (_encode_result,) +def _parse_hostinfo(netloc): + _, _, hostinfo = netloc.rpartition('@') + _, have_open_br, bracketed = hostinfo.partition('[') + if have_open_br: + hostname, _, port = bracketed.partition(']') + _, _, port = port.partition(':') + else: + hostname, _, port = hostinfo.partition(':') + if not port: + port = None + return hostname, port + +def _validate_port(port): + if port.isdigit() and port.isascii(): + port = int(port) + else: + raise ValueError(f"Port could not be cast to integer value as {port!r}") + if not (0 <= port <= 65535): + raise ValueError("Port out of range 0-65535") + return port + # Result objects are more helpful than simple tuples class _ResultMixinStr(object): """Standard approach to encoding parsed results from str to bytes""" @@ -198,12 +219,7 @@ def hostname(self): def port(self): port = self._hostinfo[1] if port is not None: - if port.isdigit() and port.isascii(): - port = int(port) - else: - raise ValueError(f"Port could not be cast to integer value as {port!r}") - if not (0 <= port <= 65535): - raise ValueError("Port out of range 0-65535") + port = _validate_port(port) return port __class_getitem__ = classmethod(types.GenericAlias) @@ -231,16 +247,7 @@ def _hostinfo(self): netloc = self.netloc if netloc is None: return None, None - _, _, hostinfo = netloc.rpartition('@') - _, have_open_br, bracketed = hostinfo.partition('[') - if have_open_br: - hostname, _, port = bracketed.partition(']') - _, _, port = port.partition(':') - else: - hostname, _, port = hostinfo.partition(':') - if not port: - port = None - return hostname, port + return _parse_hostinfo(netloc) class _NetlocResultMixinBytes(_NetlocResultMixinBase, _ResultMixinBytes): @@ -506,9 +513,7 @@ def _splitnetloc(url, start=0): delim = min(delim, wdelim) # use earliest delim position return url[start:delim], url[delim:] # return (domain, rest) -def _checknetloc(netloc): - if not netloc or netloc.isascii(): - return +def _checknetloc_nfkc(netloc): # looking for characters like \u2100 that expand to 'a/c' # IDNA uses NFKC equivalence, so normalize for this check import unicodedata @@ -524,6 +529,16 @@ def _checknetloc(netloc): raise ValueError("netloc '" + netloc + "' contains invalid " + "characters under NFKC normalization") +def _checknetloc(netloc): + if not netloc: + return + if not netloc.isascii(): + _checknetloc_nfkc(netloc) + + _, port = _parse_hostinfo(netloc) + if port is not None: + _validate_port(port) + def _check_bracketed_netloc(netloc): # Note that this function must mirror the splitting # done in NetlocResultMixins._hostinfo(). diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index bcec9f795b0c9ff..da634a205c195e4 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -101,7 +101,8 @@ from urllib.parse import ( urlparse, urlsplit, urljoin, unwrap, quote, unquote, _splittype, _splithost, _splitport, _splituser, _splitpasswd, - _splitattr, _splitvalue, _splittag, + _splitattr, _splitvalue, _splittag, _splitnetloc, + _WHATWG_C0_CONTROL_OR_SPACE, _UNSAFE_URL_BYTES_TO_REMOVE, unquote_to_bytes, urlunparse) from urllib.response import addinfourl, addclosehook @@ -1753,7 +1754,22 @@ def url2pathname(url, *, require_scheme=False, resolve_host=False): """ if not require_scheme: url = 'file:' + url - scheme, authority, url = urlsplit(url)[:3] # Discard query and fragment. + # Match urlsplit()'s own control-character handling up front, so the + # except branch below can reuse this same cleaned-up string. + url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) + for b in _UNSAFE_URL_BYTES_TO_REMOVE: + url = url.replace(b, '') + try: + scheme, authority, url = urlsplit(url)[:3] # Discard query and fragment. + except ValueError: + # The authority here is not a real host[:port] pair -- it may + # hold a Windows drive letter (e.g. 'C:\\') -- so urlsplit()'s + # eager port check doesn't apply; split without it instead. + scheme, _, url = url.partition(':') + scheme = scheme.lower() + authority = '' + if url[:2] == '//': + authority, url = _splitnetloc(url, 2) if scheme != 'file': raise URLError("URL is missing a 'file:' scheme") if os.name == 'nt': diff --git a/Misc/NEWS.d/next/Library/2026-10-11-13-08-07.gh-issue-88037.CzKcxh.rst b/Misc/NEWS.d/next/Library/2026-10-11-13-08-07.gh-issue-88037.CzKcxh.rst new file mode 100644 index 000000000000000..697b44ba2985a0d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-11-13-08-07.gh-issue-88037.CzKcxh.rst @@ -0,0 +1,3 @@ +:func:`urllib.parse.urlsplit` and :func:`urllib.parse.urlparse` now raise +:exc:`ValueError` for an invalid port when the URL is parsed, instead of +when the :attr:`!port` attribute is read. Patch by Miguel Brito and Senthil Kumaran. From 72718034979bf81a39817b76bfed53838982bddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ahmed=20Semih=20=C3=96zmekik?= Date: Sun, 8 Nov 2026 19:13:51 +0300 Subject: [PATCH 039/229] gh-160924: Don't run the build Python with RUNSHARED when cross-compiling (#161321) When cross-compiling, `PYTHON_FOR_BUILD` uses the interpreter specified by `--with-build-python`. With `--enable-shared`, running it with `RUNSHARED` adds the target build directory to `LD_LIBRARY_PATH`. A shared build Python can then load the target's `libpython` instead of its own, failing if the target uses an incompatible libc or architecture. Introduce `RUNSHARED_FOR_BUILD`, set to `$(RUNSHARED)` for native builds and empty for cross builds, and use it for `PYTHON_FOR_BUILD` invocations during building and installation. Leave `RUNSHARED` unchanged for PGO profiling (gh-141926) and `TESTPYTHON`, which uses `HOSTRUNNER` to launch the target interpreter. --- Makefile.pre.in | 23 ++++++++++--------- ...-11-04-15-00-00.gh-issue-160924.WQgVVh.rst | 3 +++ configure | 6 +++++ configure.ac | 5 ++++ 4 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-11-04-15-00-00.gh-issue-160924.WQgVVh.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index 3955afae436e9c5..1d109dac77fa906 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -242,6 +242,7 @@ LIPO_INTEL64_FLAGS=@LIPO_INTEL64_FLAGS@ # Environment to run shared python without installed libraries RUNSHARED= @RUNSHARED@ +RUNSHARED_FOR_BUILD= @RUNSHARED_FOR_BUILD@ # ensurepip options ENSUREPIP= @ENSUREPIP@ @@ -1013,7 +1014,7 @@ $(BUILDPYTHON): Programs/python.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_DEPS) $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform + $(RUNSHARED_FOR_BUILD) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform # Create build directory and generate the sysconfig build-time data there. # pybuilddir.txt contains the name of the build dir and is used for @@ -1024,7 +1025,7 @@ platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt # or removed in case of failure. pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS) @echo "none" > ./pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -X pathconfig_warnings=0 -m sysconfig --generate-posix-vars ;\ + $(RUNSHARED_FOR_BUILD) $(PYTHON_FOR_BUILD) -S -X pathconfig_warnings=0 -m sysconfig --generate-posix-vars ;\ if test $$? -ne 0 ; then \ echo "generate-posix-vars failed" ; \ rm -f ./pybuilddir.txt ; \ @@ -1032,7 +1033,7 @@ pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS) fi $(BUILD_DETAILS): pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py `cat pybuilddir.txt`/$(BUILD_DETAILS) + $(RUNSHARED_FOR_BUILD) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py `cat pybuilddir.txt`/$(BUILD_DETAILS) # Build static library $(LIBRARY): $(LIBRARY_OBJS) @@ -1604,11 +1605,11 @@ sharedmods: $(SHAREDMODS) pybuilddir.txt .PHONY: checksharedmods checksharedmods: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) @if [ -n "@MISSING_STDLIB_CONFIG@" ]; then \ - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py --generate-missing-stdlib-info --with-missing-stdlib-config="@MISSING_STDLIB_CONFIG@"; \ + $(RUNSHARED_FOR_BUILD) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py --generate-missing-stdlib-info --with-missing-stdlib-config="@MISSING_STDLIB_CONFIG@"; \ else \ - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py --generate-missing-stdlib-info; \ + $(RUNSHARED_FOR_BUILD) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py --generate-missing-stdlib-info; \ fi - @$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py + @$(RUNSHARED_FOR_BUILD) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py .PHONY: rundsymutil rundsymutil: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) @@ -2437,7 +2438,7 @@ install: @FRAMEWORKINSTALLFIRST@ @INSTALLTARGETS@ @FRAMEWORKINSTALLLAST@ upgrade) ensurepip="--upgrade" ;; \ install|*) ensurepip="" ;; \ esac; \ - $(RUNSHARED) $(PYTHON_FOR_BUILD) -m ensurepip \ + $(RUNSHARED_FOR_BUILD) $(PYTHON_FOR_BUILD) -m ensurepip \ $$ensurepip --root=$(DESTDIR)/ ; \ fi @@ -2448,7 +2449,7 @@ altinstall: commoninstall upgrade) ensurepip="--altinstall --upgrade" ;; \ install|*) ensurepip="--altinstall" ;; \ esac; \ - $(RUNSHARED) $(PYTHON_FOR_BUILD) -m ensurepip \ + $(RUNSHARED_FOR_BUILD) $(PYTHON_FOR_BUILD) -m ensurepip \ $$ensurepip --root=$(DESTDIR)/ ; \ fi @@ -2914,12 +2915,12 @@ libinstall: all $(srcdir)/Modules/xxmodule.c patch --force --reject-file "$(abs_builddir)/app-store-compliance.rej" --strip 2 --directory "$(DESTDIR)$(LIBDEST)" --input "$(abs_srcdir)/$(APP_STORE_COMPLIANCE_PATCH)" || true ; \ fi @ # Build PYC files for the 3 optimization levels (0, 1, 2) - -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ + -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED_FOR_BUILD) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ -o 0 -o 1 -o 2 $(COMPILEALL_OPTS) -d $(LIBDEST) -f \ -x 'bad_coding|badsyntax|site-packages' \ $(DESTDIR)$(LIBDEST) - -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ + -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED_FOR_BUILD) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ -o 0 -o 1 -o 2 $(COMPILEALL_OPTS) -d $(LIBDEST)/site-packages -f \ -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages @@ -3139,7 +3140,7 @@ frameworkinstallunversionedstructure: $(LDLIBRARY) rm -rf $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/include; \ fi $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR) - sed 's/%VERSION%/'"`$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import platform; print(platform.python_version())'`"'/g' < $(RESSRCDIR)/Info.plist > $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Info.plist + sed 's/%VERSION%/'"`$(RUNSHARED_FOR_BUILD) $(PYTHON_FOR_BUILD) -c 'import platform; print(platform.python_version())'`"'/g' < $(RESSRCDIR)/Info.plist > $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Info.plist $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(LIBDIR) $(LN) -fs "../$(LDLIBRARY)" "$(DESTDIR)$(prefix)/lib/libpython$(LDVERSION).dylib" diff --git a/Misc/NEWS.d/next/Build/2026-11-04-15-00-00.gh-issue-160924.WQgVVh.rst b/Misc/NEWS.d/next/Build/2026-11-04-15-00-00.gh-issue-160924.WQgVVh.rst new file mode 100644 index 000000000000000..2817321dc7e4d1b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-11-04-15-00-00.gh-issue-160924.WQgVVh.rst @@ -0,0 +1,3 @@ +Fix cross-compilation failures with ``--enable-shared`` caused by the +build Python loading the target's ``libpython`` through the shared-library +search path (such as musl from glibc). diff --git a/configure b/configure index fca5669c3756b4d..65d58fd1e810e23 100755 --- a/configure +++ b/configure @@ -1035,6 +1035,7 @@ CONFIG_ARGS SOVERSION VERSION PYTHON_FOR_REGEN +RUNSHARED_FOR_BUILD PYTHON_FOR_BUILD_DEPS FREEZE_MODULE_DEPS FREEZE_MODULE @@ -3863,6 +3864,9 @@ then : FREEZE_MODULE='$(FREEZE_MODULE_BOOTSTRAP)' FREEZE_MODULE_DEPS='$(FREEZE_MODULE_BOOTSTRAP_DEPS)' PYTHON_FOR_BUILD_DEPS='' + # The build Python is a separate interpreter. RUNSHARED would make it + # load the in-tree libpython, which may need a different libc. + RUNSHARED_FOR_BUILD='' else case e in #( e) @@ -3871,6 +3875,7 @@ else case e in #( FREEZE_MODULE='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' FREEZE_MODULE_DEPS="_bootstrap_python \$(srcdir)/Programs/_freeze_module.py" PYTHON_FOR_BUILD_DEPS='$(BUILDPYTHON)' + RUNSHARED_FOR_BUILD='$(RUNSHARED)' ;; esac @@ -3881,6 +3886,7 @@ fi + for ac_prog in python$PACKAGE_VERSION python3.16 python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. diff --git a/configure.ac b/configure.ac index 753f997a20da4a2..a041f0115bd7ab3 100644 --- a/configure.ac +++ b/configure.ac @@ -194,6 +194,9 @@ AS_VAR_IF([cross_compiling], [yes], FREEZE_MODULE='$(FREEZE_MODULE_BOOTSTRAP)' FREEZE_MODULE_DEPS='$(FREEZE_MODULE_BOOTSTRAP_DEPS)' PYTHON_FOR_BUILD_DEPS='' + # The build Python is a separate interpreter. RUNSHARED would make it + # load the in-tree libpython, which may need a different libc. + RUNSHARED_FOR_BUILD='' ], [ dnl internal build tools also depend on Programs/_freeze_module and _bootstrap_python. @@ -202,6 +205,7 @@ AS_VAR_IF([cross_compiling], [yes], FREEZE_MODULE='$(PYTHON_FOR_FREEZE) $(srcdir)/Programs/_freeze_module.py' FREEZE_MODULE_DEPS="_bootstrap_python \$(srcdir)/Programs/_freeze_module.py" PYTHON_FOR_BUILD_DEPS='$(BUILDPYTHON)' + RUNSHARED_FOR_BUILD='$(RUNSHARED)' ] ) AC_SUBST([FREEZE_MODULE_BOOTSTRAP]) @@ -209,6 +213,7 @@ AC_SUBST([FREEZE_MODULE_BOOTSTRAP_DEPS]) AC_SUBST([FREEZE_MODULE]) AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) +AC_SUBST([RUNSHARED_FOR_BUILD]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], [python$PACKAGE_VERSION python3.16 python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python], From a61b19b2bc3b5252163c88d78815fb7903149734 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 8 Nov 2026 18:09:03 +0100 Subject: [PATCH 040/229] gh-161259: Fix statistics._sqrtprod() for subnormal products (gh-161260) --- Lib/statistics.py | 6 +- Lib/test/test_statistics.py | 60 +++++++++++++++++++ ...-11-03-18-56-28.gh-issue-161259.iyZz6y.rst | 3 + 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-03-18-56-28.gh-issue-161259.iyZz6y.rst diff --git a/Lib/statistics.py b/Lib/statistics.py index eee3e6eb94e990b..aaf3799716d8fb6 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -1776,9 +1776,11 @@ def _sqrtprod(x: float, y: float) -> float: return _sqrtprod(scale * x, scale * y) / scale return h - if not h: + subnormal_cutoff = 2.0 ** -485 # sqrt(sys.float_info.min / sys.float_info.epsilon) + if h < subnormal_cutoff: + # x * y is subnormal or underflowed, or the correction term + # below would be. Rescale non-zero inputs. if x and y: - # Non-zero inputs underflowed, so scale up, and recompute. # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) scale = 2.0 ** 537 return _sqrtprod(scale * x, scale * y) / scale diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 700c5ac304f7179..266d92828fe154a 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2790,6 +2790,66 @@ def test_sqrtprod_helper_function_fundamentals(self): self.assertEqual(actual, expected) self.assertEqual(sign(actual), sign(expected)) + @requires_IEEE_754 + def test_sqrtprod_helper_function_subnormal_products(self): + # gh-161259: accuracy was lost when the product or the correction + # term was subnormal. The reference scales the inputs by an exact + # power of two so that the product is a normal float. + scale = 2.0 ** 600 + for x, y in [ + (2e-200, 2e-124), + (1e-160, 3e-150), + (3e-170, 7e-140), + (1e-160, 1e-135), + ]: + expected = math.sqrt((scale * x) * (scale * y)) / scale + actual = statistics._sqrtprod(x, y) + with self.subTest(x=x, y=y, expected=expected, actual=actual): + self.assertAlmostEqual(actual / expected, 1.0, delta=2**-50) + + @requires_IEEE_754 + def test_sqrtprod_helper_function_subnormal_boundary(self): + # sqrt(3 * 7) * 2**e straddles the 2**-485 cutoff where the + # correction term can become subnormal. + scale = 2.0 ** 600 + for e in (-488, -487): + x = 3.0 * 2.0 ** e + y = 7.0 * 2.0 ** e + expected = math.sqrt((scale * x) * (scale * y)) / scale + actual = statistics._sqrtprod(x, y) + with self.subTest(e=e, expected=expected, actual=actual): + self.assertAlmostEqual(actual / expected, 1.0, delta=2**-50) + + cutoff = 2.0 ** -485 + self.assertEqual(statistics._sqrtprod(cutoff, cutoff), cutoff) + + @requires_IEEE_754 + def test_correlation_tiny_inputs(self): + # gh-161259: correlation() is scale-invariant, even when the + # intermediate sums of squares are tiny. + x = [1e-100, 2e-100, 3e-100] + y = [1e-62, 2e-62, 3e-62] + self.assertAlmostEqual(statistics.correlation(x, y), 1.0) + + @requires_IEEE_754 + def test_correlation_asymmetric_scales(self): + for a, b in [ + (2.0 ** 50, 2.0 ** -537), + (1.3 * 2.0 ** -260, 2.7 * 2.0 ** -270), + ]: + x = [-a, 0.0, a] + y = [0.0, -b, b] + with self.subTest(a=a, b=b): + self.assertAlmostEqual(statistics.correlation(x, y), 0.5, + delta=2**-50) + + @requires_IEEE_754 + def test_correlation_tiny_products_max_recursion(self): + # Make both sums of squares round to the smallest subnormal. + a = 2.0 ** -537 + x = [0.0, 0.0, 0.0, a] + self.assertEqual(statistics.correlation(x, x), 1.0) + @requires_IEEE_754 @skip_if_double_rounding @support.cpython_only # Allow for a weaker sumprod() implementation diff --git a/Misc/NEWS.d/next/Library/2026-11-03-18-56-28.gh-issue-161259.iyZz6y.rst b/Misc/NEWS.d/next/Library/2026-11-03-18-56-28.gh-issue-161259.iyZz6y.rst new file mode 100644 index 000000000000000..400f6e5b716941c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-03-18-56-28.gh-issue-161259.iyZz6y.rst @@ -0,0 +1,3 @@ +Improved the accuracy of :func:`statistics.correlation` for very small +inputs, where the product of the sums of squares is subnormal rather +than zero. From e973cbc19db8a4709a613e5a3e34a1348efd9db0 Mon Sep 17 00:00:00 2001 From: Alisher Muhammadaliyev Date: Sun, 8 Nov 2026 22:29:08 +0500 Subject: [PATCH 041/229] gh-161077: Replace "truthy"/"falsy" with "true"/"false" in the docs (gh-161436) --- Doc/builtins/functions.rst | 2 +- Doc/library/smtplib.rst | 4 ++-- Doc/library/tkinter.rst | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/builtins/functions.rst b/Doc/builtins/functions.rst index efd6bb9e28c89ae..a1db1544c7bf4ed 100644 --- a/Doc/builtins/functions.rst +++ b/Doc/builtins/functions.rst @@ -1950,7 +1950,7 @@ are always available. They are listed here in alphabetical order. >>> MISSING - Sentinel objects are truthy and compare equal only to themselves. They are + Sentinel objects are true and compare equal only to themselves. They are intended to be compared with the :keyword:`is` operator. ``sentinel`` does not support subclassing. diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 5c97199bc453e85..fe58c7fd2414fe0 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -24,7 +24,7 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). An :class:`SMTP` instance encapsulates an SMTP connection. It has methods that support a full repertoire of SMTP and ESMTP operations. - If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with + If the host parameter is set to a true value, :meth:`SMTP.connect` is called with host and port automatically when the object is created; otherwise, :meth:`!connect` must be called manually. @@ -88,7 +88,7 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). required from the beginning of the connection and using :meth:`SMTP.starttls` is not appropriate. - If the host parameter is set to a truthy value, :meth:`SMTP.connect` is called with host + If the host parameter is set to a true value, :meth:`SMTP.connect` is called with host and port automatically when the object is created; otherwise, :meth:`!SMTP.connect` must be called manually. diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 653a91c5cbedba1..8ed3428610fef48 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -6820,13 +6820,13 @@ and :mod:`!tkinter.constants` namespaces. YES ON - Truthy values, all equal to the integer ``1``. + Values that are true, all equal to the integer ``1``. .. data:: FALSE NO OFF - Falsy values, all equal to the integer ``0``. + Values that are false, all equal to the integer ``0``. .. data:: N S From c8eabcc7c9c0611af17949b3e1c1da79ed0cf4ac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 8 Nov 2026 18:45:48 +0100 Subject: [PATCH 042/229] gh-161625: Fix compiler warning in _uuid (#161628) Fix the warning on Windows when building Python in 32-bit: Modules\_uuidmodule.c(302,50): warning C4244: 'initializing': conversion from 'uint64_t' to 'Py_ssize_t', possible loss of data --- Modules/_uuidmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index e957071a4ecef95..04bce1bd18901b8 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -187,7 +187,7 @@ typedef struct { // We overfetch entropy to speed up successive uuid generations; // this enables 10x peformance boost. uint8_t random_buf[RANDOM_BUF_SIZE]; - uint64_t random_idx; + Py_ssize_t random_idx; uint64_t random_last_pid; // UUID v7 state From dcf90877e47b15d3e364b7b1855c44f1832fa253 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 8 Nov 2026 20:04:21 +0100 Subject: [PATCH 043/229] gh-161625: Fix compiler warnings in _remote_debugging (#161626) Fix compiler warnings when Python is built in 32-bit mode on Windows. --- Modules/_remote_debugging/binary_io_reader.c | 16 ++++++++-------- Modules/_remote_debugging/frames.c | 6 +++--- Modules/_remote_debugging/gc_stats.c | 4 ++-- Modules/_remote_debugging/object_reading.c | 2 +- Modules/_remote_debugging/threads.c | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 80627db913ea21b..a51a2e475c9e06b 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -365,7 +365,7 @@ static inline int reader_parse_string_table(BinaryReader *reader, const uint8_t *data, size_t file_size) { if (reader_validate_count("string", reader->strings_count, - file_size - reader->string_table_offset, + file_size - (size_t)reader->string_table_offset, MIN_STRING_ENTRY_SIZE) < 0) { return -1; } @@ -376,7 +376,7 @@ reader_parse_string_table(BinaryReader *reader, const uint8_t *data, size_t file return -1; } - size_t offset = reader->string_table_offset; + size_t offset = (size_t)reader->string_table_offset; for (uint32_t i = 0; i < reader->strings_count; i++) { size_t prev_offset = offset; uint32_t str_len = decode_varint_u32(data, &offset, file_size); @@ -411,7 +411,7 @@ reader_parse_frame_table(BinaryReader *reader, const uint8_t *data, size_t file_ #endif if (reader_validate_count("frame", reader->frames_count, - file_size - reader->frame_table_offset, + file_size - (size_t)reader->frame_table_offset, MIN_FRAME_ENTRY_SIZE) < 0) { return -1; } @@ -423,7 +423,7 @@ reader_parse_frame_table(BinaryReader *reader, const uint8_t *data, size_t file_ return -1; } - size_t offset = reader->frame_table_offset; + size_t offset = (size_t)reader->frame_table_offset; for (uint32_t i = 0; i < reader->frames_count; i++) { FrameEntry *frame = &reader->frames[i]; size_t prev_offset; @@ -694,7 +694,7 @@ binary_reader_open(PyObject *path) #endif } else { reader->sample_data = data + FILE_HEADER_PLACEHOLDER_SIZE; - reader->sample_data_size = reader->string_table_offset - FILE_HEADER_PLACEHOLDER_SIZE; + reader->sample_data_size = (size_t)reader->string_table_offset - FILE_HEADER_PLACEHOLDER_SIZE; } /* Parse string and frame tables */ @@ -871,7 +871,7 @@ decode_stack_suffix(ReaderThreadState *ts, const uint8_t *data, return -1; } } - ts->current_stack_depth = final_depth; + ts->current_stack_depth = (size_t)final_depth; return 0; } @@ -916,7 +916,7 @@ decode_stack_pop_push(ReaderThreadState *ts, const uint8_t *data, return -1; } } - ts->current_stack_depth = final_depth; + ts->current_stack_depth = (size_t)final_depth; return 0; } @@ -1262,7 +1262,7 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre reader->stats.total_samples += count; /* Progress callback after batch */ - if (replayed % PROGRESS_CALLBACK_INTERVAL < count) { + if ((uint64_t)replayed % PROGRESS_CALLBACK_INTERVAL < (uint64_t)count) { if (invoke_progress_callback(progress_callback, replayed, reader->sample_count) < 0) { return -1; } diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index f5b9ed72f1f98e9..19a9b5369148a98 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -465,7 +465,7 @@ clear_last_profiled_frames(RemoteUnwinderObject *unwinder) uintptr_t tstate_addr; if (_Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - current_interp + unwinder->debug_offsets.interpreter_state.threads_head, + current_interp + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_head, sizeof(void*), &tstate_addr) < 0) { // Non-fatal: just skip clearing @@ -486,7 +486,7 @@ clear_last_profiled_frames(RemoteUnwinderObject *unwinder) // Move to next thread if (_Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - tstate_addr + unwinder->debug_offsets.thread_state.next, + tstate_addr + (uintptr_t)unwinder->debug_offsets.thread_state.next, sizeof(void*), &tstate_addr) < 0) { PyErr_Clear(); @@ -497,7 +497,7 @@ clear_last_profiled_frames(RemoteUnwinderObject *unwinder) // Move to next interpreter if (_Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - current_interp + unwinder->debug_offsets.interpreter_state.next, + current_interp + (uintptr_t)unwinder->debug_offsets.interpreter_state.next, sizeof(void*), ¤t_interp) < 0) { PyErr_Clear(); diff --git a/Modules/_remote_debugging/gc_stats.c b/Modules/_remote_debugging/gc_stats.c index 23fa879b5032837..4dc77803a0a9f3c 100644 --- a/Modules/_remote_debugging/gc_stats.c +++ b/Modules/_remote_debugging/gc_stats.c @@ -88,8 +88,8 @@ get_gc_stats_from_interpreter_state(RuntimeOffsets *offsets, uintptr_t gc_stats_addr = 0; uintptr_t gc_stats_pointer_address = interpreter_state_addr - + offsets->debug_offsets.interpreter_state.gc - + offsets->debug_offsets.gc.generation_stats; + + (uintptr_t)offsets->debug_offsets.interpreter_state.gc + + (uintptr_t)offsets->debug_offsets.gc.generation_stats; if (_Py_RemoteDebug_ReadRemoteMemory(&offsets->handle, gc_stats_pointer_address, sizeof(gc_stats_addr), diff --git a/Modules/_remote_debugging/object_reading.c b/Modules/_remote_debugging/object_reading.c index e449b446a1fe544..6f138f3de3503b6 100644 --- a/Modules/_remote_debugging/object_reading.c +++ b/Modules/_remote_debugging/object_reading.c @@ -279,7 +279,7 @@ read_py_long( } // Calculate how many digits fit inline in our local buffer - Py_ssize_t ob_digit_offset = unwinder->debug_offsets.long_object.ob_digit; + Py_ssize_t ob_digit_offset = (Py_ssize_t)unwinder->debug_offsets.long_object.ob_digit; Py_ssize_t inline_digits_space = SIZEOF_LONG_OBJ - ob_digit_offset; Py_ssize_t max_inline_digits = inline_digits_space / (Py_ssize_t)sizeof(digit); diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index 120065022ddc8ee..a3bd6c9edfb58eb 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -447,9 +447,9 @@ unwind_stack_for_thread( unwinder->debug_offsets.thread_state.current_exception + sizeof(uintptr_t)); uintptr_t exc_state_addr = - *current_tstate + unwinder->debug_offsets.thread_state.exc_state; + *current_tstate + (uintptr_t)unwinder->debug_offsets.thread_state.exc_state; uintptr_t exc_value_offset = - unwinder->debug_offsets.err_stackitem.exc_value; + (uintptr_t)unwinder->debug_offsets.err_stackitem.exc_value; uintptr_t previous_item_offset = exc_value_offset + sizeof(uintptr_t); From 517b0ee6767cd1156fed2d6615cc41d63afd2fb4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 8 Nov 2026 20:37:37 +0100 Subject: [PATCH 044/229] gh-125174: Remove internal macro _Py_IsImmortalLoose() (#161632) We don't provide backward compatibility in the internal C API. --- Include/internal/pycore_object.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index a27e13f9c460bfa..30333e14655a60e 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -27,9 +27,6 @@ extern "C" { // `ob_ref_shared` are used for flags. #define _Py_REF_DEFERRED (PY_SSIZE_T_MAX / 8) -/* For backwards compatibility -- Do not use this */ -#define _Py_IsImmortalLoose(op) _Py_IsImmortal - /* Check if an object is consistent. For example, ensure that the reference counter is greater than or equal to 1, and ensure that ob_type is not NULL. From eba2a21ea6928df515e400b9e5d61ebd23ec512b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 8 Nov 2026 20:37:57 +0100 Subject: [PATCH 045/229] gh-161625: Fix compiler warnings in 32-bit mode (#161631) Fix warnigs: * Include/internal/pycore_global_objects_fini.h:45:51: warning: comparison of integer expressions of different signedness: 'digit' {aka 'unsigned int'} and 'long int' [-Wsign-compare] * Modules/_testlimitedcapi/heaptype_relative.c:71:28: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast] * Objects/moduleobject.c:412:38: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast] * Objects/moduleobject.c:474:44: warning: cast from pointer to integer of different size [-Wpointer-to-int-cast] * Python/getopt.c:163:49: warning: format '%lc' expects argument of type 'wint_t', but argument 3 has type 'wchar_t' {aka 'long int'} [-Wformat=] * Python/getopt.c:177:55: warning: format '%lc' expects argument of type 'wint_t', but argument 3 has type 'wchar_t' {aka 'long int'} [-Wformat=] In getopt.c, pass wint_t instead of wchar_t to printf("%lc"). --- Include/internal/pycore_global_objects_fini.h | 2 +- Modules/_testlimitedcapi/heaptype_relative.c | 2 +- Objects/moduleobject.c | 4 ++-- Python/getopt.c | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini.h b/Include/internal/pycore_global_objects_fini.h index ee3a593db37b69e..41bc2c5d8fb1405 100644 --- a/Include/internal/pycore_global_objects_fini.h +++ b/Include/internal/pycore_global_objects_fini.h @@ -42,7 +42,7 @@ _PyStaticObject_CheckLongSingleton(PyObject *obj, long value, int is_bool) lv_tag |= IMMORTALITY_BIT_MASK; } _PyObject_ASSERT(obj, long_value->lv_tag == lv_tag); - _PyObject_ASSERT(obj, long_value->ob_digit[0] == Py_ABS(value)); + _PyObject_ASSERT(obj, long_value->ob_digit[0] == (digit)Py_ABS(value)); } diff --git a/Modules/_testlimitedcapi/heaptype_relative.c b/Modules/_testlimitedcapi/heaptype_relative.c index a5e15026f028ca7..f21ca9250ed2666 100644 --- a/Modules/_testlimitedcapi/heaptype_relative.c +++ b/Modules/_testlimitedcapi/heaptype_relative.c @@ -68,7 +68,7 @@ make_sized_heaptypes(PyObject *module, PyObject *args) } result = Py_BuildValue("OOOKnn", base, sub, instance, - (unsigned long long)data_ptr, + (unsigned long long)(uintptr_t)data_ptr, (Py_ssize_t)(data_ptr - (char*)instance), data_size); finally: diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 7c842f3ed8b1965..568711149a8aeb6 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -409,7 +409,7 @@ module_from_slots_and_spec( createfunc_t create = NULL; PyObject *nameobj; PyObject *m = NULL; - uint64_t multiple_interpreters = (uint64_t)Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; + uint64_t multiple_interpreters = (uint64_t)(uintptr_t)Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; bool requires_gil = true; const char *name; int ret; @@ -471,7 +471,7 @@ module_from_slots_and_spec( case Py_mod_gil: { uint64_t val = it.current.sl_uint64; - requires_gil = (val != (uint64_t)Py_MOD_GIL_NOT_USED); + requires_gil = (val != (uint64_t)(uintptr_t)Py_MOD_GIL_NOT_USED); } break; case Py_mod_abi: diff --git a/Python/getopt.c b/Python/getopt.c index f6a3ef960a4021b..c64d6e65c2740ad 100644 --- a/Python/getopt.c +++ b/Python/getopt.c @@ -160,7 +160,7 @@ _PyOS_GetOpt(struct _PyOS_GetOpt *getopt) wchar_t *ptr = wcschr(SHORT_OPTS, option); if (ptr == NULL) { if (error) { - fprintf(stderr, "Invalid option '-%lc'\n", option); + fprintf(stderr, "Invalid option '-%lc'\n", (wint_t)option); } return '_'; } @@ -174,7 +174,7 @@ _PyOS_GetOpt(struct _PyOS_GetOpt *getopt) if (getopt->index >= argc) { if (error) { fprintf(stderr, - "Argument expected for the -%lc option\n", option); + "Argument expected for the -%lc option\n", (wint_t)option); } return '_'; } From 61fdee34eed9ea6f03190a5ddd7bb0c7ccc87be9 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sun, 8 Nov 2026 12:22:40 -0800 Subject: [PATCH 046/229] gh-87613: Use AC VC for io.BytesIO (#160855) BytesIO is commonly constructed in the test suite and gets significantly faster to construct. Argument error messages are unchanged. pyperf microbenchmark (PGO+LTO, run serially): | Benchmark | base | bytesio | |------------------------------|:-------:|:---------------------:| | BytesIO(bytes) | 54.2 ns | 27.0 ns: 2.01x faster | | BytesIO(bytearray) | 76.7 ns | 38.6 ns: 1.99x faster | | BytesIO(initial_bytes=bytes) | 56.5 ns | 34.8 ns: 1.63x faster | | BytesIO() | 40.9 ns | 25.4 ns: 1.61x faster | | Geometric mean | (ref) | 1.48x faster | Benchmark hidden because not significant (2): `BytesIO.__new__`, subclass(bytes) Benchmark code: ```python """BytesIO() construction: AC @vectorcall vs tp_new + tp_init.""" import pyperf runner = pyperf.Runner() setup = "from io import BytesIO; b = b'abcdefgh'" runner.timeit('BytesIO()', 'BytesIO()', setup) runner.timeit('BytesIO(bytes)', 'BytesIO(b)', setup) runner.timeit('BytesIO(bytearray)', 'BytesIO(x)', setup + '; x = bytearray(b)') runner.timeit('BytesIO(initial_bytes=bytes)', 'BytesIO(initial_bytes=b)', setup) runner.timeit('subclass(bytes)', 'S(b)', setup + '\nclass S(BytesIO): pass') runner.timeit('BytesIO.__new__', 'new(BytesIO)', setup + '; new = BytesIO.__new__') ``` Co-authored-by: Claude Opus 5.5 --- Modules/_io/bytesio.c | 4 +- Modules/_io/clinic/bytesio.c.h | 73 +++++++++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 2571da4538363f1..9bec70acebb3598 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1118,6 +1118,7 @@ bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } /*[clinic input] +@vectorcall @critical_section _io.BytesIO.__init__ initial_bytes as initvalue: object(c_default="NULL") = b'' @@ -1127,7 +1128,7 @@ Buffered I/O implementation using an in-memory bytes buffer. static int _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) -/*[clinic end generated code: output=65c0c51e24c5b621 input=3da5a74ee4c4f1ac]*/ +/*[clinic end generated code: output=65c0c51e24c5b621 input=a6dc38082204df28]*/ { if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_BufferError, @@ -1256,6 +1257,7 @@ static PyType_Slot bytesio_slots[] = { {Py_tp_getset, bytesio_getsetlist}, {Py_tp_init, _io_BytesIO___init__}, {Py_tp_new, bytesio_new}, + {Py_tp_vectorcall, _io_BytesIO_vectorcall}, {0, NULL}, }; diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index b2305e35ca81e75..e48d71397367daf 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -4,11 +4,11 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() +#include "pycore_runtime.h" // _Py_SINGLETON() PyDoc_STRVAR(_io_BytesIO_closed__doc__, "True if the file is closed."); @@ -716,7 +716,8 @@ static int _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue); static int -_io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) +_io_BytesIO___init___helper(PyObject *self, PyObject *const *args, + Py_ssize_t nargs, Py_ssize_t nkw, PyObject *kwargs, PyObject *kwnames) { int return_value = -1; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -748,11 +749,10 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) #undef KWTUPLE PyObject *argsbuf[1]; PyObject * const *fastargs; - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; + Py_ssize_t noptargs = nargs + nkw - 0; PyObject *initvalue = NULL; - fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + fastargs = _PyArg_UnpackKeywords(args, nargs, kwargs, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; @@ -770,6 +770,67 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } +static int +_io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + return _io_BytesIO___init___helper(self, _PyTuple_CAST(args)->ob_item, + PyTuple_GET_SIZE(args), + kwargs ? PyDict_GET_SIZE(kwargs) : 0, + kwargs, NULL); +} + +static PyObject * +_io_BytesIO_vectorcall(PyObject *type, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + PyObject *return_value = NULL; + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *self; + int _result; + PyObject *initvalue = NULL; + + assert(_PyType_CAST(type)->tp_init == _io_BytesIO___init__); + /* Make sure the type object is immutable: the generated + * vectorcall doesn't deal e.g. with users reassigning __init__. */ + assert(PyType_HasFeature(_PyType_CAST(type), Py_TPFLAGS_IMMUTABLETYPE)); + if (kwnames != NULL || nargs > 1) { + self = _PyType_CAST(type)->tp_new(_PyType_CAST(type), + (PyObject *)&_Py_SINGLETON(tuple_empty), NULL); + if (self == NULL) { + return NULL; + } + _result = _io_BytesIO___init___helper(self, args, nargs, + kwnames ? PyTuple_GET_SIZE(kwnames) : 0, + NULL, kwnames); + if (_result != 0) { + Py_DECREF(self); + return NULL; + } + return self; + } + if (nargs < 1) { + goto skip_optional; + } + initvalue = args[0]; +skip_optional: + self = _PyType_CAST(type)->tp_new(_PyType_CAST(type), + (PyObject *)&_Py_SINGLETON(tuple_empty), NULL); + if (self == NULL) { + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + _result = _io_BytesIO___init___impl((bytesio *)self, initvalue); + Py_END_CRITICAL_SECTION(); + if (_result != 0) { + Py_DECREF(self); + goto exit; + } + return_value = self; + +exit: + return return_value; +} + PyDoc_STRVAR(_io_BytesIO___sizeof____doc__, "__sizeof__($self, /)\n" "--\n" @@ -795,4 +856,4 @@ _io_BytesIO___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) } #define _IO_BYTESIO_CLOSED_GETSETDEF {"closed", (getter)_io_BytesIO_closed_get, (setter)NULL, _io_BytesIO_closed__doc__}, -/*[clinic end generated code: output=a2bc96c358d6e240 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b9d367ad8e0b44d2 input=a9049054013a1b77]*/ From 0bb9ef9695d30c9ca90a8773e40b05ff5073cc17 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Sun, 8 Nov 2026 14:37:43 -0700 Subject: [PATCH 047/229] Propagate errors from slot updates in type_setattro() (gh-161587) Handle failures of update_slot(), since it can fail in the default build due to non-str keys. --- Lib/test/test_descr.py | 22 ++++++++++++++++++++++ Objects/typeobject.c | 7 ++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index c4c4d0858ec00ba..2f5953ba096d439 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4181,6 +4181,28 @@ def test_pow_wrapper_error_messages(self): 'expected 1 or 2 arguments, got 3', int().__rpow__, 1, 2, 3) + def test_setattr_slot_update_error(self): + # An error raised while updating the slots of subclasses must be + # propagated by the attribute assignment. + class K: + def __hash__(self): + return hash('__repr__') + def __eq__(self, other): + raise ZeroDivisionError + + class A: + pass + + with self.assertWarns(RuntimeWarning): + B = type('B', (A,), {K(): 1}) + if support.Py_GIL_DISABLED: + # The slot update skips non-str keys without comparing them. + A.__repr__ = lambda self: 'A' + else: + with self.assertRaises(ZeroDivisionError): + A.__repr__ = lambda self: 'A' + self.assertEqual(repr(A()), 'A') + def test_mutable_bases(self): # Testing mutable bases... diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9833c338679d33f..ee8c2090e6397d7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6671,11 +6671,11 @@ type_update_dict(PyTypeObject *type, PyDictObject *dict, PyObject *name, return 0; } -static void +static int update_slot_after_setattr(PyTypeObject *type, PyObject *name) { types_stop_world(); - update_slot(type, name); + int res = update_slot(type, name); types_start_world(); #ifdef Py_GIL_DISABLED // Invalidate again, after the slots are updated and while TYPE_LOCK is @@ -6686,6 +6686,7 @@ update_slot_after_setattr(PyTypeObject *type, PyObject *name) ASSERT_TYPE_LOCK_HELD(); _PyType_Modified_Unlocked(type); #endif + return res; } static int @@ -6774,7 +6775,7 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) if (res == 0) { if (is_dunder_name(name) && has_slotdef(name)) { // The name corresponds to a type slot. - update_slot_after_setattr(type, name); + res = update_slot_after_setattr(type, name); } } END_TYPE_DICT_LOCK(); From c07d1f4f32ed37d86110fb9d472cc59b90c11eb3 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sun, 8 Nov 2026 16:28:35 -0800 Subject: [PATCH 048/229] gh-145177: Update to Emscripten 7.0.0 (#161527) Bump default Emscripten version to 7.0.0. --- Platforms/emscripten/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Platforms/emscripten/config.toml b/Platforms/emscripten/config.toml index faaa72e1d415f5a..39835844bbb365f 100644 --- a/Platforms/emscripten/config.toml +++ b/Platforms/emscripten/config.toml @@ -1,7 +1,7 @@ # Any data that can vary between Python versions is to be kept in this file. # This allows for blanket copying of the Emscripten build code between supported # Python versions. -emscripten-version = "6.0.13" +emscripten-version = "7.0.0" node-version = "24" test-args = [ "-m", "test", From e1a4c150e7416ec89df0be6526741d3aac93733e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 8 Nov 2026 19:29:31 -0800 Subject: [PATCH 049/229] gh-159739: Preserve large years in time conversions (GH-160175) * gh-159739: Preserve large years in time conversions The time module added 1900 to the C library year using int arithmetic, which overflowed for timestamps whose calendar year exceeds INT_MAX. On platforms that accept these timestamps, gmtime(), localtime(), and ctime() could report a negative year; perform the addition in a wider type. Backport to 3.15 and 3.14. * gh-159739: Round-trip large years and validate datetime timestamps Time tuple consumers parsed calendar years as C int before subtracting 1900, so they rejected valid struct_time results. Date and datetime timestamp conversion could also overflow before reporting the invalid year. Preserve the full time tuple range and report oversized datetime years correctly. Backport to 3.15 and 3.14. --- Lib/test/datetimetester.py | 9 ++++++ Lib/test/test_time.py | 28 +++++++++++++++---- ...-10-22-03-55-33.gh-issue-159739.ce26e1.rst | 4 +++ Modules/_datetimemodule.c | 22 +++++++++++++-- Modules/timemodule.c | 20 ++++++------- 5 files changed, 66 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.ce26e1.rst diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 7d242f663824b39..00a9d2bca3f4779 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1590,6 +1590,15 @@ def test_insane_fromtimestamp(self): self.assertRaises(OverflowError, self.theclass.fromtimestamp, insane) + def test_fromtimestamp_large_year(self): + timestamp = 67_767_976_249_084_800 # Year 2147483648. + try: + year = _time.localtime(timestamp).tm_year + except (OverflowError, OSError): + self.skipTest('timestamp out of range on this platform') + with self.assertRaisesRegex(ValueError, rf'\b{year}\b'): + self.theclass.fromtimestamp(timestamp) + def test_fromtimestamp_with_none_arg(self): # See gh-120268 for more details with self.assertRaises(TypeError): diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index dac4dd9060ccdd1..c5754afd0e84a81 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -21,10 +21,11 @@ from test.support import skip_if_buggy_ucrt_strfptime, SuppressCrashReport -# Max year is only limited by the size of C int. +# Years are limited by the range of struct tm's int tm_year field. SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4 -TIME_MAXYEAR = (1 << 8 * SIZEOF_INT - 1) - 1 -TIME_MINYEAR = -TIME_MAXYEAR - 1 + 1900 +C_INT_MAX = (1 << 8 * SIZEOF_INT - 1) - 1 +TIME_MAXYEAR = C_INT_MAX + 1900 +TIME_MINYEAR = -C_INT_MAX - 1 + 1900 SEC_TO_US = 10 ** 6 US_TO_NS = 10 ** 3 @@ -153,6 +154,23 @@ def test_clock_settime(self): self.assertRaises(OSError, time.clock_settime, time.CLOCK_MONOTONIC, 0) + def test_large_timestamp_year(self): + # 2147483648-06-29 00:00:00 UTC, away from local year boundaries. + timestamp = 67_767_976_249_084_800 + for convert in (time.gmtime, time.localtime, time.ctime): + with self.subTest(convert=convert): + try: + result = convert(timestamp) + except (OverflowError, OSError): + self.skipTest('timestamp out of range on this platform') + if isinstance(result, str): + year = int(result.rsplit(' ', 1)[-1]) + else: + year = result.tm_year + self.assertEqual(year, 2**31) + if convert is time.localtime: + self.assertEqual(time.asctime(result), time.ctime(timestamp)) + def test_conversions(self): self.assertEqual(time.ctime(self.t), time.asctime(time.localtime(self.t))) @@ -456,7 +474,7 @@ def test_strptime_leap_year(self): def test_asctime(self): time.asctime(time.gmtime(self.t)) - # Max year is only limited by the size of C int. + # The year offset must fit into a C int. for bigyear in TIME_MAXYEAR, TIME_MINYEAR: asc = time.asctime((bigyear, 6, 1) + (0,) * 6) self.assertEqual(asc[-len(str(bigyear)):], str(bigyear)) @@ -827,7 +845,7 @@ def test_year(self, fmt=None, func=None): def test_large_year(self): self.assertEqual(self.yearstr(12345).lstrip('+'), '12345') self.assertEqual(self.yearstr(123456789).lstrip('+'), '123456789') - self.assertEqual(self.yearstr(TIME_MAXYEAR).lstrip('+'), str(TIME_MAXYEAR)) + self.assertEqual(self.yearstr(C_INT_MAX).lstrip('+'), str(C_INT_MAX)) self.assertRaises(OverflowError, self.yearstr, TIME_MAXYEAR + 1) def test_negative(self): diff --git a/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.ce26e1.rst b/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.ce26e1.rst new file mode 100644 index 000000000000000..94f97951af2712d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.ce26e1.rst @@ -0,0 +1,4 @@ +Fix :mod:`time` functions returning a negative year or failing to accept +their own output for large timestamps on platforms that support them. Avoid +reporting an incorrect year in errors raised by :mod:`datetime` for those +timestamps. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 380d8b435e73d6c..b9939d5cc71a319 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -668,6 +668,18 @@ check_date_args(int year, int month, int day) return 0; } +static int +year_from_tm(const struct tm *tm, int *year) +{ + if (tm->tm_year > INT_MAX - 1900) { + PyErr_Format(PyExc_ValueError, "year must be in %d..%d, not %lld", + MINYEAR, MAXYEAR, (long long)tm->tm_year + 1900); + return -1; + } + *year = tm->tm_year + 1900; + return 0; +} + /* Check that time arguments are in range. Return 0 if they are. If they * aren't, raise ValueError and return -1. */ @@ -3297,14 +3309,18 @@ date_fromtimestamp(PyTypeObject *cls, PyObject *obj) { struct tm tm; time_t t; + int year; if (_PyTime_ObjectToTime_t(obj, &t, _PyTime_ROUND_FLOOR) == -1) return NULL; if (_PyTime_localtime(t, &tm) != 0) return NULL; + if (year_from_tm(&tm, &year) < 0) { + return NULL; + } - return new_date_subclass_ex(tm.tm_year + 1900, + return new_date_subclass_ex(year, tm.tm_mon + 1, tm.tm_mday, cls); @@ -5675,7 +5691,9 @@ datetime_from_timet_and_us(PyTypeObject *cls, TM_FUNC f, time_t timet, int us, if (f(timet, &tm) != 0) return NULL; - year = tm.tm_year + 1900; + if (year_from_tm(&tm, &year) < 0) { + return NULL; + } month = tm.tm_mon + 1; day = tm.tm_mday; hour = tm.tm_hour; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 064ab2cf2364f1a..395d6ec0f92d5f8 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -487,7 +487,7 @@ tmtotuple(time_module_state *state, struct tm *p #define SET(INDEX, VAL) \ SET_ITEM((INDEX), PyLong_FromLong((long) (VAL))) - SET(0, p->tm_year + 1900); + SET_ITEM(0, PyLong_FromLongLong((long long)p->tm_year + 1900)); SET(1, p->tm_mon + 1); /* Want January == 1 */ SET(2, p->tm_mday); SET(3, p->tm_hour); @@ -629,7 +629,7 @@ static int gettmarg(time_module_state *state, PyObject *args, struct tm *p, const char *format) { - int y; + long long y; memset((void *) p, '\0', sizeof(struct tm)); @@ -645,12 +645,12 @@ gettmarg(time_module_state *state, PyObject *args, &p->tm_wday, &p->tm_yday, &p->tm_isdst)) return 0; - if (y < INT_MIN + 1900) { + if (y < (long long)INT_MIN + 1900 || y > (long long)INT_MAX + 1900) { PyErr_SetString(PyExc_OverflowError, "year out of range"); return 0; } - p->tm_year = y - 1900; + p->tm_year = (int)(y - 1900); p->tm_mon--; p->tm_wday = (p->tm_wday + 1) % 7; p->tm_yday--; @@ -906,7 +906,7 @@ time_strftime(PyObject *module, PyObject *args) return NULL; } else if (!gettmarg(state, tup, &buf, - "iiiiiiiii;strftime(): illegal time tuple argument") || + "Liiiiiiii;strftime(): illegal time tuple argument") || !checktm(&buf)) { return NULL; @@ -918,7 +918,7 @@ time_strftime(PyObject *module, PyObject *args) // physical devices (#123017). #if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) \ || defined(__VXWORKS__) || defined(__ANDROID__) - if (buf.tm_year + 1900 < 1 || 9999 < buf.tm_year + 1900) { + if (buf.tm_year < 1 - 1900 || buf.tm_year > 9999 - 1900) { PyErr_SetString(PyExc_ValueError, "strftime() requires year in [1; 9999]"); return NULL; @@ -1075,12 +1075,12 @@ _asctime(struct tm *timeptr) "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; return PyUnicode_FromFormat( - "%s %s%3d %.2d:%.2d:%.2d %d", + "%s %s%3d %.2d:%.2d:%.2d %lld", wday_name[timeptr->tm_wday], mon_name[timeptr->tm_mon], timeptr->tm_mday, timeptr->tm_hour, timeptr->tm_min, timeptr->tm_sec, - 1900 + timeptr->tm_year); + (long long)timeptr->tm_year + 1900); } /*[clinic input] @@ -1108,7 +1108,7 @@ time_asctime_impl(PyObject *module, PyObject *tup) return NULL; } else if (!gettmarg(state, tup, &buf, - "iiiiiiiii;asctime(): illegal time tuple argument") || + "Liiiiiiii;asctime(): illegal time tuple argument") || !checktm(&buf)) { return NULL; @@ -1164,7 +1164,7 @@ time_mktime(PyObject *module, PyObject *tm_tuple) time_module_state *state = get_time_state(module); if (!gettmarg(state, tm_tuple, &tm, - "iiiiiiiii;mktime(): illegal time tuple argument")) + "Liiiiiiii;mktime(): illegal time tuple argument")) { return NULL; } From 0aec939d7d1e9728a0d4a55f280d08771f598388 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 9 Nov 2026 12:51:42 +0900 Subject: [PATCH 050/229] Doc: Update threading FAQ and use local tables of contents (#161554) --- Doc/faq/design.rst | 1 + Doc/faq/extending.rst | 1 + Doc/faq/general.rst | 1 + Doc/faq/gui.rst | 1 + Doc/faq/library.rst | 80 +++++---------------------------------- Doc/faq/programming.rst | 1 + Doc/faq/windows.rst | 1 + Doc/tools/removed-ids.txt | 1 + 8 files changed, 17 insertions(+), 70 deletions(-) diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 02a3a3a22070dae..3a3e16b3bcf3db4 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -5,6 +5,7 @@ Design and History FAQ .. only:: html .. contents:: + :local: Why does Python use indentation for grouping of statements? diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 1d5abed2317b0ce..e0e6ddd04af76df 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -5,6 +5,7 @@ Extending/Embedding FAQ .. only:: html .. contents:: + :local: .. highlight:: c diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index ff38ba395fba22c..ba4fb89713c66d2 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -7,6 +7,7 @@ General Python FAQ .. only:: html .. contents:: + :local: General Information diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index cfa60feceb31b77..37e77ee08fc8ec9 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -7,6 +7,7 @@ Graphic User Interface FAQ .. only:: html .. contents:: + :local: .. XXX need review for Python 3. diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index d8d75ca6f2ec96c..abfd35ea8846418 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -7,6 +7,7 @@ Library and Extension FAQ .. only:: html .. contents:: + :local: General Library Questions ========================= @@ -241,51 +242,6 @@ The :mod:`threading` module builds convenient abstractions on top of the low-level primitives provided by the :mod:`_thread` module. -None of my threads seem to run: why? ------------------------------------- - -As soon as the main thread exits, all threads are killed. Your main thread is -running too quickly, giving the threads no time to do any work. - -A simple fix is to add a sleep to the end of the program that's long enough for -all the threads to finish:: - - import threading, time - - def thread_task(name, n): - for i in range(n): - print(name, i) - - for i in range(10): - T = threading.Thread(target=thread_task, args=(str(i), i)) - T.start() - - time.sleep(10) # <---------------------------! - -But now (on many platforms) the threads don't run in parallel, but appear to run -sequentially, one at a time! The reason is that the OS thread scheduler doesn't -start a new thread until the previous thread is blocked. - -A simple fix is to add a tiny sleep to the start of the run function:: - - def thread_task(name, n): - time.sleep(0.001) # <--------------------! - for i in range(n): - print(name, i) - - for i in range(10): - T = threading.Thread(target=thread_task, args=(str(i), i)) - T.start() - - time.sleep(10) - -Instead of trying to guess a good delay value for :func:`time.sleep`, -it's better to use some kind of semaphore mechanism. One idea is to use the -:mod:`queue` module to create a queue object, let each thread append a token to -the queue when it finishes, and let the main thread read as many tokens from the -queue as there are threads. - - How do I parcel out work among a bunch of worker threads? --------------------------------------------------------- @@ -410,35 +366,17 @@ deployment on high-end multiprocessor server machines, because a multi-threaded Python program effectively only uses one CPU, due to the insistence that (almost) all Python code can only run while the GIL is held. -With the approval of :pep:`703` work is now underway to remove the GIL from the -CPython implementation of Python. Initially it will be implemented as an -optional compiler flag when building the interpreter, and so separate -builds will be available with and without the GIL. Long-term, the hope is -to settle on a single build, once the performance implications of removing the -GIL are fully understood. Python 3.13 is likely to be the first release -containing this work, although it may not be completely functional in this -release. - -The current work to remove the GIL is based on a -`fork of Python 3.9 with the GIL removed `_ -by Sam Gross. -Prior to that, -in the days of Python 1.5, Greg Stein actually implemented a comprehensive -patch set (the "free threading" patches) that removed the GIL and replaced it -with fine-grained locking. Adam Olsen did a similar experiment -in his `python-safethread `_ -project. Unfortunately, both of these earlier experiments exhibited a sharp -drop in single-thread -performance (at least 30% slower), due to the amount of fine-grained locking -necessary to compensate for the removal of the GIL. The Python 3.9 fork -is the first attempt at removing the GIL with an acceptable performance -impact. +Since Python 3.13, CPython provides an optional free-threaded build that can run +with the GIL disabled, allowing Python threads to run in parallel on multiple +CPU cores (see :pep:`703`). Free-threaded Python is officially supported as of +Python 3.14 (see :pep:`779`), but is not the default build. See +:doc:`/howto/free-threading-python` for details. The presence of the GIL in current Python releases doesn't mean that you can't make good use of Python on multi-CPU machines! You just have to be creative with dividing the work up between multiple *processes* rather than multiple *threads*. The -:class:`~concurrent.futures.ProcessPoolExecutor` class in the new +:class:`~concurrent.futures.ProcessPoolExecutor` class in the :mod:`concurrent.futures` module provides an easy way of doing so; the :mod:`multiprocessing` module provides a lower-level API in case you want more control over dispatching of tasks. @@ -452,7 +390,9 @@ already do this. An alternative approach to reducing the impact of the GIL is to make the GIL a per-interpreter-state lock rather than truly global. This was :ref:`first implemented in Python 3.12 ` and is -available in the C API. A Python interface to it is expected in Python 3.13. +available in the C API. Python interfaces were added in Python 3.14 through the +:mod:`concurrent.interpreters` module and the +:class:`~concurrent.futures.InterpreterPoolExecutor` class. The main limitation to it at the moment is likely to be 3rd party extension modules, since these must be written with multiple interpreters in mind in order to be usable, so many older extension modules will not be usable. diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 4e1157e6ebe7296..5eb8dff1ee8052b 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -7,6 +7,7 @@ Programming FAQ .. only:: html .. contents:: + :local: General questions ================= diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index c0c92fdbbc84d12..7f6170f8ade99e9 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -11,6 +11,7 @@ Python on Windows FAQ .. only:: html .. contents:: + :local: .. XXX need review for Python 3. XXX need review for Windows Vista/Seven? diff --git a/Doc/tools/removed-ids.txt b/Doc/tools/removed-ids.txt index cc5ec51b541ab31..f846e4b8dc15249 100644 --- a/Doc/tools/removed-ids.txt +++ b/Doc/tools/removed-ids.txt @@ -29,6 +29,7 @@ c-api/interp-lifecycle.html: global-configuration-variables c-api/threads.html: c.PyEval_InitThreads # Removed sections +faq/library.html: none-of-my-threads-seem-to-run-why library/asyncio-task.html: terminating-a-task-group library/dis.html: opcode-DELETE_ATTR library/dis.html: opcode-DELETE_GLOBAL From 3dee86d70560329437c2e774a1a89a3c57b14119 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 9 Nov 2026 12:56:53 +0900 Subject: [PATCH 051/229] Doc: Correct built-in function and type descriptions (#161552) --- Doc/builtins/functions.rst | 8 +----- Doc/builtins/stdtypes.rst | 33 +++++++++++----------- Lib/pydoc_data/topics.py | 56 ++++++++++++++++++++++---------------- 3 files changed, 49 insertions(+), 48 deletions(-) diff --git a/Doc/builtins/functions.rst b/Doc/builtins/functions.rst index a1db1544c7bf4ed..717efe50c130744 100644 --- a/Doc/builtins/functions.rst +++ b/Doc/builtins/functions.rst @@ -740,7 +740,7 @@ are always available. They are listed here in alphabetical order. This function supports dynamic execution of Python code. *source* must be either a string or a code object. If it is a string, the string is parsed as a suite of Python statements which is then executed (unless a syntax error - occurs). [#]_ If it is a code object, it is simply executed. In all cases, + occurs). If it is a code object, it is simply executed. In all cases, the code that's executed is expected to be valid as file input (see the section :ref:`file-input` in the Reference Manual). Be aware that the :keyword:`nonlocal`, :keyword:`yield`, and :keyword:`return` @@ -2470,9 +2470,3 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.9 When the command line options :option:`-E` or :option:`-I` are being used, the environment variable :envvar:`PYTHONCASEOK` is now ignored. - -.. rubric:: Footnotes - -.. [#] Note that the parser only accepts the Unix-style end of line convention. - If you are reading the code from a file, make sure to use newline conversion - mode to convert Windows or Mac-style newlines. diff --git a/Doc/builtins/stdtypes.rst b/Doc/builtins/stdtypes.rst index b998331cb60c5b2..7072f01e09aa250 100644 --- a/Doc/builtins/stdtypes.rst +++ b/Doc/builtins/stdtypes.rst @@ -1225,8 +1225,8 @@ the underlying generator function: If the asynchronous generator function then exits gracefully, is already closed, or raises :exc:`GeneratorExit` (by not catching the exception), then the returned awaitable will raise a :exc:`StopIteration` exception. - Any further awaitables returned by subsequent calls to the asynchronous - generator will raise a :exc:`StopAsyncIteration` exception. + Awaitables returned by subsequent calls to :meth:`~agen.__anext__` + or :meth:`~agen.asend` will raise a :exc:`StopAsyncIteration` exception. If the asynchronous generator yields a value, a :exc:`RuntimeError` is raised by the awaitable. @@ -2313,19 +2313,19 @@ expression support in the :mod:`re` module). Perform a string formatting operation. The string on which this method is called can contain literal text or replacement fields delimited by braces - ``{}``. Each replacement field contains either the numeric index of a - positional argument, or the name of a keyword argument. Returns a copy of + ``{}``. Each replacement field refers to a positional argument (numbered + explicitly or automatically) or a keyword argument. Returns a copy of the string where each replacement field is replaced with the string value of the corresponding argument. For example: .. doctest:: - >>> "The sum of 1 + 2 is {0}".format(1+2) - 'The sum of 1 + 2 is 3' - >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + >>> "The sum of {} + {} is {}".format(1, 2, 1+2) 'The sum of 1 + 2 is 3' >>> "{1} expects the {0} Inquisition!".format("Spanish", "Nobody") 'Nobody expects the Spanish Inquisition!' + >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + 'The sum of 1 + 2 is 3' See :ref:`formatstrings` for a description of the various formatting options that can be specified in format strings. @@ -3180,7 +3180,7 @@ expression support in the :mod:`re` module). Return a copy of the string with all the cased characters [4]_ converted to uppercase. Note that ``s.upper().isupper()`` might be ``False`` if ``s`` - contains uncased characters or if the Unicode category of the resulting + contains no cased characters or if the Unicode category of the resulting character(s) is not "Lu" (Letter, uppercase), but e.g. "Lt" (Letter, titlecase). @@ -3537,9 +3537,8 @@ The conversion types are: | | character in the result. | | +------------+-----------------------------------------------------+-------+ -For floating-point formats, the result should be correctly rounded to a given -precision ``p`` of digits after the decimal point. The rounding mode matches -that of the :func:`round` builtin. +For floating-point formats, the result should be correctly rounded to the +specified precision. The rounding mode matches that of the :func:`round` builtin. Notes: @@ -5928,8 +5927,9 @@ can be used interchangeably to index the same dictionary entry. .. seealso:: - :class:`frozendict` and :class:`types.MappingProxyType` can be used to - create a read-only view of a :class:`dict`. + :class:`types.MappingProxyType` can be used to create a read-only view + of a :class:`dict`. :class:`frozendict` is an immutable version of + :class:`dict`. .. seealso:: @@ -6082,8 +6082,7 @@ Frozen dictionaries Similar to :meth:`dict.fromkeys`, but call again the type constructor with an initialized :class:`frozendict` if the type is a - :class:`frozendict` subclass or if the constructor returned a - :class:`frozendict`. + :class:`frozendict` subclass. .. versionadded:: 3.15 @@ -6532,8 +6531,8 @@ The user-exposed type for the union object can be accessed from .. note:: The :meth:`!__or__` method for type objects was added to support the syntax - ``X | Y``. If a metaclass implements :meth:`!__or__`, the Union may - override it: + ``X | Y``. If a metaclass implements :meth:`!__or__`, it may override + the default behavior of creating a union: .. doctest:: diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 7278c2dba6d219a..8a237ef4d5e927d 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Wed Oct 14 15:38:08 2026 +# Autogenerated by Sphinx on Sat Nov 7 11:32:29 2026 # as part of the release process. topics = { @@ -4112,9 +4112,9 @@ def add(self, arg: T) -> None: def overly_generic[ SimpleTypeVar, - TypeVarWithDefault = int, TypeVarWithBound: int, TypeVarWithConstraints: (str, bytes), + TypeVarWithDefault = int, *SimpleTypeVarTuple = (int, float), **SimpleParamSpec = (str, bytearray), ]( @@ -4212,7 +4212,7 @@ class Bag(typing.Generic[T]): outside that scope. This is illustrated by this example: @decorator - class Bag(Base[T], arg=T): ... + class Bag[T](Base[T], arg=T): ... This is equivalent to: @@ -4959,8 +4959,8 @@ def double(x): awaitable pdb.set_trace_async(*, header=None, commands=None) - async version of "set_trace()". This function should be used inside - an async function with "await". + Asynchronous version of "set_trace()". This function should be used + inside a *coroutine function* with "await". For instance: async def f(): await pdb.set_trace_async() @@ -5188,7 +5188,8 @@ class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=Fa With a *function* argument, set a break at the first executable statement within that function. *function* can be any expression - that evaluates to a function in the current namespace. + that evaluates to a function in the current namespace, or in the + form "package.module.function" if "package.module" is importable. If a second argument is present, it is an expression which must evaluate to true before the breakpoint is honored. @@ -5536,7 +5537,7 @@ class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=Fa List or jump between chained exceptions. - When using "pdb.pm()" or "Pdb.post_mortem(...)" with a chained + When using "pdb.pm()" or "pdb.post_mortem(...)" with a chained exception instead of a traceback, it allows the user to move between the chained exceptions using "exceptions" command to list exceptions, and "exceptions " to switch to that exception. @@ -11742,7 +11743,14 @@ class is used in a class pattern with positional arguments, each str.capitalize() Return a copy of the string with its first character capitalized - and the rest lowercased. + and the rest lowercased. For example: + + >>> 'PYTHON IS AMAZING'.capitalize() + 'Python is amazing' + >>> 'Njemačka starts With a non-ASCII digraph'.capitalize() + 'Njemačka starts with a non-ascii digraph' + + See also "title()". Changed in version 3.8: The first character is now put into titlecase rather than uppercase. This means that characters like @@ -11903,18 +11911,18 @@ class is used in a class pattern with positional arguments, each Perform a string formatting operation. The string on which this method is called can contain literal text or replacement fields - delimited by braces "{}". Each replacement field contains either - the numeric index of a positional argument, or the name of a + delimited by braces "{}". Each replacement field refers to a + positional argument (numbered explicitly or automatically) or a keyword argument. Returns a copy of the string where each replacement field is replaced with the string value of the corresponding argument. For example: - >>> "The sum of 1 + 2 is {0}".format(1+2) - 'The sum of 1 + 2 is 3' - >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + >>> "The sum of {} + {} is {}".format(1, 2, 1+2) 'The sum of 1 + 2 is 3' >>> "{1} expects the {0} Inquisition!".format("Spanish", "Nobody") 'Nobody expects the Spanish Inquisition!' + >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + 'The sum of 1 + 2 is 3' See Format string syntax for a description of the various formatting options that can be specified in format strings. @@ -12644,7 +12652,7 @@ class is used in a class pattern with positional arguments, each >>> titlecase("they're bill's friends.") "They're Bill's Friends." - See also "istitle()". + See also "istitle()" and "capitalize()". str.translate(table, /) @@ -12673,7 +12681,7 @@ class is used in a class pattern with positional arguments, each Return a copy of the string with all the cased characters [4] converted to uppercase. Note that "s.upper().isupper()" might be - "False" if "s" contains uncased characters or if the Unicode + "False" if "s" contains no cased characters or if the Unicode category of the resulting character(s) is not “Lu” (Letter, uppercase), but e.g. “Lt” (Letter, titlecase). @@ -14221,7 +14229,8 @@ def foo(): * "__name__" is the function’s name. See "function.__name__". -* "__self__" is set to "None" (but see the next item). +* "__self__" is the module object the function was defined in or + "None" if unavailable. * "__module__" is the name of the module the function was defined in or "None" if unavailable. See "function.__module__". @@ -15124,7 +15133,7 @@ class dict(iterable, /, **kwargs) "{'jack': 4098, 'sjoerd': 4127}" or "{4098: 'jack', 4127: 'sjoerd'}" - * Use a dict comprehension: "{}", "{x: x ** 2 for x in range(10)}" + * Use a dict comprehension: "{x: x ** 2 for x in range(10)}" * Use the type constructor: "dict()", "dict([('foo', 100), ('bar', 200)])", "dict(foo=100, bar=200)" @@ -15340,9 +15349,9 @@ class dict(iterable, /, **kwargs) Return a new view of the dictionary’s values. See the documentation of view objects. - An equality comparison between one "dict.values()" view and - another will always return "False". This also applies when - comparing "dict.values()" to itself: + An equality comparison between two distinct "dict.values()" + views will always return "False", even for views of the same + dictionary: >>> d = {'a': 1} >>> d.values() == d.values() @@ -15381,8 +15390,8 @@ class dict(iterable, /, **kwargs) See also: - "frozendict" and "types.MappingProxyType" can be used to create a - read-only view of a "dict". + "types.MappingProxyType" can be used to create a read-only view + of a "dict". "frozendict" is an immutable version of "dict". See also: @@ -15540,8 +15549,7 @@ class frozendict(iterable, /, **kwargs) Similar to "dict.fromkeys()", but call again the type constructor with an initialized "frozendict" if the type is a - "frozendict" subclass or if the constructor returned a - "frozendict". + "frozendict" subclass. Added in version 3.15. ''', From 4f64990486f3a28b2edf01d7fdc3b18cfa0981f1 Mon Sep 17 00:00:00 2001 From: excusemejoe <1658389+devjoe@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:19:46 +0800 Subject: [PATCH 052/229] gh-66333: Allow to break into pdb with Ctrl-C for all the execution commands (#13269) --- Doc/library/pdb.rst | 7 +- Lib/pdb.py | 25 ++++--- Lib/test/test_pdb.py | 69 +++++++++++++++++++ ...-28-16-40-00.gh-issue-66333.pdb-sigint.rst | 3 + 4 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-28-16-40-00.gh-issue-66333.pdb-sigint.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index df5de52a412efac..1a494e4a0f9fa5a 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -278,7 +278,8 @@ access further features, you have to do this yourself: that matches one of these patterns. [1]_ By default, Pdb sets a handler for the SIGINT signal (which is sent when the - user presses :kbd:`Ctrl-C` on the console) when you give a :pdbcmd:`continue` command. + user presses :kbd:`Ctrl-C` on the console) when you resume execution with + :pdbcmd:`continue`, :pdbcmd:`next`, :pdbcmd:`until`, or :pdbcmd:`return`. This allows you to break into the debugger again by pressing :kbd:`Ctrl-C`. If you want Pdb not to touch the SIGINT handler, set *nosigint* to true. @@ -328,6 +329,10 @@ access further features, you have to do this yourself: Inline breakpoints like :func:`breakpoint` or :func:`pdb.set_trace` will always stop the program at calling frame, ignoring the *skip* pattern (if any). + .. versionchanged:: 3.16 + SIGINT handling also applies to :pdbcmd:`next`, :pdbcmd:`until`, and + :pdbcmd:`return`. + .. method:: run(statement, globals=None, locals=None) runeval(expression, globals=None, locals=None) runcall(function, *args, **kwds) diff --git a/Lib/pdb.py b/Lib/pdb.py index 114634a4469c131..13f629ae8180591 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -628,11 +628,20 @@ async def set_trace_async(self, frame=None, *, commands=None): self.start_trace() + def _install_sigint_handler(self): + if self.nosigint: + return + try: + Pdb._previous_sigint_handler = \ + signal.signal(signal.SIGINT, self.sigint_handler) + except ValueError: + # signal only works in the main thread. + pass + def sigint_handler(self, signum, frame): if self.allow_kbdint: raise KeyboardInterrupt self.message("\nProgram interrupted. (Use 'cont' to resume).") - self.set_step() self.set_trace(frame) def reset(self): @@ -2045,6 +2054,7 @@ def do_until(self, arg): else: lineno = None self.set_until(self.curframe, lineno) + self._install_sigint_handler() return 1 do_unt = do_until @@ -2072,6 +2082,7 @@ def do_next(self, arg): self._print_invalid_arg(arg) return self.set_next(self.curframe) + self._install_sigint_handler() return 1 do_n = do_next @@ -2110,6 +2121,7 @@ def do_return(self, arg): self._print_invalid_arg(arg) return self.set_return(self.curframe) + self._install_sigint_handler() return 1 do_r = do_return @@ -2121,16 +2133,7 @@ def do_continue(self, arg): if arg: self._print_invalid_arg(arg) return - if not self.nosigint: - try: - Pdb._previous_sigint_handler = \ - signal.signal(signal.SIGINT, self.sigint_handler) - except ValueError: - # ValueError happens when do_continue() is invoked from - # a non-main thread in which case we just continue without - # SIGINT set. Would printing a message here (once) make - # sense? - pass + self._install_sigint_handler() self.set_continue() return 1 do_c = do_cont = do_continue diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index cfcc8c6df57b948..bc9591d05de6b3d 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -7,6 +7,7 @@ import os import pdb import re +import signal import sys import types import codecs @@ -2973,6 +2974,68 @@ def test_pdb_with_inline_breakpoint(): (Pdb) c """ +def test_pdb_sigint(): + """SIGINT should pause continue, next, until and return + + >>> def interrupt(): + ... signal.raise_signal(signal.SIGINT) + + >>> def test_function(): + ... import pdb; pdb.Pdb(readrc=False).set_trace() + ... interrupt() + ... interrupt() + ... interrupt() + ... interrupt() + + >>> with PdbTestInput([ + ... 'n', + ... 'n', + ... 'up', + ... 'until 5', + ... 'up', + ... 'r', + ... 'up', + ... 'c', + ... 'c', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(readrc=False).set_trace() + (Pdb) n + > (3)test_function() + -> interrupt() + (Pdb) n + + Program interrupted. (Use 'cont' to resume). + > (2)interrupt() + -> signal.raise_signal(signal.SIGINT) + (Pdb) up + > (3)test_function() + -> interrupt() + (Pdb) until 5 + + Program interrupted. (Use 'cont' to resume). + > (2)interrupt() + -> signal.raise_signal(signal.SIGINT) + (Pdb) up + > (4)test_function() + -> interrupt() + (Pdb) r + + Program interrupted. (Use 'cont' to resume). + > (2)interrupt() + -> signal.raise_signal(signal.SIGINT) + (Pdb) up + > (5)test_function() + -> interrupt() + (Pdb) c + + Program interrupted. (Use 'cont' to resume). + > (2)interrupt() + -> signal.raise_signal(signal.SIGINT) + (Pdb) c + """ + def test_pdb_issue_20766(): """Test for reference leaks when the SIGINT handler is set. @@ -5432,9 +5495,15 @@ def setUpPdbBackend(backend): def setUp(test): import pdb pdb.set_default_backend(backend) + test.original_sigint_handler = signal.getsignal(signal.SIGINT) + test.previous_sigint_handler = pdb.Pdb._previous_sigint_handler return setUp def tearDown(test): + # Restore signal state even if a SIGINT doctest fails. + signal.signal(signal.SIGINT, test.original_sigint_handler) + pdb.Pdb._previous_sigint_handler = test.previous_sigint_handler + # Ensure that asyncio state has been cleared at the end of the test. # This prevents a "test altered the execution environment" warning if # asyncio features are used. diff --git a/Misc/NEWS.d/next/Library/2026-10-28-16-40-00.gh-issue-66333.pdb-sigint.rst b/Misc/NEWS.d/next/Library/2026-10-28-16-40-00.gh-issue-66333.pdb-sigint.rst new file mode 100644 index 000000000000000..057caa487e9631b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-28-16-40-00.gh-issue-66333.pdb-sigint.rst @@ -0,0 +1,3 @@ +Allow Ctrl-C to interrupt execution resumed with the :mod:`pdb` +``next``, ``until``, and ``return`` commands without unwinding the debugged +program, as with ``continue``. From 392d6d1e8438c602943f7fecaebbe22b6393e158 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Mon, 9 Nov 2026 10:28:16 +0100 Subject: [PATCH 053/229] gh-151633: avoid use-after-free in `Counter.update` (#151634) Co-authored-by: Pieter Eendebak --- .../test_free_threading/test_collections.py | 15 ++- ...-06-18-11-26-00.gh-issue-151633.uJzHdc.rst | 2 + Modules/_collectionsmodule.c | 119 +++++++++++------- 3 files changed, 88 insertions(+), 48 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-06-18-11-26-00.gh-issue-151633.uJzHdc.rst diff --git a/Lib/test/test_free_threading/test_collections.py b/Lib/test/test_free_threading/test_collections.py index a0a73c965d072a2..d6efde55e698d0a 100644 --- a/Lib/test/test_free_threading/test_collections.py +++ b/Lib/test/test_free_threading/test_collections.py @@ -1,5 +1,5 @@ import unittest -from collections import OrderedDict, deque +from collections import Counter, OrderedDict, deque from copy import copy from test.support import threading_helper @@ -49,6 +49,19 @@ def mutate(): ) +class TestCounter(unittest.TestCase): + def test_update_concurrent(self): + # gh-151633: concurrent Counter.update calls must not cause use-after-free + NTHREADS = 4 + PER_THREAD = 5000 + c = Counter() + data = ('x',) * PER_THREAD + threading_helper.run_concurrently( + c.update, nthreads=NTHREADS, args=(data,) + ) + self.assertEqual(c['x'], NTHREADS * PER_THREAD) + + class TestOrderedDict(unittest.TestCase): def test_iterator_update_clear_race(self): # gh-151627: OrderedDict iterator construction must not race with diff --git a/Misc/NEWS.d/next/Library/2026-06-18-11-26-00.gh-issue-151633.uJzHdc.rst b/Misc/NEWS.d/next/Library/2026-06-18-11-26-00.gh-issue-151633.uJzHdc.rst new file mode 100644 index 000000000000000..73d533c0d109bbe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-18-11-26-00.gh-issue-151633.uJzHdc.rst @@ -0,0 +1,2 @@ +Fix a use-after-free under free-threading when multiple threads +update the same :class:`~collections.Counter` concurrently. diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 0626cc6a70bc8f7..c63dc0661bd69e7 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() -#include "pycore_dict.h" // _PyDict_GetItem_KnownHash() +#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() +#include "pycore_dict.h" // _PyDict_GetItemRef_KnownHash_LockHeld() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyatomic_ft_wrappers.h" @@ -2525,6 +2526,66 @@ static PyType_Spec defdict_spec = { /* helper function for Counter *********************************************/ +/* Fast path advantages: + 1. Eliminate double hashing + (by re-using the same hash for both the get and set) + 2. Avoid argument overhead of PyObject_CallFunctionObjArgs + (argument tuple creation and parsing) + 3. Avoid indirection through a bound method object + (creates another argument tuple) + 4. Avoid initial increment from zero + (reuse an existing one-object instead) +*/ +static int +_count_elements_dict_lock_held(PyDictObject *mapping, PyObject *it) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mapping); + PyObject *one = _PyLong_GetOne(); // borrowed reference + + while (1) { + Py_hash_t hash; + PyObject *oldval; + int status; + + PyObject *key = PyIter_Next(it); + if (key == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + + hash = _PyObject_HashDictKey(key); + if (hash == -1) { + Py_DECREF(key); + return -1; + } + + int found = _PyDict_GetItemRef_KnownHash_LockHeld(mapping, key, hash, + &oldval); + if (found < 0) { + Py_DECREF(key); + return -1; + } + if (found == 0) { + status = _PyDict_SetItem_KnownHash_LockHeld(mapping, key, one, + hash); + } + else { + PyObject *newval = PyNumber_Add(oldval, one); + Py_DECREF(oldval); + if (newval == NULL) { + Py_DECREF(key); + return -1; + } + status = _PyDict_SetItem_KnownHash_LockHeld(mapping, key, newval, + hash); + Py_DECREF(newval); + } + Py_DECREF(key); + if (status < 0) { + return -1; + } + } +} + /*[clinic input] _collections._count_elements @@ -2541,7 +2602,6 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, /*[clinic end generated code: output=7e0c1789636b3d8f input=e79fad04534a0b45]*/ { PyObject *it, *oldval; - PyObject *newval = NULL; PyObject *key = NULL; PyObject *bound_get = NULL; PyObject *mapping_get; @@ -2566,48 +2626,12 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, mapping_setitem != NULL && mapping_setitem == dict_setitem && PyDict_Check(mapping)) { - while (1) { - /* Fast path advantages: - 1. Eliminate double hashing - (by re-using the same hash for both the get and set) - 2. Avoid argument overhead of PyObject_CallFunctionObjArgs - (argument tuple creation and parsing) - 3. Avoid indirection through a bound method object - (creates another argument tuple) - 4. Avoid initial increment from zero - (reuse an existing one-object instead) - */ - Py_hash_t hash; - - key = PyIter_Next(it); - if (key == NULL) - break; - - hash = _PyObject_HashDictKey(key); - if (hash == -1) { - goto done; - } - - oldval = _PyDict_GetItem_KnownHash(mapping, key, hash); - if (oldval == NULL) { - if (PyErr_Occurred()) - goto done; - if (_PyDict_SetItem_KnownHash(mapping, key, one, hash) < 0) - goto done; - } else { - /* oldval is a borrowed reference. Keep it alive across - PyNumber_Add(), which can execute arbitrary user code and - mutate (or even clear) the underlying dict. */ - Py_INCREF(oldval); - newval = PyNumber_Add(oldval, one); - Py_DECREF(oldval); - if (newval == NULL) - goto done; - if (_PyDict_SetItem_KnownHash(mapping, key, newval, hash) < 0) - goto done; - Py_CLEAR(newval); - } - Py_DECREF(key); + int status; + Py_BEGIN_CRITICAL_SECTION(mapping); + status = _count_elements_dict_lock_held((PyDictObject *)mapping, it); + Py_END_CRITICAL_SECTION(); + if (status < 0) { + goto done; } } else { @@ -2623,6 +2647,7 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, oldval = PyObject_CallFunctionObjArgs(bound_get, key, zero, NULL); if (oldval == NULL) break; + PyObject *newval; if (oldval == zero) { newval = Py_NewRef(one); } else { @@ -2631,9 +2656,10 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, Py_DECREF(oldval); if (newval == NULL) break; - if (PyObject_SetItem(mapping, key, newval) < 0) + int status = PyObject_SetItem(mapping, key, newval); + Py_DECREF(newval); + if (status < 0) break; - Py_CLEAR(newval); Py_DECREF(key); } } @@ -2643,7 +2669,6 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, Py_XDECREF(mapping_setitem); Py_DECREF(it); Py_XDECREF(key); - Py_XDECREF(newval); Py_XDECREF(bound_get); if (PyErr_Occurred()) return NULL; From bf6623aaeb3516606f89851b0ba8b5756f18e9cf Mon Sep 17 00:00:00 2001 From: Neko Asakura Date: Mon, 9 Nov 2026 19:46:43 +0800 Subject: [PATCH 054/229] gh-160782: reject symlinked subdirectories in the venv directory (GH-160973) --- Lib/test/test_venv.py | 18 ++++++++++++++++++ Lib/venv/__init__.py | 6 ++++++ ...6-10-30-10-36-30.gh-issue-160782.n6JCUU.rst | 2 ++ 3 files changed, 26 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-10-30-10-36-30.gh-issue-160782.n6JCUU.rst diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 678615ea172aefa..43c2a552ee90b7f 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -663,6 +663,24 @@ def test_unoverwritable_fails(self): self.assertRaises((ValueError, OSError), venv.create, self.env_dir) self.clear_directory(self.env_dir) + @unittest.skipUnless(can_symlink(), 'Needs symlinks') + def test_symlinked_directory_error(self): + """ + Test that venv creation fails when a directory inside the target + directory is a symlink. + """ + outside = tempfile.mkdtemp() + self.addCleanup(rmtree, outside) + for n in range(1, len(self.lib) + 1): + subdirs = self.lib[:n] + with self.subTest(subdirs=subdirs): + rmtree(self.env_dir) + os.makedirs(os.path.join(self.env_dir, *subdirs[:-1])) + os.symlink(outside, os.path.join(self.env_dir, *subdirs), + target_is_directory=True) + self.assertRaises(ValueError, venv.create, self.env_dir) + self.assertEqual(os.listdir(outside), []) + def test_upgrade(self): """ Test upgrading an existing environment directory. diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 620168be16862ec..60fe070b1f9f4b1 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -141,6 +141,12 @@ def ensure_directories(self, env_dir): """ def create_if_needed(d): + path = env_dir + for part in os.path.relpath(d, env_dir).split(os.sep)[:-1]: + path = os.path.join(path, part) + if os.path.islink(path) or os.path.isjunction(path): + raise ValueError(f'Refusing to create a venv in {env_dir} ' + f'because {path} is a link.') if not os.path.exists(d): os.makedirs(d) elif os.path.islink(d) or os.path.isfile(d): diff --git a/Misc/NEWS.d/next/Library/2026-10-30-10-36-30.gh-issue-160782.n6JCUU.rst b/Misc/NEWS.d/next/Library/2026-10-30-10-36-30.gh-issue-160782.n6JCUU.rst new file mode 100644 index 000000000000000..d4da24bdca0b5e8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-30-10-36-30.gh-issue-160782.n6JCUU.rst @@ -0,0 +1,2 @@ +:mod:`venv`: a symbolic link or junction in place of a directory inside the +environment now raises :exc:`ValueError` instead of being followed. From 6e9a900916e32f4565f0ecebf7b6f48f5a60bc66 Mon Sep 17 00:00:00 2001 From: John Date: Mon, 9 Nov 2026 13:24:57 +0000 Subject: [PATCH 055/229] gh-161366: Fix deadlock with FT GC debug collectables print (#161444) --- Lib/test/test_free_threading/test_gc.py | 30 +++++++++++++++++++ ...-11-05-21-00-57.gh-issue-161366.ouxAvv.rst | 4 +++ Python/gc_free_threading.c | 22 ++++++++------ 3 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-21-00-57.gh-issue-161366.ouxAvv.rst diff --git a/Lib/test/test_free_threading/test_gc.py b/Lib/test/test_free_threading/test_gc.py index b287b51826999c7..8c97b04ed4f71b1 100644 --- a/Lib/test/test_free_threading/test_gc.py +++ b/Lib/test/test_free_threading/test_gc.py @@ -9,6 +9,7 @@ from test import support from test.support import threading_helper +from test.support.script_helper import assert_python_ok class MyObj: @@ -110,6 +111,35 @@ def run(self): self.assertTrue(C().run()) + def test_debug_collectable_python_stderr(self): + # The DEBUG_COLLECTABLE output is written with sys.stderr.write(), + # which may run Python code or wait for another thread (e.g. for the + # lock of a buffered stream), so it must not be written while the + # world is stopped. Debug builds assert that the world is not stopped + # when Python code runs, so use a sys.stderr written in Python. + code = """if 1: + import gc + import io + import sys + + class Stderr(io.StringIO): + def write(self, s): + return super().write(s) + + sys.stderr = Stderr() + gc.set_debug(gc.DEBUG_COLLECTABLE) + a = [] + a.append(a) + del a + gc.collect() + gc.set_debug(0) + output = sys.stderr.getvalue() + sys.stderr = sys.__stderr__ + print(output) + """ + _, out, _ = assert_python_ok("-c", code) + self.assertRegex(out, b"gc: collectable ") + def test_merge_brc_queue_of_detached_thread(self): # GH-157838: objects queued for merging by a thread that is detached # (blocked in a lock acquire, sleep, etc.) are merged and freed on its diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-21-00-57.gh-issue-161366.ouxAvv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-21-00-57.gh-issue-161366.ouxAvv.rst new file mode 100644 index 000000000000000..94098a0cf5195cb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-21-00-57.gh-issue-161366.ouxAvv.rst @@ -0,0 +1,4 @@ +Fix a deadlock in the garbage collector of the :term:`free-threaded build` +when :data:`gc.DEBUG_COLLECTABLE` is set and another thread is writing to +:data:`sys.stderr`. The debug output is now written after the other threads +are resumed. diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 6eaeeea8848cdbf..f4f1a7777f35223 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2158,20 +2158,24 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // Deallocations after the refcount merge may run Python code. bool needs_resurrection_check = state->objs_to_decref.head != 0; - // Print debugging information. - if (FT_ATOMIC_LOAD_INT_RELAXED(interp->gc.debug) & _PyGC_DEBUG_COLLECTABLE) { - needs_resurrection_check = true; - PyObject *op; - WORKSTACK_FOR_EACH(&state->unreachable, op) { - debug_cycle("collectable", op); - } - } + // The debug output is written with sys.stderr.write(), which may run + // Python code, so it is printed below after the world is started. + bool debug_print = (FT_ATOMIC_LOAD_INT_RELAXED(interp->gc.debug) & + _PyGC_DEBUG_COLLECTABLE) != 0; // Find weakref callbacks and pending finalizers (but do not call them). bool has_finalizers = prepare_finalization(state, &needs_resurrection_check); - if (needs_resurrection_check) { + if (needs_resurrection_check || debug_print) { _PyEval_StartTheWorld(interp); + // Print debugging information. + if (debug_print) { + PyObject *op; + WORKSTACK_FOR_EACH(&state->unreachable, op) { + debug_cycle("collectable", op); + } + } + // Deallocate any object from the refcount merge step cleanup_worklist(&state->objs_to_decref); From ff042f13775b0d905b20e425df18d35c2289cb43 Mon Sep 17 00:00:00 2001 From: RM Date: Mon, 9 Nov 2026 14:58:25 +0100 Subject: [PATCH 056/229] gh-122901: Speed up `str` creation from non-ASCII data with a branch-free `find_max_char()` (#122902) Co-authored-by: Stan Ulbrych Co-authored-by: Victor Stinner --- ...-08-11-12-06-23.gh-issue-122901.B2Huom.rst | 1 + Objects/stringlib/find_max_char.h | 231 ++++++++++-------- 2 files changed, 132 insertions(+), 100 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-08-11-12-06-23.gh-issue-122901.B2Huom.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-11-12-06-23.gh-issue-122901.B2Huom.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-11-12-06-23.gh-issue-122901.B2Huom.rst new file mode 100644 index 000000000000000..734bbc63f381ab4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-11-12-06-23.gh-issue-122901.B2Huom.rst @@ -0,0 +1 @@ +Speed up creating :class:`str` objects from non-ASCII character data. diff --git a/Objects/stringlib/find_max_char.h b/Objects/stringlib/find_max_char.h index 7ab3fc88b331b14..8c7da7b7fad38ee 100644 --- a/Objects/stringlib/find_max_char.h +++ b/Objects/stringlib/find_max_char.h @@ -5,123 +5,157 @@ # error "find_max_char.h is specific to Unicode" #endif -/* Mask to quickly check whether a C 'size_t' contains a - non-ASCII, UTF8-encoded char. */ -#if (SIZEOF_SIZE_T == 8) -# define UCS1_ASCII_CHAR_MASK 0x8080808080808080ULL -#elif (SIZEOF_SIZE_T == 4) -# define UCS1_ASCII_CHAR_MASK 0x80808080U -#else -# error C 'size_t' size should be either 4 or 8! -#endif +#define MAX_CHAR_ASCII 0x7f +#define MAX_CHAR_UCS1 0xff +#define MAX_CHAR_UCS2 0xffff +#define MAX_CHAR_UCS4 0x10ffff +/* Masks selecting the bits that are set in a C 'size_t' holding several + characters when at least one of them is wider than ASCII, UCS1 or UCS2. */ #if STRINGLIB_SIZEOF_CHAR == 1 +# if SIZEOF_SIZE_T == 8 +# define MASK_ASCII 0x8080808080808080ULL +# elif SIZEOF_SIZE_T == 4 +# define MASK_ASCII 0x80808080U +# else +# error C 'size_t' size should be either 4 or 8! +# endif +# define MASK_MAX_CHAR MASK_ASCII +# define MAX_CHAR MAX_CHAR_UCS1 +#elif STRINGLIB_SIZEOF_CHAR == 2 +# if SIZEOF_SIZE_T == 8 +# define MASK_ASCII 0xFF80FF80FF80FF80ULL +# define MASK_UCS1 0xFF00FF00FF00FF00ULL +# elif SIZEOF_SIZE_T == 4 +# define MASK_ASCII 0xFF80FF80U +# define MASK_UCS1 0xFF00FF00U +# else +# error C 'size_t' size should be either 4 or 8! +# endif +# define MASK_MAX_CHAR MASK_UCS1 +# define MAX_CHAR MAX_CHAR_UCS2 +#elif STRINGLIB_SIZEOF_CHAR == 4 +# if SIZEOF_SIZE_T == 8 +# define MASK_ASCII 0xFFFFFF80FFFFFF80ULL +# define MASK_UCS1 0xFFFFFF00FFFFFF00ULL +# define MASK_UCS2 0xFFFF0000FFFF0000ULL +# elif SIZEOF_SIZE_T == 4 +# define MASK_ASCII 0xFFFFFF80U +# define MASK_UCS1 0xFFFFFF00U +# define MASK_UCS2 0xFFFF0000U +# else +# error C 'size_t' size should be either 4 or 8! +# endif +# define MASK_MAX_CHAR MASK_UCS2 +# define MAX_CHAR MAX_CHAR_UCS4 +#else +# error Invalid STRINGLIB_SIZEOF_CHAR (must be 1, 2 or 4) +#endif +/* OR all characters into a single size_t accumulator and test the high bits + at the end, instead of comparing each character. The accumulation loop has + no branches, so compilers vectorize it. The start is first read in pieces + until the pointer is aligned, so the loops can use plain word loads on + every platform. Every 16 words, check whether the widest kind is already + known and exit early. */ Py_LOCAL_INLINE(Py_UCS4) STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) { - const unsigned char *p = (const unsigned char *) begin; + const unsigned char *p = (const unsigned char *)begin; const unsigned char *_end = (const unsigned char *)end; + size_t value = 0; - while (p < _end) { - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { - /* Help register allocation */ - const unsigned char *_p = p; - while (_p + SIZEOF_SIZE_T <= _end) { - size_t value = *(const size_t *) _p; - if (value & UCS1_ASCII_CHAR_MASK) - return 255; - _p += SIZEOF_SIZE_T; + if (p + SIZEOF_SIZE_T <= _end) { + if (!_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { +#if STRINGLIB_SIZEOF_CHAR == 1 + if (!_Py_IS_ALIGNED(p, sizeof(uint16_t))) { + value |= *p++; } - p = _p; - if (p == _end) - break; - } - if (*p++ & 0x80) - return 255; - } - return 127; -} - -#undef ASCII_CHAR_MASK - -#else /* STRINGLIB_SIZEOF_CHAR == 1 */ - -#define MASK_ASCII 0xFFFFFF80 -#define MASK_UCS1 0xFFFFFF00 -#define MASK_UCS2 0xFFFF0000 - -#define MAX_CHAR_ASCII 0x7f -#define MAX_CHAR_UCS1 0xff -#define MAX_CHAR_UCS2 0xffff -#define MAX_CHAR_UCS4 0x10ffff - -Py_LOCAL_INLINE(Py_UCS4) -STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) -{ -#if STRINGLIB_SIZEOF_CHAR == 2 - const Py_UCS4 mask_limit = MASK_UCS1; - const Py_UCS4 max_char_limit = MAX_CHAR_UCS2; -#elif STRINGLIB_SIZEOF_CHAR == 4 - const Py_UCS4 mask_limit = MASK_UCS2; - const Py_UCS4 max_char_limit = MAX_CHAR_UCS4; -#else -#error Invalid STRINGLIB_SIZEOF_CHAR (must be 1, 2 or 4) #endif - Py_UCS4 mask; - Py_ssize_t n = end - begin; - const STRINGLIB_CHAR *p = begin; - const STRINGLIB_CHAR *unrolled_end = begin + _Py_SIZE_ROUND_DOWN(n, 4); - Py_UCS4 max_char; - - max_char = MAX_CHAR_ASCII; - mask = MASK_ASCII; - while (p < unrolled_end) { - STRINGLIB_CHAR bits = p[0] | p[1] | p[2] | p[3]; - if (bits & mask) { - if (mask == mask_limit) { - /* Limit reached */ - return max_char_limit; +#if STRINGLIB_SIZEOF_CHAR <= 2 + if (!_Py_IS_ALIGNED(p, sizeof(uint32_t))) { + value |= *(const uint16_t *)p; + p += sizeof(uint16_t); } - if (mask == MASK_ASCII) { - max_char = MAX_CHAR_UCS1; - mask = MASK_UCS1; +#endif +#if SIZEOF_SIZE_T == 8 + if (!_Py_IS_ALIGNED(p, SIZEOF_SIZE_T)) { + value |= *(const uint32_t *)p; + p += sizeof(uint32_t); } - else { - /* mask can't be MASK_UCS2 because of mask_limit above */ - assert(mask == MASK_UCS1); - max_char = MAX_CHAR_UCS2; - mask = MASK_UCS2; +#endif + if (value & MASK_MAX_CHAR) { + return MAX_CHAR; } - /* We check the new mask on the same chars in the next iteration */ - continue; + assert(_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)); } - p += 4; - } - while (p < end) { - if (p[0] & mask) { - if (mask == mask_limit) { - /* Limit reached */ - return max_char_limit; - } - if (mask == MASK_ASCII) { - max_char = MAX_CHAR_UCS1; - mask = MASK_UCS1; + + for (; p + SIZEOF_SIZE_T * 16 <= _end; p += SIZEOF_SIZE_T * 16) { + /* Four independent accumulators so the loop does not serialize on + one register when the compiler does not vectorize it. */ + size_t v0 = 0, v1 = 0, v2 = 0, v3 = 0; + for (int i = 0; i < 16 * SIZEOF_SIZE_T; i += 4 * SIZEOF_SIZE_T) { + const size_t *w = (const size_t *)(p + i); + v0 |= w[0]; + v1 |= w[1]; + v2 |= w[2]; + v3 |= w[3]; } - else { - /* mask can't be MASK_UCS2 because of mask_limit above */ - assert(mask == MASK_UCS1); - max_char = MAX_CHAR_UCS2; - mask = MASK_UCS2; + size_t block = v0 | v1 | v2 | v3; + if (block & MASK_MAX_CHAR) { + return MAX_CHAR; } - /* We check the new mask on the same chars in the next iteration */ - continue; + value |= block; + } + + for (; p + SIZEOF_SIZE_T <= _end; p += SIZEOF_SIZE_T) { + value |= *(const size_t *)p; } - p++; } - return max_char; + + /* Fewer than SIZEOF_SIZE_T bytes are left. Read them as two overlapping + pieces, one from each end. OR does not mind seeing a byte twice. */ + if (p < _end) { + size_t n = (size_t)(_end - p); + if (n >= 4) { + uint32_t a, b; + memcpy(&a, p, 4); + memcpy(&b, _end - 4, 4); + value |= a | b; + } +#if STRINGLIB_SIZEOF_CHAR <= 2 + else if (n >= 2) { + uint16_t a, b; + memcpy(&a, p, 2); + memcpy(&b, _end - 2, 2); + value |= a | b; + } +#endif +#if STRINGLIB_SIZEOF_CHAR == 1 + else { + value |= *p; + } +#endif + } + +#ifdef MASK_UCS2 + if (value & MASK_UCS2) { + return MAX_CHAR_UCS4; + } +#endif +#ifdef MASK_UCS1 + if (value & MASK_UCS1) { + return MAX_CHAR_UCS2; + } +#endif + if (value & MASK_ASCII) { + return MAX_CHAR_UCS1; + } + return MAX_CHAR_ASCII; } +#undef MASK_MAX_CHAR +#undef MAX_CHAR #undef MASK_ASCII #undef MASK_UCS1 #undef MASK_UCS2 @@ -129,6 +163,3 @@ STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) #undef MAX_CHAR_UCS1 #undef MAX_CHAR_UCS2 #undef MAX_CHAR_UCS4 - -#endif /* STRINGLIB_SIZEOF_CHAR == 1 */ - From 37b5cc4b793960d30938ac2e2fa14d8c58bc329a Mon Sep 17 00:00:00 2001 From: Shanzhi Chen Date: Mon, 9 Nov 2026 22:17:16 +0800 Subject: [PATCH 057/229] gh-161467: Enable `__attribute__((hot))` for Clang (#161468) --- Include/pyport.h | 9 ++++----- .../Build/2026-11-06-09-16-30.gh-issue-161467.oSuzx_.rst | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-11-06-09-16-30.gh-issue-161467.oSuzx_.rst diff --git a/Include/pyport.h b/Include/pyport.h index c6cc47f7a14103d..718a9b50bae1039 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -306,16 +306,15 @@ extern "C" { * Usage: * int _Py_HOT_FUNCTION x(void) { return 3; } * - * Issue #28618: This attribute must not be abused, otherwise it can have a + * Issue gh-72804: This attribute must not be abused, otherwise it can have a * negative effect on performance. Only the functions were Python spend most of * its time must use it. Use a profiler when running performance benchmark * suite to find these functions. */ -#if defined(__GNUC__) \ - && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 3)) -#define _Py_HOT_FUNCTION __attribute__((hot)) +#if _Py__has_attribute(hot) +# define _Py_HOT_FUNCTION __attribute__((hot)) #else -#define _Py_HOT_FUNCTION +# define _Py_HOT_FUNCTION #endif // Ask the compiler to always inline a static inline function. The compiler can diff --git a/Misc/NEWS.d/next/Build/2026-11-06-09-16-30.gh-issue-161467.oSuzx_.rst b/Misc/NEWS.d/next/Build/2026-11-06-09-16-30.gh-issue-161467.oSuzx_.rst new file mode 100644 index 000000000000000..ae34c60e989ead1 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-11-06-09-16-30.gh-issue-161467.oSuzx_.rst @@ -0,0 +1 @@ +Enable the use of ``__attribute__((hot))`` when building CPython with Clang and other compilers that support it. From c7fd6522bf37a362cd4a433b4d99295b9683f600 Mon Sep 17 00:00:00 2001 From: Maik Schulze Date: Mon, 9 Nov 2026 15:39:32 +0100 Subject: [PATCH 058/229] gh-161407: Allocate audit hook entries with malloc() (#161409) A hook added with PySys_AddAuditHook() before Py_Initialize() was allocated with PyMem_RawMalloc() before the startup configuration was applied, and freed with PyMem_RawFree() at finalization. When PYTHONMALLOC=mimalloc, -X dev or a debug allocator changed the raw allocator in between, the entry was freed with a different allocator and the process crashed at exit. Allocate and free the entries with plain malloc() and free(), which nothing replaces at startup. --- .../C_API/2026-11-05-10-51-17.gh-issue-161407.rVoqaQ.rst | 3 +++ Python/sysmodule.c | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-11-05-10-51-17.gh-issue-161407.rVoqaQ.rst diff --git a/Misc/NEWS.d/next/C_API/2026-11-05-10-51-17.gh-issue-161407.rVoqaQ.rst b/Misc/NEWS.d/next/C_API/2026-11-05-10-51-17.gh-issue-161407.rVoqaQ.rst new file mode 100644 index 000000000000000..421329930f961a7 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-11-05-10-51-17.gh-issue-161407.rVoqaQ.rst @@ -0,0 +1,3 @@ +Fix a crash at exit when an audit hook was added with :c:func:`PySys_AddAuditHook` +before :c:func:`Py_Initialize` and the raw memory allocator changed during startup, for +example with ``PYTHONMALLOC=mimalloc`` or in the :ref:`Python Development Mode `. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 6f336dac058379e..e8dd956c3f48828 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -448,7 +448,7 @@ _PySys_ClearAuditHooks(PyThreadState *ts) runtime->audit_hooks.head = NULL; while (e) { n = e->next; - PyMem_RawFree(e); + free(e); e = n; } } @@ -496,7 +496,11 @@ PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) } } - _Py_AuditHookEntry *e = (_Py_AuditHookEntry*)PyMem_RawMalloc( + /* Use malloc(), not PyMem_RawMalloc(): a hook can be added before + Py_Initialize(), and the entry is freed during finalization, after + PYTHONMALLOC may have changed the raw allocator (and, in debug builds, + the allocator under the debug hooks). */ + _Py_AuditHookEntry *e = (_Py_AuditHookEntry*)malloc( sizeof(_Py_AuditHookEntry)); if (!e) { if (tstate != NULL) { From 7b1bc410015f1a8dc31fc707e324c44708d14c7f Mon Sep 17 00:00:00 2001 From: Sendao Yan Date: Mon, 9 Nov 2026 23:50:21 +0800 Subject: [PATCH 059/229] gh-159452: Run the regen-configure container as the invoking user (#159454) `Tools/build/regen-configure.sh` ran the Autoconf container as root, so with a rootful Docker daemon the regenerated files and `autom4te.cache` ended up owned by root. The stale cache broke subsequent runs and couldn't be removed without sudo. Run the container as the invoking user instead: `--userns=keep-id` for Podman, `-u UID:GID` for rootful Docker. Rootless runtimes already map container root to the invoking user and need nothing. Also remove `autom4te.cache` before and after the run, using the same image, so an existing root-owned cache gets cleaned up too. --- ...-10-15-17-19-41.gh-issue-159452.oJ1saO.rst | 2 ++ Tools/build/regen-configure.sh | 25 ++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2026-10-15-17-19-41.gh-issue-159452.oJ1saO.rst diff --git a/Misc/NEWS.d/next/Build/2026-10-15-17-19-41.gh-issue-159452.oJ1saO.rst b/Misc/NEWS.d/next/Build/2026-10-15-17-19-41.gh-issue-159452.oJ1saO.rst new file mode 100644 index 000000000000000..531239d3e33f4a4 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-10-15-17-19-41.gh-issue-159452.oJ1saO.rst @@ -0,0 +1,2 @@ +``Tools/build/regen-configure.sh`` no longer leaves +root-owned files in the source tree. diff --git a/Tools/build/regen-configure.sh b/Tools/build/regen-configure.sh index c7683eb36763aff..a7a95b7878dc79a 100755 --- a/Tools/build/regen-configure.sh +++ b/Tools/build/regen-configure.sh @@ -26,4 +26,27 @@ if command -v selinuxenabled >/dev/null && selinuxenabled; then PATH_OPT=":Z" fi -"$RUNTIME" run --rm -v "$abs_srcdir:$WORK_DIR$PATH_OPT" "$IMAGE" +# Run the container as the invoking user so that generated files are owned by +# that user and not by root. Rootless runtimes already map root to the invoking +# user, so a user mapping is only needed for rootful Docker. +USER_OPT="" +case "$RUNTIME" in + podman) + USER_OPT="--userns=keep-id" + ;; + docker) + if ! docker info --format '{{.SecurityOptions}}' | grep -q rootless; then + USER_OPT="-u $(id -u):$(id -g)" + fi + ;; +esac + +purge_autom4te_cache() { + [ -d "$abs_srcdir/autom4te.cache" ] || return 0 + "$RUNTIME" run --rm -v "$abs_srcdir:$WORK_DIR$PATH_OPT" \ + "$IMAGE" sh -c "rm -rf $WORK_DIR/autom4te.cache" +} + +purge_autom4te_cache +trap 'purge_autom4te_cache || true' EXIT +"$RUNTIME" run --rm $USER_OPT -v "$abs_srcdir:$WORK_DIR$PATH_OPT" "$IMAGE" From 9e220787624afba3e20f0067da6dee3e50fcc008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20G=C3=B6rgens?= Date: Mon, 9 Nov 2026 23:58:50 +0800 Subject: [PATCH 060/229] gh-158336: Fix race in lock_tests test_set_and_clear (#158337) test_set_and_clear() started N threads calling event.wait() and then slept 50 ms (wait_threads_blocked) before calling event.set() and event.clear(). If a thread only reached wait() after clear(), it saw the flag unset and blocked for LONG_TIMEOUT, and Bunch.__exit__ gave up after SHORT_TIMEOUT. This happened on CI in the TSan free-threading job. Wait until all N threads are in the Condition's waiter list instead. Condition.wait() appends the waiter while holding the lock, and notify_all() releases every waiter in the list, so each of those threads is guaranteed to be woken by set(). On a TSan free-threading debug build, pinned to 2 CPUs shared with 4 busy-looping processes, the test failed in 9 of 40 runs before this change and 0 of 40 after. Co-authored-by: Victor Stinner --- Lib/test/lock_tests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 47025c041daf1e2..043bcf8f1c1aa34 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -566,6 +566,8 @@ def test_nan_timeout(self): evt = self.eventtype() self.assertRaises(ValueError, evt.wait, float('nan')) + # Use private attributes (event._cond._waiters) + @support.cpython_only def test_set_and_clear(self): # gh-57711: check that wait() returns true even when the event is # cleared before the waiting thread is woken up. @@ -576,8 +578,11 @@ def f(): N = 5 with Bunch(f, N): - # Threads blocked on event.wait() - wait_threads_blocked(N) + # Wait until all threads are registered as waiters in + # event.wait(). + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(event._cond._waiters) >= N: + break # Threads unblocked event.set() From cb2eeed01697bb1a6b29e6dc582e7b172d158b1e Mon Sep 17 00:00:00 2001 From: Doremon <113814705+raman118@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:56:56 +0530 Subject: [PATCH 061/229] gh-160334: Accept special frames in collapsed profiling test (#161515) Co-authored-by: Victor Stinner --- .../test_sampling_profiler/test_integration.py | 5 ++++- .../Tests/2026-11-07-01-45-00.gh-issue-160334.F2n8qA.rst | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-11-07-01-45-00.gh-issue-160334.F2n8qA.rst diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py index 3487647b76683ee..0b9d55bacc6659c 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py @@ -508,7 +508,10 @@ def test_sampling_with_collapsed_export(self): if ";" in stack_trace: stack_parts = stack_trace.split(";") for part in stack_parts: - # Each part should be file:function:line + # Special frames, such as and , + # do not have file and line information. + if part.startswith("<") and part.endswith(">"): + continue self.assertIn(":", part) def test_sampling_all_threads(self): diff --git a/Misc/NEWS.d/next/Tests/2026-11-07-01-45-00.gh-issue-160334.F2n8qA.rst b/Misc/NEWS.d/next/Tests/2026-11-07-01-45-00.gh-issue-160334.F2n8qA.rst new file mode 100644 index 000000000000000..458f192ce0be77d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-11-07-01-45-00.gh-issue-160334.F2n8qA.rst @@ -0,0 +1,2 @@ +Fix the sampling profiler integration test for special frames without file and +line information. From e3262d9c3b9492d22b4f7dc4df903299846bf320 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 9 Nov 2026 09:50:17 -0700 Subject: [PATCH 062/229] gh-160267: Keep thread and interpreter state memory alive for remote profilers (#161674) --- ...-11-09-14-30-00.gh-issue-160267.sTfrEe.rst | 4 + Python/pystate.c | 86 ++++++++++++++++--- Tools/c-analyzer/cpython/ignored.tsv | 1 + 3 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst new file mode 100644 index 000000000000000..c4467c7b64cddf1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst @@ -0,0 +1,4 @@ +Fix a crash when :mod:`profiling.sampling` profiles a process that creates and +destroys threads or subinterpreters quickly. Memory of deleted thread and +interpreter states is now reused for new states instead of being freed until +the runtime is finalized. diff --git a/Python/pystate.c b/Python/pystate.c index 81360f4999849d7..ce7197c4e054283 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -309,9 +309,25 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime, ""); _Py_COMP_DIAG_POP +/* Remote profilers access interpreter and thread states without synchronizing + with their lifetime. Until the runtime is finalized, keep deleted states in + per-type free lists, so that a stale access only reaches a state of the same + type. */ +typedef struct state_header { + struct state_header *next; // Next entry in the free list + void *mem; // Block to free at finalization +} state_header; + +static struct { + PyMutex mutex; + state_header *tstates; + state_header *interpreters; +} state_freelists; + #define LOCKS_INIT(runtime) \ { \ &(runtime)->interpreters.mutex, \ + &state_freelists.mutex, \ &(runtime)->xi.data_lookup.registry.mutex, \ &(runtime)->unicode_state.ids.mutex, \ &(runtime)->imports.extensions.mutex, \ @@ -390,6 +406,19 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) assert(runtime->object_state.interpreter_leaks == 0); #endif gilstate_clear(); + + state_header **lists[] = {&state_freelists.tstates, &state_freelists.interpreters}; + for (size_t i = 0; i < Py_ARRAY_LENGTH(lists); i++) { + PyMutex_LockFlags(&state_freelists.mutex, _Py_LOCK_DONT_DETACH); + state_header *entry = *lists[i]; + *lists[i] = NULL; + PyMutex_Unlock(&state_freelists.mutex); + while (entry != NULL) { + state_header *next = entry->next; + PyMem_RawFree(entry->mem); + entry = next; + } + } } #ifdef HAVE_FORK @@ -445,24 +474,50 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) return _PyStatus_OK(); } -static PyInterpreterState * -alloc_interpreter(void) +static void * +alloc_state(state_header **freelist, size_t size, size_t alignment) { - // Aligned allocation for PyInterpreterState. - // the first word of the memory block is used to store - // the original pointer to be used later to free the memory. - size_t alignment = _Alignof(PyInterpreterState); - size_t allocsize = sizeof(PyInterpreterState) + sizeof(void *) + alignment - 1; + PyMutex_LockFlags(&state_freelists.mutex, _Py_LOCK_DONT_DETACH); + state_header *entry = *freelist; + if (entry != NULL) { + *freelist = entry->next; + } + PyMutex_Unlock(&state_freelists.mutex); + if (entry != NULL) { + return entry + 1; + } + + size_t allocsize = size + sizeof(state_header) + alignment - 1; void *mem = PyMem_RawCalloc(1, allocsize); if (mem == NULL) { return NULL; } - void *ptr = _Py_ALIGN_UP((char *)mem + sizeof(void *), alignment); - ((void **)ptr)[-1] = mem; + void *ptr = _Py_ALIGN_UP((char *)mem + sizeof(state_header), alignment); + ((state_header *)ptr - 1)->mem = mem; assert(_Py_IS_ALIGNED(ptr, alignment)); return ptr; } +static void +recycle_state(state_header **freelist, void *state, size_t size) +{ + memset(state, 0, size); + state_header *entry = (state_header *)state - 1; + + PyMutex_LockFlags(&state_freelists.mutex, _Py_LOCK_DONT_DETACH); + entry->next = *freelist; + // Keep the list consistent for a child forked by another thread. + _Py_atomic_store_ptr_release(freelist, entry); + PyMutex_Unlock(&state_freelists.mutex); +} + +static PyInterpreterState * +alloc_interpreter(void) +{ + return alloc_state(&state_freelists.interpreters, sizeof(PyInterpreterState), + _Alignof(PyInterpreterState)); +} + static void free_interpreter(PyInterpreterState *interp) { @@ -491,7 +546,13 @@ free_interpreter(PyInterpreterState *interp) interp->obmalloc = NULL; } assert(_Py_IS_ALIGNED(interp, _Alignof(PyInterpreterState))); - PyMem_RawFree(((void **)interp)[-1]); + // A daemon thread may still own the embedded initial thread state. + if (interp->_initialized + && _Py_atomic_load_ptr(&interp->threads.preallocated) == NULL) + { + return; + } + recycle_state(&state_freelists.interpreters, interp, sizeof(*interp)); } } @@ -1517,7 +1578,8 @@ alloc_threadstate(PyInterpreterState *interp) // Fall back to the allocator. if (tstate == NULL) { - tstate = PyMem_RawCalloc(1, sizeof(_PyThreadStateImpl)); + tstate = alloc_state(&state_freelists.tstates, sizeof(*tstate), + _Alignof(_PyThreadStateImpl)); if (tstate == NULL) { return NULL; } @@ -1544,7 +1606,7 @@ free_threadstate(_PyThreadStateImpl *tstate) _Py_atomic_store_ptr(&interp->threads.preallocated, tstate); } else { - PyMem_RawFree(tstate); + recycle_state(&state_freelists.tstates, tstate, sizeof(*tstate)); } } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index a51ac686268f411..3863872fd542624 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -56,6 +56,7 @@ Python/pyhash.c - _Py_HashSecret - ## thread-safe hashtable (internal locks) Python/parking_lot.c - buckets - +Python/pystate.c - state_freelists - ## shared empty sentinel for the per-type method cache Python/typecache.c - empty_cache - From ad126c954cc2f42b31bf383ecb5f1bcadff2973d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Tue, 10 Nov 2026 04:01:17 +0800 Subject: [PATCH 063/229] gh-160811: fix TextIOWrapper.tell() after seek() reading past end of buffer (#160812) TextIOWrapper keeps a "bytes to character" ratio as a float so it can optimize operations. If that got to NaN or Infinity as a value then `.tell()` could compute `skip_bytes` larger than the snapshot buffer. In the C implementation this read past the end of the buffer, and _pyio returned a wrong position. --- Lib/_pyio.py | 7 +++-- Lib/test/test_io/test_textio.py | 27 +++++++++++++++++++ ...-10-28-08-12-33.gh-issue-160811.wQ3xT7.rst | 2 ++ Modules/_io/textio.c | 11 ++++++-- 4 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-28-08-12-33.gh-issue-160811.wQ3xT7.rst diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 370b518f1e5e5a6..77239f026efbb17 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -2458,9 +2458,12 @@ def tell(self): # be O(1) in most situations (common decoders, sensible input). # Actually, it will be exactly 1 for fixed-size codecs (all # 8-bit codecs, also UTF-16 and UTF-32). - skip_bytes = int(self._b2cratio * chars_to_skip) + skip_estimate = self._b2cratio * chars_to_skip + if 0 <= skip_estimate <= len(next_input): + skip_bytes = int(skip_estimate) + else: + skip_bytes = 0 skip_back = 1 - assert skip_bytes <= len(next_input) while skip_bytes > 0: decoder.setstate((b'', dec_flags)) # Decode up to temptative start point diff --git a/Lib/test/test_io/test_textio.py b/Lib/test/test_io/test_textio.py index 6e649240f08d5bf..246b2dc6087bd3a 100644 --- a/Lib/test/test_io/test_textio.py +++ b/Lib/test/test_io/test_textio.py @@ -707,6 +707,22 @@ def test_tell_after_readline_with_cr(self): remaining = f.read() self.assertEqual(remaining, "") + def test_tell_after_seek_past_cr(self): + # tell() must tolerate a stale bytes-per-char ratio after seek(). + for data in (b'one\rtwo\rthree\r', + ('ab\r' + '\u20ac' * 15).encode()): + expected = data.decode().replace('\r', '\n') + for i in range(len(expected)): + with self.subTest(data=data, i=i): + f = self.TextIOWrapper(self.BytesIO(data), + encoding='utf-8') + f.read(i) + pos = f.tell() + f.read(1) + f.seek(pos) + self.assertEqual(f.read(1), expected[i]) + f.seek(f.tell()) + self.assertEqual(f.read(), expected[i + 1:]) def test_seek_with_encoder_state(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") @@ -1772,6 +1788,17 @@ def search(name): class PyTextIOWrapperTest(TextIOWrapperTest, PyTestCase): shutdown_error = "LookupError: unknown encoding: ascii" + def test_tell_with_invalid_b2cratio(self): + for ratio in (float('nan'), float('inf'), -float('inf'), 1e100): + with self.subTest(ratio=ratio): + with self.TextIOWrapper(self.BytesIO(b'abc'), + encoding='utf-8') as f: + self.assertEqual(f.read(1), 'a') + f._b2cratio = ratio + position = f.tell() + f.seek(position) + self.assertEqual(f.read(), 'bc') + class IncrementalNewlineDecoderTest: diff --git a/Misc/NEWS.d/next/Library/2026-10-28-08-12-33.gh-issue-160811.wQ3xT7.rst b/Misc/NEWS.d/next/Library/2026-10-28-08-12-33.gh-issue-160811.wQ3xT7.rst new file mode 100644 index 000000000000000..220845de5845a2d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-28-08-12-33.gh-issue-160811.wQ3xT7.rst @@ -0,0 +1,2 @@ +Fix :meth:`io.TextIOWrapper.tell` reading past the end of the buffer after +:meth:`~io.TextIOWrapper.seek`. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index b6f3f4f17119289..4929e41ba9ccb41 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2982,9 +2982,16 @@ _io_TextIOWrapper_tell_impl(textio *self) /* Fast search for an acceptable start point, close to our current pos */ - skip_bytes = (Py_ssize_t) (self->b2cratio * chars_to_skip); + double skip_estimate = self->b2cratio * chars_to_skip; + if (skip_estimate >= 0 && + skip_estimate <= PyBytes_GET_SIZE(next_input)) + { + skip_bytes = (Py_ssize_t)skip_estimate; + } + else { + skip_bytes = 0; + } skip_back = 1; - assert(skip_bytes <= PyBytes_GET_SIZE(next_input)); input = PyBytes_AS_STRING(next_input); while (skip_bytes > 0) { /* Decode up to temptative start point */ From 8c212aa0de538baeaa71f9de73d79f360f1116d3 Mon Sep 17 00:00:00 2001 From: Nishant Bhandari <46300244+n1snt@users.noreply.github.com> Date: Tue, 10 Nov 2026 03:46:40 +0530 Subject: [PATCH 064/229] gh-158525: Avoid reacquiring dict lock when copying in free-threaded build (#158535) Co-authored-by: Kumar Aditya --- Lib/test/test_dict.py | 14 ++++++++++++++ .../2026-10-01-00-00-00.gh-issue-158525.Xw3K9a.rst | 1 + Objects/dictobject.c | 13 +++++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-00-00-00.gh-issue-158525.Xw3K9a.rst diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index ea697a5a920f777..e8b2aeb1d53125c 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1958,6 +1958,20 @@ def test_copy(self): self.assertEqual(d2, frozendict(x=1, y=2)) self.assertEqual(type(d2), frozendict) + def test_copy_iter_override(self): + class CustomReversedDict(frozendict): + def keys(self): + return reversed(list(frozendict.keys(self))) + + __iter__ = keys + + pairs = [('a', 1), ('b', 2), ('c', 3)] + d = CustomReversedDict(pairs) + d2 = d.copy() + self.assertIsNot(d2, d) + self.assertEqual(type(d2), frozendict) + self.assertEqual(list(d2.items()), pairs[::-1]) + def test_merge(self): # test "a | b" operator self.assertEqual(frozendict(x=1) | frozendict(y=2), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-00-00-00.gh-issue-158525.Xw3K9a.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-00-00-00.gh-issue-158525.Xw3K9a.rst new file mode 100644 index 000000000000000..18ab4418020d1f0 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-00-00-00.gh-issue-158525.Xw3K9a.rst @@ -0,0 +1 @@ +Fix slow ``dict.copy()`` for dicts with many deleted entries in the free-threaded build. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 88a8d61828503f7..b8ca74aedc2adfa 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4300,7 +4300,9 @@ static int dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override, PyObject **dupkey) { assert(can_modify_dict(mp)); - ASSERT_DICT_LOCKED(other); + if (!PyFrozenDict_Check(other)) { + ASSERT_DICT_LOCKED(other); + } if (other == mp || other->ma_used == 0) /* a.update(a) or a.update({}); nothing to do */ @@ -4666,7 +4668,14 @@ copy_lock_held_untracked(PyObject *o, int as_frozendict) } if (copy == NULL) return NULL; - if (dict_merge(copy, o, 1, NULL) < 0) { + int res; + if (Py_TYPE(mp)->tp_iter == dict_iter) { + res = dict_dict_merge((PyDictObject *)copy, mp, 1, NULL); + } + else { + res = dict_merge(copy, o, 1, NULL); + } + if (res < 0) { Py_DECREF(copy); return NULL; } From 78407daea0ff0c29e32d2cb03896957a51b294e2 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Mon, 9 Nov 2026 22:21:54 +0000 Subject: [PATCH 065/229] gh-158529: Prevent mimalloc calling system calls directly on Android (#158530) --- .../2026-09-30-21-29-54.gh-issue-158529.kZO7mX.rst | 2 ++ Objects/mimalloc/prim/unix/prim.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-21-29-54.gh-issue-158529.kZO7mX.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-21-29-54.gh-issue-158529.kZO7mX.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-21-29-54.gh-issue-158529.kZO7mX.rst new file mode 100644 index 000000000000000..712f26f6493877c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-21-29-54.gh-issue-158529.kZO7mX.rst @@ -0,0 +1,2 @@ +Fixed crash on Android x86_64, API levels 27-28, caused by mimalloc calling +system calls directly. diff --git a/Objects/mimalloc/prim/unix/prim.c b/Objects/mimalloc/prim/unix/prim.c index c66198464e72a21..477e2d8b2274c46 100644 --- a/Objects/mimalloc/prim/unix/prim.c +++ b/Objects/mimalloc/prim/unix/prim.c @@ -78,10 +78,11 @@ terms of the MIT license. A copy of the license can be found in the file // Use syscalls for some primitives to allow for libraries that override open/read/close etc. // and do allocation themselves; using syscalls prevents recursion when mimalloc is // still initializing (issue #713) +// Not used on Android, see issue python/cpython/pull/158530 // Declare inline to avoid unused function warnings. //------------------------------------------------------------------------------------ -#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access) +#if defined(MI_HAS_SYSCALL_H) && defined(SYS_open) && defined(SYS_close) && defined(SYS_read) && defined(SYS_access) && !defined(__ANDROID__) static inline int mi_prim_open(const char* fpath, int open_flags) { return syscall(SYS_open,fpath,open_flags,0); From b4201c7a502be3406fa5ebf99241402f994f4b82 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 9 Nov 2026 15:52:10 -0700 Subject: [PATCH 066/229] GH-161367: Introduce `Platforms/WASI test` command (#161435) Co-authored-by: Hood Chatham --- .github/workflows/reusable-wasi.yml | 4 +- ...-11-04-16-34-48.gh-issue-161367.bO5MGy.rst | 3 + Platforms/WASI/README.md | 31 +++- Platforms/WASI/__main__.py | 41 ++++- Platforms/WASI/_build.py | 157 +++--------------- Platforms/WASI/_package.py | 62 +++---- Platforms/WASI/_shared.py | 146 ++++++++++++++++ Platforms/WASI/_test.py | 73 ++++++++ 8 files changed, 334 insertions(+), 183 deletions(-) create mode 100644 Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst create mode 100644 Platforms/WASI/_test.py diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 12dd790897ad392..5b2fda1fe4f82dc 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -64,6 +64,4 @@ jobs: - name: "Display build info of the host/WASI Python" run: python3 Platforms/WASI pythoninfo-host - name: "Test" - run: | - CROSS_BUILD_WASI=$(python3 Platforms/WASI path) - make --directory "${CROSS_BUILD_WASI}" test + run: python3 Platforms/WASI test diff --git a/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst b/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst new file mode 100644 index 000000000000000..7d3e16444338007 --- /dev/null +++ b/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst @@ -0,0 +1,3 @@ +Add a ``test`` command to :file:`Platforms/WASI` with ``--fast-ci`` and +``--slow-ci`` to select fast CI or slow Buildbot tests and ``--target`` to +test the build, gathered distribution, or release archive. diff --git a/Platforms/WASI/README.md b/Platforms/WASI/README.md index 5fe610615aec336..c99b628d7b88a7e 100644 --- a/Platforms/WASI/README.md +++ b/Platforms/WASI/README.md @@ -199,7 +199,36 @@ python3 Platforms/WASI clean ### Testing -To find out where the WASI build directory is, you can run: +After building, you can run the CI tests with: + +```shell +python3 Platforms/WASI test +``` + +The mutually exclusive CI flags to select which tests to run: + +- `--fast-ci` (the default): `make test` +- `--slow-ci`: `make buildbottest` + +The `--target` flag selects the files to test: + +- `build` (the default): the WASI build in `cross-build/`. +- `dist`: the distribution files gathered in `dist/`. +- `archive`: the contents of the release archive, unpacked into `dist/`. + +To thoroughly test the contents of the release archive, use: + +```shell +python3 Platforms/WASI test --slow-ci --target archive +``` + +This requires an archive created by `package` or `release`. It deletes the +selected versioned distribution directory inside `dist/`, then restores it +from the matching `.tar.xz` archive before running the tests. +The archive and other distributions in `dist/` are left untouched. + +You can also run tests more directly. To find out where the WASI build directory +is, you can run: ```shell python3 Platforms/WASI path diff --git a/Platforms/WASI/__main__.py b/Platforms/WASI/__main__.py index 0d504d43283297a..2fea2b7d4476cb0 100644 --- a/Platforms/WASI/__main__.py +++ b/Platforms/WASI/__main__.py @@ -5,6 +5,7 @@ "_build", "_package", "_shared", + "_test", ] import argparse @@ -14,6 +15,7 @@ import _build import _package import _shared +import _test def main(): @@ -76,6 +78,30 @@ def main(): release = subcommands.add_parser( "release", help="Build and package for a release" ) + test = subcommands.add_parser("test", help="Run the tests") + ci_mode = test.add_mutually_exclusive_group() + ci_mode.add_argument( + "--fast-ci", + action="store_const", + dest="ci_mode", + const="fast", + default="fast", + help="Run `make test` (the default)", + ) + ci_mode.add_argument( + "--slow-ci", + action="store_const", + dest="ci_mode", + const="slow", + help="Run `make buildbottest`", + ) + test.add_argument( + "--target", + choices={"build", "dist", "archive"}, + default="build", + help="Test the build (the default), gathered distribution, or release archive; " + "'archive' replaces the distribution directory with the archive's contents", + ) path = subcommands.add_parser( "path", help="Print the path to a build or distribution directory" ) @@ -100,6 +126,7 @@ def main(): pythoninfo_host, build_host, release, + test, ): subcommand.add_argument( "--quiet", @@ -129,6 +156,13 @@ def main(): dest="clean", help="Delete any relevant directories before building", ) + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "--release", + help="Configure build for release mode", + action="store_true", + default=False, + ) for subcommand in ( build, configure_build, @@ -137,7 +171,7 @@ def main(): build_host, ): subcommand.add_argument( - "args", nargs="*", help="Extra arguments to pass to `configure`" + "_args", nargs="*", help="Extra arguments to pass to `configure`" ) for subcommand in build, configure_host, build_host, release: subcommand.add_argument( @@ -167,6 +201,7 @@ def main(): package, path, release, + test, ): subcommand.add_argument( "--host-triple", @@ -219,7 +254,7 @@ def main(): _package.gather(context) _package.archive(context) case "release": - context.args = ["--without-static-libpython"] + context.release = True _build.clean_contents(context) _build.configure_build_python(context) _build.make_build_python(context) @@ -227,6 +262,8 @@ def main(): _build.make_wasi_python(context) _package.gather(context) _package.archive(context) + case "test": + _test.test(context) case "path": paths = { "build-python": "build_python_path", diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py index 9ea92a58ef5bb34..8efda324c4b048a 100644 --- a/Platforms/WASI/_build.py +++ b/Platforms/WASI/_build.py @@ -1,22 +1,16 @@ __lazy_modules__ = [ - "contextlib", - "functools", "os", "pathlib", "shutil", "subprocess", "sys", - "tempfile", ] -import contextlib -import functools import os import pathlib import shutil import subprocess import sys -import tempfile try: from os import process_cpu_count as cpu_count @@ -29,131 +23,19 @@ WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" -def separator(): - """Print a separator line across the terminal width.""" - try: - tput_output = subprocess.check_output( - ["tput", "cols"], encoding="utf-8" - ) - except subprocess.CalledProcessError: - terminal_width = 80 - else: - terminal_width = int(tput_output.strip()) - print("⎯" * terminal_width) - - -def updated_env(updates={}): - """Create a new dict representing the environment to use. - - The changes made to the execution environment are printed out. - """ - env_defaults = {} - # https://reproducible-builds.org/docs/source-date-epoch/ - git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] - try: - epoch = subprocess.check_output( - git_epoch_cmd, encoding="utf-8" - ).strip() - env_defaults["SOURCE_DATE_EPOCH"] = epoch - except subprocess.CalledProcessError: - pass # Might be building from a tarball. - # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. - environment = env_defaults | os.environ | updates - - env_diff = {} - for key, value in environment.items(): - if os.environ.get(key) != value: - env_diff[key] = value - - env_vars = [ - f"\n {key}={item}" for key, item in sorted(env_diff.items()) - ] - _shared.log("🌎", f"Environment changes:{''.join(env_vars)}") - - return environment - - -def subdir(context_attr, *, clean_ok=False): - """Decorator to change to a working directory.""" - - def decorator(func): - @functools.wraps(func) - def wrapper(context): - nonlocal context_attr - - working_dir = getattr(context, context_attr) - separator() - _shared.log("📁", os.fsdecode(working_dir)) - if clean_ok and context.clean and working_dir.exists(): - _shared.log("🚮", "Deleting directory (--clean)...") - shutil.rmtree(working_dir) - - working_dir.mkdir(parents=True, exist_ok=True) - - with contextlib.chdir(working_dir): - return func(context, working_dir) - - return wrapper - - return decorator - - -def call(command, *, context=None, quiet=False, **kwargs): - """Execute a command. - - If 'quiet' is true, then redirect stdout and stderr to a temporary file. - """ - if context is not None: - quiet = context.quiet - - _shared.log("❯", " ".join(map(str, command)), spacing=" ") - if not quiet: - stdout = None - stderr = None - else: - if (log_path := getattr(context, "log_path", None)) is None: - log_path = pathlib.Path(tempfile.gettempdir()) - log_path.mkdir(parents=True, exist_ok=True) - stdout = tempfile.NamedTemporaryFile( - "w", - encoding="utf-8", - delete=False, - dir=log_path, - prefix="cpython-wasi-", - suffix=".log", - ) - stderr = subprocess.STDOUT - _shared.log("📝", f"Logging output to {stdout.name} (--quiet)...") - - try: - subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) - except subprocess.CalledProcessError as error: - if quiet: - _shared.log("❌", f"Exit code {error.returncode}") - separator() - with open(stdout.name, encoding="utf-8") as file: - lines = file.readlines() - # Inefficient, but the log shouldn't be dramatically large. - print("".join(lines[-10:]), end="") - if not lines[-1].endswith("\n"): - print() - sys.exit(error.returncode) - - -@subdir("build_python_path", clean_ok=True) +@_shared.subdir("build_python_path", clean_ok=True) def configure_build_python(context, working_dir): """Configure the build/host Python.""" configure = [os.path.relpath(context.checkout / "configure", working_dir)] - if context.args: - configure.extend(context.args) + configure.extend(context.configure_args) - call(configure, context=context) + context.call(configure) -@subdir("build_python_path") +@_shared.subdir("build_python_path") def make_build_python(context, _working_dir): """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], context=context) + context.call(["make", "--jobs", str(cpu_count()), "all"]) binary = context.build_python_interpreter cmd = [ @@ -206,7 +88,7 @@ def wasi_sdk_env(context): return env -@subdir("wasi_build_path", clean_ok=True) +@_shared.subdir("wasi_build_path", clean_ok=True) def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" config_site = os.fsdecode(context.here / "config.site-wasm32-wasi") @@ -247,12 +129,10 @@ def configure_wasi_python(context, working_dir): ] if context.is_debug: configure.append("--with-pydebug") - if context.args: - configure.extend(context.args) - call( + configure.extend(context.configure_args) + context.call( configure, - env=updated_env(env_additions | wasi_sdk_env(context)), - context=context, + env=_shared.updated_env(env_additions | wasi_sdk_env(context)), ) python_wasm = working_dir / "python.wasm" @@ -264,17 +144,18 @@ def configure_wasi_python(context, working_dir): sys.stdout.flush() -@subdir("wasi_build_path") +@_shared.subdir("wasi_build_path") def make_wasi_python(context, working_dir): """Run `make` for the WASI/host build.""" - call( + context.call( ["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - context=context, + env=_shared.updated_env(), ) exec_script = working_dir / "python.sh" - call([exec_script, "-c", "import sys; print(sys.version)"], quiet=False) + context.call( + [exec_script, "-c", "import sys; print(sys.version)"], quiet=False + ) _shared.log( "🎉", f"Use `{exec_script.relative_to(pathlib.Path().absolute())}` " @@ -291,13 +172,13 @@ def clean_contents(context): shutil.rmtree(path) -@subdir("build_python_path") +@_shared.subdir("build_python_path") def pythoninfo_build_python(context, working_dir): """Display build info of the build Python.""" - call(["make", "pythoninfo"], context=context) + context.call(["make", "pythoninfo"]) -@subdir("wasi_build_path") +@_shared.subdir("wasi_build_path") def pythoninfo_wasi_python(context, working_dir): """Display build info of the host/WASI Python.""" - call(["make", "pythoninfo"], context=context) + context.call(["make", "pythoninfo"]) diff --git a/Platforms/WASI/_package.py b/Platforms/WASI/_package.py index 862ffb30130d048..403247d4734a3d6 100644 --- a/Platforms/WASI/_package.py +++ b/Platforms/WASI/_package.py @@ -26,32 +26,18 @@ def wasmtime_script(context): script_dir=$(CDPATH= cd "$(dirname "$0")" && pwd -P) root=$(CDPATH= cd "$script_dir/.." && pwd -P) -wasm_file="$root/lib/python{python_version(context)}/lib-wasm/python{python_version(context, debug_ok=True)}.wasm" +wasm_file="$root/lib/python{context.python_version()}/lib-wasm/python{context.python_version(debug_ok=True)}.wasm" exec wasmtime run \\ --argv0 "$wasm_file" \\ - --config "$root/etc/python{python_version(context)}/wasmtime.toml" \\ + --config "$root/etc/python{context.python_version()}/wasmtime.toml" \\ --dir "/" \\ "$wasm_file" "$@" """ -def python_version(context, debug_ok=False): - """Calculate the M.N part of Python's version. - - If *debug_ok* is True, then "d" is appended as appropriate. - """ - details = context.wasi_build_details - major = details["language"]["version_info"]["major"] - minor = details["language"]["version_info"]["minor"] - version = f"{major}.{minor}" - if debug_ok and context.is_debug: - version += "d" - return version - - def lib_python(context): - return pathlib.PurePath("lib") / f"python{python_version(context)}" + return pathlib.PurePath("lib") / f"python{context.python_version()}" def license_file(context): @@ -101,12 +87,12 @@ def pkgconfig_files(context): pkgconfig = pathlib.PurePath("lib") / "pkgconfig" return [ ( - pkgconfig / f"python-{python_version(context, debug_ok=True)}.pc", + pkgconfig / f"python-{context.python_version(debug_ok=True)}.pc", misc_dir / "python.pc", ), ( pkgconfig - / f"python-{python_version(context, debug_ok=True)}-embed.pc", + / f"python-{context.python_version(debug_ok=True)}-embed.pc", misc_dir / "python-embed.pc", ), ] @@ -127,11 +113,11 @@ def pkgconfig_symlinks(pkgconfig_files, context): paths += [ ( embed_path.parent - / f"python-{python_version(context)}-embed.pc", + / f"python-{context.python_version()}-embed.pc", embed_path, ), ( - plain_path.parent / f"python-{python_version(context)}.pc", + plain_path.parent / f"python-{context.python_version()}.pc", plain_path, ), ] @@ -143,7 +129,7 @@ def pyconfig_file(context): """Have /pyconfig.h end up in include/pythonXYd?/pyconfig.h.""" return ( pathlib.PurePath("include") - / f"python{python_version(context, debug_ok=True)}" + / f"python{context.python_version(debug_ok=True)}" / "pyconfig.h", context.wasi_build_path / "pyconfig.h", ) @@ -158,7 +144,7 @@ def header_files(context): file_path = pathlib.Path(root) / filename details = ( pathlib.PurePath("include") - / f"python{python_version(context, debug_ok=True)}" + / f"python{context.python_version(debug_ok=True)}" / file_path.relative_to(include_dir), file_path, ) @@ -171,7 +157,7 @@ def man_file(context): man_dir = pathlib.PurePath("share", "man", "man1") man_file = context.checkout / "Misc" / "python.man" return ( - man_dir / f"python{python_version(context)}.1", + man_dir / f"python{context.python_version()}.1", man_file, ) @@ -187,7 +173,7 @@ def wasmtime_config_file(context): config = context.checkout / "Platforms" / "WASI" / "wasmtime.toml" return ( pathlib.PurePath("etc") - / f"python{python_version(context)}" + / f"python{context.python_version()}" / "wasmtime.toml", config, ) @@ -198,17 +184,15 @@ def wasm_file(context): return ( lib_python(context) / "lib-wasm" - / f"python{python_version(context, debug_ok=True)}.wasm", + / f"python{context.python_version(debug_ok=True)}.wasm", context.wasi_build_path / "python.wasm", ) -def python_wasmtime_script(base, context): +def python_wasmtime_script(context): """Create bin/pythonN.Md?.wasmtime.""" script = wasmtime_script(context) - path = ( - base / f"bin/python{python_version(context, debug_ok=True)}.wasmtime" - ) + path = context.dist_python_interpreter path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w") as f: f.write(script) @@ -221,7 +205,7 @@ def python_wasmtime_symlink(path, context): """Symlink bin/pythonN.Md?.wasmtime. - bin/pythonN.M.wasmtime (if debug build) - - bin/pythonN.wasmtime (if debug build) + - bin/pythonN.wasmtime """ symlinks = [ path.parent / f"python{context.wasi_build_version['major']}.wasmtime" @@ -229,7 +213,7 @@ def python_wasmtime_symlink(path, context): if context.is_debug: # The file already has the debug name, so the missing symlink is the non-debug name. symlinks.append( - path.parent / f"python{python_version(context)}.wasmtime" + path.parent / f"python{context.python_version()}.wasmtime" ) return [(symlink, path) for symlink in symlinks] @@ -239,7 +223,7 @@ def config_file(context): """Have Misc/config.sh end up at bin/pythonN.Md?.config.""" return ( pathlib.PurePath("bin") - / f"python{python_version(context, debug_ok=True)}-config", + / f"python{context.python_version(debug_ok=True)}-config", context.wasi_build_path / "Misc" / "python-config.sh", ) @@ -258,7 +242,7 @@ def config_symlink(config_path, context): if context.is_debug: # The file already has the debug name, so the missing symlink is the non-debug name. symlinks.append( - config_path.parent / f"python{python_version(context)}-config" + config_path.parent / f"python{context.python_version()}-config" ) return [(symlink, config_path) for symlink in symlinks] @@ -281,8 +265,8 @@ def symlink_files(files, base): def gather(context): - py_version = python_version(context) - py_d_version = python_version(context, debug_ok=True) + py_version = context.python_version() + py_d_version = context.python_version(debug_ok=True) dist = context.dist_path if dist.exists(): @@ -300,7 +284,7 @@ def gather(context): symlink_files(config_symlink(config_location[0], context), base) _shared.log("📄", f"python{py_d_version}.wasmtime", spacing=indent * 3) - script_path = python_wasmtime_script(base, context) + script_path = python_wasmtime_script(context) symlink_files(python_wasmtime_symlink(script_path, context), base) _shared.log("📁", "etc/", spacing=indent * 2) @@ -353,8 +337,7 @@ def gather(context): def archive(context): - file_name = f"{context.archive_stem}.tar.xz" - file_path = context.dist_path / file_name + file_path = context.archive_path if file_path.exists(): _shared.log("🧹", f"Deleting {file_path} ...") file_path.unlink() @@ -399,6 +382,7 @@ def archive(context): "--owner=0", "--group=0", "--numeric-owner", + "--format=pax", "--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime", "--mode=go+u,go-w", # Explicitly using `-T` because if you don't compress with threads you can't diff --git a/Platforms/WASI/_shared.py b/Platforms/WASI/_shared.py index 306aabd1f1c0977..8e5f13d03adc57a 100644 --- a/Platforms/WASI/_shared.py +++ b/Platforms/WASI/_shared.py @@ -1,19 +1,25 @@ __lazy_modules__ = [ + "contextlib", "functools", "json", "os", "pathlib", + "shutil", "subprocess", + "sys", "sysconfig", "tempfile", "tomllib", ] +import contextlib import functools import json import os import pathlib +import shutil import subprocess +import sys import sysconfig import tempfile import tomllib @@ -178,6 +184,13 @@ def wasi_sdk_path(self): return wasi_sdk_path + @functools.cached_property + def configure_args(self): + args = getattr(self, "_args", []) + if self.release: + args.extend(["--without-static-libpython"]) + return args + @functools.cached_property def log_path(self): if self._log_path is not None: @@ -206,6 +219,83 @@ def archive_stem(self): def archive_dir(self): return self.dist_path / self.archive_stem + @functools.cached_property + def archive_path(self): + return self.dist_path / f"{self.archive_stem}.tar.xz" + + @functools.cached_property + def dist_python_interpreter(self): + version = self.python_version(debug_ok=True) + return self.archive_dir / "bin" / f"python{version}.wasmtime" + + def python_version(self, debug_ok=False): + """Calculate the M.N part of Python's version. + + If *debug_ok* is True, then "d" is appended as appropriate. + """ + major = self.wasi_build_version["major"] + minor = self.wasi_build_version["minor"] + version = f"{major}.{minor}" + if debug_ok and "d" in self.wasi_build_details["abi"]["flags"]: + version += "d" + return version + + def call(self, command, *, env=None, quiet=None, **kwargs): + """Execute a command. + + Print changes to the execution environment if 'env' is provided. + If 'quiet' is None, use the context's setting. If true, redirect + stdout and stderr to a temporary file. + """ + if quiet is None: + quiet = self.quiet + + if env is not None: + env_diff = {} + for key, value in env.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + env_vars = [ + f"\n {key}={item}" + for key, item in sorted(env_diff.items()) + ] + log("🌎", f"Environment changes:{''.join(env_vars)}") + + log("❯", " ".join(map(str, command)), spacing=" ") + if not quiet: + stdout = None + stderr = None + else: + log_path = self.log_path + log_path.mkdir(parents=True, exist_ok=True) + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + dir=log_path, + prefix="cpython-wasi-", + suffix=".log", + ) + stderr = subprocess.STDOUT + log("📝", f"Logging output to {stdout.name} (--quiet)...") + + try: + subprocess.check_call( + command, **kwargs, env=env, stdout=stdout, stderr=stderr + ) + except subprocess.CalledProcessError as error: + if quiet: + log("❌", f"Exit code {error.returncode}") + separator() + with open(stdout.name, encoding="utf-8") as file: + lines = file.readlines() + # Inefficient, but the log shouldn't be dramatically large. + print("".join(lines[-10:]), end="") + if not lines[-1].endswith("\n"): + print() + sys.exit(error.returncode) + def log(emoji, message, *, spacing=None): """Print a notification with an emoji. @@ -216,3 +306,59 @@ def log(emoji, message, *, spacing=None): if spacing is None: spacing = " " if len(emoji) == 1 else " " print("".join([emoji, spacing, message])) + + +def separator(): + """Print a separator line across the terminal width.""" + try: + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) + except subprocess.CalledProcessError: + terminal_width = 80 + else: + terminal_width = int(tput_output.strip()) + print("⎯" * terminal_width) + + +def updated_env(updates={}): + """Create a new dict representing the environment to use.""" + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + return environment + + +def subdir(context_attr, *, clean_ok=False): + """Decorator to change to a working directory.""" + + def decorator(func): + @functools.wraps(func) + def wrapper(context): + nonlocal context_attr + + working_dir = getattr(context, context_attr) + separator() + log("📁", os.fsdecode(working_dir)) + if clean_ok and context.clean and working_dir.exists(): + log("🚮", "Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator diff --git a/Platforms/WASI/_test.py b/Platforms/WASI/_test.py new file mode 100644 index 000000000000000..a44838659c70111 --- /dev/null +++ b/Platforms/WASI/_test.py @@ -0,0 +1,73 @@ +__lazy_modules__ = [ + "shlex", + "shutil", + "_shared", +] + +import shlex +import shutil + +import _shared + + +def dist_test_args(context): + runner = context.dist_python_interpreter + if not runner.is_file(): + raise FileNotFoundError( + f"Packaged WASI interpreter not found at {runner}; " + "run `gather`, `package`, or `release` first" + ) + + test_opts = shlex.join(["--python", shlex.quote(str(runner))]) + return [ + "TESTPYTHON=$(RUNSHARED) $(PYTHON_FOR_FREEZE) $(TESTPYTHONOPTS)", + f"TESTOPTS=$(EXTRATESTOPTS) {test_opts}", + ] + + +@_shared.subdir("wasi_build_path") +def test(context, _working_dir): + match context.ci_mode: + case "fast": + make_target = "test" + case "slow": + make_target = "buildbottest" + case _: + raise ValueError(f"Unknown CI mode {context.ci_mode}") + command = ["make", make_target] + match context.target: + case "archive": + archive = context.archive_path + if not archive.is_file(): + raise FileNotFoundError( + f"Release archive not found at {archive}; " + "run `package` or `release` first" + ) + if context.archive_dir.exists(): + _shared.log("🧹", f"Deleting {context.archive_dir} ...") + shutil.rmtree(context.archive_dir) + context.call( + [ + "tar", + "--extract", + "--file", + archive, + "--directory", + context.dist_path, + "--", + context.archive_dir.name, + ], + ) + command.extend(dist_test_args(context)) + case "dist": + if not context.dist_python_interpreter.is_file(): + raise FileNotFoundError( + f"Python not found at {context.dist_python_interpreter}" + ) + command.extend(dist_test_args(context)) + case "build": + pass + case _: + raise ValueError(f"Unknown target {context.target}") + + context.call(command, env=_shared.updated_env()) From 4e10d4fef5c073faaaf04ce5e03749a5dbe7716b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 10 Nov 2026 00:25:19 +0100 Subject: [PATCH 067/229] gh-111178: Set function=False by default in support.check_sanitizer() (#161640) Update also libregrtest.utils to detect function sanitizer (-fsanitize=function). --- Lib/test/libregrtest/utils.py | 8 +++++++- Lib/test/support/__init__.py | 14 ++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 0b8f7f51395a643..41a798c71089f9e 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -414,6 +414,9 @@ def get_build_info(): # --with-thread-sanitizer if support.check_sanitizer(thread=True): sanitizers.append("TSAN") + # --with-thread-sanitizer + if support.check_sanitizer(function=True): + sanitizers.append("fsanitize=function") if sanitizers: build.append('+'.join(sanitizers)) @@ -701,6 +704,7 @@ def display_header(use_resources: dict[str, str | None], msan = support.check_sanitizer(memory=True) ubsan = support.check_sanitizer(ub=True) tsan = support.check_sanitizer(thread=True) + function_san = support.check_sanitizer(function=True) sanitizers = [] if asan: sanitizers.append("address") @@ -710,12 +714,14 @@ def display_header(use_resources: dict[str, str | None], sanitizers.append("undefined behavior") if tsan: sanitizers.append("thread") + if function_san: + sanitizers.append("function") if sanitizers: print(f"== sanitizers: {', '.join(sanitizers)}") for sanitizer, env_var in ( (asan, "ASAN_OPTIONS"), (msan, "MSAN_OPTIONS"), - (ubsan, "UBSAN_OPTIONS"), + (ubsan or function_san, "UBSAN_OPTIONS"), (tsan, "TSAN_OPTIONS"), ): options= os.environ.get(env_var) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index df3ae0890c0f126..f69734037a8fe57 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -462,11 +462,11 @@ def skip_if_buildbot(reason=None): return unittest.skipIf(isbuildbot, reason) def check_sanitizer(*, address=False, hwaddress=False, memory=False, ub=False, - thread=False, function=True): + thread=False, function=False): """Returns True if Python is compiled with sanitizer support""" - if not (address or hwaddress or memory or ub or thread): - raise ValueError('At least one of address, hwaddress, memory, ub or ' - 'thread must be True') + if not (address or hwaddress or memory or ub or thread or function): + raise ValueError('At least one of address, hwaddress, memory, ub, ' + 'thread or function must be True') cflags = sysconfig.get_config_var('CFLAGS') or '' @@ -517,11 +517,13 @@ def configured_with_sanitizer(name): ) -def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False, thread=False): +def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False, + thread=False, function=False): """Decorator raising SkipTest if running with a sanitizer active.""" if not reason: reason = 'not working with sanitizers active' - skip = check_sanitizer(address=address, memory=memory, ub=ub, thread=thread) + skip = check_sanitizer(address=address, memory=memory, ub=ub, + thread=thread, function=function) return unittest.skipIf(skip, reason) # gh-89363: True if fork() can hang if Python is built with Address Sanitizer From e891a9117985888df3e5d48145cf07ddda098335 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 10 Nov 2026 00:49:01 +0100 Subject: [PATCH 068/229] gh-161705: Prevent shadowing stdlib in gzip/tarfile/zipfile CLI (#161706) Remove sys.path[0] in gzip, tarfile and zipfile command line interface to prevent shadowing standard library modules from extracted files. * Add sys._path0 variable. * pymain_sys_path_add_path0() sets sys._path0. * Add runpy._remove_sys_path0(). * gzip, tarfile and zipfile main functions now call _remove_sys_path0() to prevent shadowing stdlib modules from extracted files. Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> Co-authored-by: Ethan Furman Co-authored-by: Kirill Podoprigora --- Doc/c-api/init_config.rst | 4 +- Doc/library/sys.rst | 18 ++++ Doc/using/cmdline.rst | 24 +++-- Lib/gzip.py | 5 + Lib/runpy.py | 20 ++++ Lib/tarfile.py | 4 + Lib/test/test_runpy.py | 91 +++++++++++++++++++ Lib/test/test_sys.py | 9 ++ Lib/test/test_zipfile/test_core.py | 22 ++++- Lib/zipfile/__init__.py | 4 + ...-11-07-15-56-47.gh-issue-161705.OjDK2d.rst | 6 ++ Modules/main.c | 4 + Python/pathconfig.c | 2 +- Python/sysmodule.c | 2 + 14 files changed, 194 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-11-07-15-56-47.gh-issue-161705.OjDK2d.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index f9190cbafc6a24e..cc96ba67c4e85be 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1227,8 +1227,8 @@ PyConfig .. c:member:: int safe_path - If equals to zero, ``Py_RunMain()`` prepends a potentially unsafe path to - :data:`sys.path` at startup: + If equals to zero, ``Py_RunMain()`` prepends a potentially unsafe + :data:`sys._path0` to :data:`sys.path` at startup: * If :c:member:`argv[0] ` is equal to ``L"-m"`` (``python -m module``), prepend the current working directory. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 33de85b04f4b0ea..f08a06d3edae9ba 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1651,6 +1651,24 @@ always available. Unless explicitly noted otherwise, all variables are read-only * Module :mod:`site` This describes how to use .pth files to extend :data:`sys.path`. +.. data:: _path0 + + Path prepended to :data:`sys.path` at Python startup: + + * Current directory if running a module + with ``python -m module`` command line. + * Script's directory if running a script + with ``python script.py``. + * Otherwise an empty string, which means the current working directory, + for ``python -c code`` or ``python`` (REPL) command line. + + If :option:`-P` or :option:`-I` command line options are used or if + :envvar:`PYTHONSAFEPATH` environment variable is used, :data:`sys._path0` is + ``None`` and :data:`sys.path` is left unchanged. + + .. versionadded:: next + + .. data:: path_hooks A list of callables that take a path argument to try to create a diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index e0ad7945a8ecef2..ffb2587840be5a1 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -322,10 +322,12 @@ Miscellaneous options Run Python in isolated mode. This also implies :option:`-E`, :option:`-P` and :option:`-s` options. - In isolated mode :data:`sys.path` contains neither the script's directory nor - the user's site-packages directory. All ``PYTHON*`` environment - variables are ignored, too. Further restrictions may be imposed to prevent - the user from injecting malicious code. + In isolated mode, Python does not prepend the potentially unsafe path + :data:`sys._path0` to :data:`sys.path`, and ignores all ``PYTHON*`` + environment variables, e.g. :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, + that might be set. + Further restrictions may be imposed to prevent the user from injecting + malicious code. .. versionadded:: 3.4 @@ -353,14 +355,8 @@ Miscellaneous options .. option:: -P - Don't prepend a potentially unsafe path to :data:`sys.path`: - - * ``python -m module`` command line: Don't prepend the current working - directory. - * ``python script.py`` command line: Don't prepend the script's directory. - If it's a symbolic link, resolve symbolic links. - * ``python -c code`` and ``python`` (REPL) command lines: Don't prepend an - empty string, which means the current working directory. + Don't prepend the potentially unsafe path :data:`sys._path0` to + :data:`sys.path`. See also the :envvar:`PYTHONSAFEPATH` environment variable, and :option:`-E` and :option:`-I` (isolated) options. @@ -820,7 +816,9 @@ conflict. .. envvar:: PYTHONSAFEPATH If this is set to a non-empty string, don't prepend a potentially unsafe - path to :data:`sys.path`: see the :option:`-P` option for details. + path :data:`sys._path0` to :data:`sys.path`. + + See also the :option:`-P` option. .. versionadded:: 3.11 diff --git a/Lib/gzip.py b/Lib/gzip.py index 3ed2055d801a3c8..48c4ae4920f6d35 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -746,6 +746,11 @@ def opener(path, flags): def main(): from argparse import ArgumentParser + import runpy + + # Prevent shadowing a stdlib module from the extracted file + runpy._remove_sys_path0() + parser = ArgumentParser(description= "A simple command line interface for the `gzip` module: act like `gzip`, " "but do not delete the input file.", diff --git a/Lib/runpy.py b/Lib/runpy.py index a535b4f651a5ae8..ff639369da08a1c 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -315,6 +315,26 @@ def run_path(path_name, init_globals=None, run_name=None): pass +def _remove_sys_path0(): + """ + Remove sys._path0 from sys.path. Do nothing if sys._path0 is + sys._stdlib_dir or if it has already been removed. Similar to using -P + (safe path) command line option. + """ + path0 = sys._path0 + if path0 is None: + return + + if path0 == sys._stdlib_dir: + return + + # Remove all path0 copies, not only the first one. + # We are intentionally not trying to normalize & match alternate + # representations of path0, this is only intended to remove copies + # of the exact value the runtime itself may have added. + sys.path[:] = [path for path in sys.path if path != path0] + + if __name__ == "__main__": # Run the module specified as the next command line argument if len(sys.argv) < 2: diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 227a28766b634d6..abf7786812a0634 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -3172,6 +3172,10 @@ def is_tarfile(name): def main(): import argparse + import runpy + + # Prevent shadowing stdlib modules from extracted files + runpy._remove_sys_path0() description = 'A simple command-line interface for tarfile module.' parser = argparse.ArgumentParser(description=description) diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 55b9673ef6c91c0..f6aae9bed570d21 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -21,6 +21,7 @@ verbose, ) from test import support +from test.support import os_helper from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script @@ -917,5 +918,95 @@ def test_pymain_run_module(self): self.assertSigInt(["-m", ham.stem], cwd=ham.parent) +class MiscTests(unittest.TestCase): + def test_remove_sys_path0(self): + # Check that all path0 copies are removed + PATH0 = '/path/zero' + STDLIB_PATH = '/to/stdlib' + OTHER_PATH = '/other/path' + sys_path = [PATH0, STDLIB_PATH, PATH0, '', OTHER_PATH, PATH0] + expected = [STDLIB_PATH, '', OTHER_PATH] + with (support.swap_attr(sys, '_stdlib_dir', STDLIB_PATH), + support.swap_attr(sys, '_path0', PATH0), + support.swap_attr(sys, 'path', sys_path)): + runpy._remove_sys_path0() + self.assertEqual(sys.path, expected) + + # Calling it twice does nothing + runpy._remove_sys_path0() + self.assertEqual(sys.path, expected) + + # Do nothing if sys._path0 is sys._stdlib_dir + sys_path = ['', STDLIB_PATH, OTHER_PATH] + expected = sys_path.copy() + with (support.swap_attr(sys, '_stdlib_dir', STDLIB_PATH), + support.swap_attr(sys, '_path0', STDLIB_PATH), + support.swap_attr(sys, 'path', sys_path)): + runpy._remove_sys_path0() + self.assertEqual(sys.path, expected) + + # Do nothing if sys._path0 is None + sys_path = ['', STDLIB_PATH, OTHER_PATH] + expected = sys_path.copy() + with (support.swap_attr(sys, '_stdlib_dir', STDLIB_PATH), + support.swap_attr(sys, '_path0', None), + support.swap_attr(sys, 'path', sys_path)): + runpy._remove_sys_path0() + self.assertEqual(sys.path, expected) + + def run_python(self, *args, PYTHONPATH=None): + cmd = [sys.executable, *args] + env = support.make_clean_env() + if PYTHONPATH: + env['PYTHONPATH'] = PYTHONPATH + proc = subprocess.run(cmd, text=True, stderr=subprocess.PIPE, env=env) + self.assertEqual(proc.returncode, 0, proc.stderr) + + @support.requires_subprocess() + def test_remove_sys_path0_run_python(self): + CODE = textwrap.dedent(""" + import runpy, sys + + # Remove sys.path[0] + orig_path = sys.path.copy() + path0 = sys._path0 + runpy._remove_sys_path0() + + if path0 is not None and path0 in sys.path: + raise Exception(f"path0 {path0!r} found in sys.path {sys.path!r}") + if not sys.flags.safe_path: + if sys.path == orig_path: + raise Exception("sys.path was not changed") + else: + if sys.path != orig_path: + raise Exception("sys.path was changed") + + # Calling it twice must do nothing + clean_path = sys.path.copy() + runpy._remove_sys_path0() + if sys.path != clean_path: + raise Exception("sys.path was changed") + """) + + # Test "python -c code" + self.run_python('-c', CODE) + self.run_python('-P', '-c', CODE) + + with tempfile.TemporaryDirectory() as tmpdir: + name = 'test_remove_sys_path0' + filename = os.path.join(tmpdir, f'{name}.py') + with open(filename, 'w') as fp: + fp.write(CODE) + + # Test "python script.py" + self.run_python(filename) + self.run_python('-P', filename) + + with os_helper.change_cwd(tmpdir): + # Test "python -m module" + self.run_python('-m', name) + self.run_python('-P', '-m', name, PYTHONPATH=tmpdir) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 201c209d8214150..3b65bf5128cdbf2 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1493,6 +1493,15 @@ class PythonType: self.assertFalse(is_immortal(b'abc'.decode())) self.assertFalse(is_immortal('abc'.encode())) + def test_path(self): + self.assertIsInstance(sys.path, list) + for path in sys.path: + self.assertIsInstance(path, str) + + def test_path0(self): + self.assertTrue(sys._path0 is None or isinstance(sys._path0, str), + sys._path0) + @test.support.cpython_only @test.support.force_not_colorized_test_class diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 84000f472c4df9e..2f8fef2a006c381 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -58,6 +58,15 @@ def get_files(test): test.assertFalse(f.closed) +@contextlib.contextmanager +def save_restore_sys_path(): + old_path = sys.path.copy() + try: + yield + finally: + sys.path[:] = old_path + + class LazyImportTest(unittest.TestCase): @cpython_only def test_lazy_import(self): @@ -5987,17 +5996,18 @@ def test_add_comment(self): def test_cli_with_metadata_encoding(self): errmsg = "Non-conforming encodings not supported with -c." args = ["--metadata-encoding=shift_jis", "-c", "nonesuch", "nonesuch"] - with captured_stdout() as stdout: - with captured_stderr() as stderr: + with (captured_stdout() as stdout, + captured_stderr() as stderr, + save_restore_sys_path()): self.assertRaises(SystemExit, zipfile.main, args) self.assertEqual(stdout.getvalue(), "") self.assertIn(errmsg, stderr.getvalue()) - with captured_stdout() as stdout: + with captured_stdout() as stdout, save_restore_sys_path(): zipfile.main(["--metadata-encoding=shift_jis", "-t", TESTFN]) listing = stdout.getvalue() - with captured_stdout() as stdout: + with captured_stdout() as stdout, save_restore_sys_path(): zipfile.main(["--metadata-encoding=shift_jis", "-l", TESTFN]) listing = stdout.getvalue() for name in self.file_names: @@ -6016,7 +6026,9 @@ def test_cli_with_metadata_encoding_extract(self): except UnicodeEncodeError: self.skipTest(f'cannot encode file name {fn!a}') - zipfile.main(["--metadata-encoding=shift_jis", "-e", TESTFN, TESTFN2]) + with save_restore_sys_path(): + zipfile.main(["--metadata-encoding=shift_jis", "-e", + TESTFN, TESTFN2]) listing = os.listdir(TESTFN2) for name in self.file_names: self.assertIn(name, listing) diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 3110759b29f2713..346bc02ce058eef 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -2926,6 +2926,10 @@ def _get_code(self, pathname, basename): def main(args=None): import argparse + import runpy + + # Prevent shadowing stdlib modules from extracted files + runpy._remove_sys_path0() description = 'A simple command-line interface for zipfile module.' parser = argparse.ArgumentParser(description=description) diff --git a/Misc/NEWS.d/next/Security/2026-11-07-15-56-47.gh-issue-161705.OjDK2d.rst b/Misc/NEWS.d/next/Security/2026-11-07-15-56-47.gh-issue-161705.OjDK2d.rst new file mode 100644 index 000000000000000..d9a862ed858e012 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-11-07-15-56-47.gh-issue-161705.OjDK2d.rst @@ -0,0 +1,6 @@ +.. cve: CVE-2026-115791 + +Remove ``sys.path[0]`` in :mod:`gzip`, :mod:`tarfile` and :mod:`zipfile` +command line interface to prevent shadowing standard library modules from +extracted files. Issue reported by Jelle Zijlstra, patch by Gregory P. +Smith and Victor Stinner. diff --git a/Modules/main.c b/Modules/main.c index d2587e9fffdad26..d826ed17e0293c1 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -214,6 +214,10 @@ pymain_sys_path_add_path0(PyObject *path0) return -1; } + if (PyDict_SetItemString(sysdict, "_path0", path0) < 0) { + return -1; + } + if (PyList_Insert(sys_path, 0, path0)) { return -1; } diff --git a/Python/pathconfig.c b/Python/pathconfig.c index b4ba1c52066879b..89684d9da8e6291 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -340,7 +340,7 @@ Py_GetProgramName(void) /* Compute module search path from argv[0] or the current working - directory ("-m module" case) which will be prepended to sys.argv: + directory ("-m module" case) which will be prepended to sys.path: sys.path[0]. Return 1 if the path is correctly resolved and written into *path0_p. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index e8dd956c3f48828..f8c72b096eb4529 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -4161,6 +4161,8 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS("path_importer_cache", PyDict_New()); SET_SYS("path_hooks", PyList_New(0)); + SET_SYS("_path0", Py_None); + if (_PyErr_Occurred(tstate)) { goto err_occurred; } From 1ce279ed32c8a6b7850111a21efc39448926ab91 Mon Sep 17 00:00:00 2001 From: digit50 <93092234+digit50@users.noreply.github.com> Date: Tue, 10 Nov 2026 09:45:40 +0300 Subject: [PATCH 069/229] gh-159643: Fix EOFError in tarfile stream mode with bz2, xz and zstd (GH-159647) In stream mode (`r|bz2`, `r|xz`, `r|zst`, and `r|*` when it detects one of them), `_Stream._read` called `decompress(b"")` on a decompressor that had already reached the end of its data, because an exhausted `BZ2Decompressor`, `LZMADecompressor` or `ZstdDecompressor` reports `needs_input` as false. That raised a bare `EOFError` whenever tarfile read past the end of the compressed stream, for example with `ignore_zeros=True` or on an archive without end-of-archive blocks. Stop the read loop once the decompressor reports `eof`. This restores the pre-gh-121109 results for the cases above. Data after the end of the compressed stream is now always ignored; before gh-121109 it was ignored only if it arrived in the same read as the end of the stream, and raised `EOFError` otherwise. The gzip branch is unchanged. --------- Co-authored-by: Gregory P. Smith --- Lib/tarfile.py | 2 + Lib/test/test_tarfile.py | 38 +++++++++++++++++++ ...17-19-33-13.gh-issue-159643.EJfe0IXetV.rst | 3 ++ 3 files changed, 43 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-10-17-19-33-13.gh-issue-159643.EJfe0IXetV.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index abf7786812a0634..b052ca0dd407292 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -594,6 +594,8 @@ def _read(self, size): else: # Other decompressors have needs_input. # decompress() can buffer data internally. + if self.cmp.eof: + break if self.cmp.needs_input: cbuf = self.fileobj.read(self.bufsize) if not cbuf: diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 2ad6d0452ce29ab..1927ff227d5356f 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1061,6 +1061,44 @@ def test_compare_members(self): finally: tar1.close() + def test_ignore_zeros_concatenated(self): + # gh-159643: reading past the end of the compressed data + # must not raise EOFError. + with self.open(tmpname, "wb") as fobj: + for name in ("a", "b"): + with tarfile.open(fileobj=fobj, mode="w") as tar: + tar.addfile(tarfile.TarInfo(name)) + with tarfile.open(tmpname, self.mode, ignore_zeros=True) as tar: + self.assertEqual(tar.getnames(), ["a", "b"]) + + def test_no_end_of_archive_blocks(self): + # gh-159643 + tarinfo = tarfile.TarInfo("a") + tarinfo.size = 5 + with self.open(tmpname, "wb") as fobj: + fobj.write(tarinfo.tobuf()) + fobj.write(b"hello".ljust(tarfile.BLOCKSIZE, b"\0")) + with tarfile.open(tmpname, self.mode) as tar: + self.assertEqual(tar.getnames(), ["a"]) + + def test_data_after_end_of_stream(self): + # gh-159643: with a small bufsize, the compressed data spans + # several reads of the file and the padding that follows it + # is not read together with the end of the compressed data. + bufsize = tarfile.BLOCKSIZE + data = Random(0).randbytes(8 * bufsize) + tarinfo = tarfile.TarInfo("a") + tarinfo.size = len(data) + with self.open(tmpname, "wb") as fobj: + with tarfile.open(fileobj=fobj, mode="w") as tar: + tar.addfile(tarinfo, io.BytesIO(data)) + with open(tmpname, "ab") as fobj: + fobj.write(bytes(4 * bufsize)) + with tarfile.open(tmpname, self.mode, bufsize=bufsize, + ignore_zeros=True) as tar: + members = [(m.name, tar.extractfile(m).read()) for m in tar] + self.assertEqual(members, [("a", data)]) + class GzipStreamReadTest(GzipTest, GzipReadTestBase, StreamReadTest): pass diff --git a/Misc/NEWS.d/next/Library/2026-10-17-19-33-13.gh-issue-159643.EJfe0IXetV.rst b/Misc/NEWS.d/next/Library/2026-10-17-19-33-13.gh-issue-159643.EJfe0IXetV.rst new file mode 100644 index 000000000000000..a76e54de38cc4e0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-17-19-33-13.gh-issue-159643.EJfe0IXetV.rst @@ -0,0 +1,3 @@ +Fix :exc:`EOFError` raised by :mod:`tarfile` in stream mode with bz2, xz or +zstd compression when reading past the end of the compressed data, for +example with *ignore_zeros* or an archive without end-of-archive blocks. From 4575fb91eab84ec8ee02b7cad7c38b309006a010 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Tue, 10 Nov 2026 01:36:46 -0700 Subject: [PATCH 070/229] gh-158194: Exercise type freezing in concurrent setattr deadlock test (#161655) --- Lib/test/test_free_threading/test_type.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 6e750e0bc6084e5..f7e4adf2e02ce8a 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -375,6 +375,7 @@ def test_concurrent_setattr_deadlock(self): # dict mutex and waited for the type lock. # This is fairly difficult to trigger the race but this N seems to do # it at least sometimes. + type_freeze = import_helper.import_module("_testlimitedcapi").type_freeze N = 200 done = False @@ -401,6 +402,15 @@ def lister(): while not done: Base.__subclasses__() + def freezer(): + # gh-158194: Exercise type freezing concurrently with slot + # updates and competing stop-the-world requests. + barrier.wait() + for _ in range(10): + class Frozen: + pass + type_freeze(Frozen) + def basesetter(): nonlocal done barrier.wait() @@ -418,7 +428,7 @@ class B(A): # to keep the type lock and the stop-the-world mutex contended, which # is what gets the setters into the window where it happens. targets = (setter, setter, subclasser, subclasser, - lister, lister, basesetter) + lister, lister, basesetter, freezer) barrier = threading.Barrier(len(targets)) threads = [Thread(target=target) for target in targets] with threading_helper.start_threads(threads): From 9d05428b29508873fb26832eea5d17e0e8de60df Mon Sep 17 00:00:00 2001 From: Timofei Ivankov <128279579+deadlovelll@users.noreply.github.com> Date: Tue, 10 Nov 2026 14:28:27 +0300 Subject: [PATCH 071/229] gh-161083: Fix stale awaited_by nodes on invalid asyncio.gather() arguments (#161084) Co-authored-by: Kumar Aditya --- Lib/asyncio/tasks.py | 7 +++++-- Lib/test/test_asyncio/test_tasks.py | 12 ++++++++++++ .../2026-11-01-19-38-33.gh-issue-161083.PM21lv.rst | 2 ++ 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-01-19-38-33.gh-issue-161083.PM21lv.rst diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 6c8d7cbbda64407..70da1e25ebaf081 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -926,8 +926,6 @@ def _done_callback(fut, cur_task=cur_task): if fut.done(): done_futs.append(fut) else: - if cur_task is not None: - futures.future_add_to_awaited_by(fut, cur_task) fut.add_done_callback(_done_callback) else: @@ -937,6 +935,11 @@ def _done_callback(fut, cur_task=cur_task): children.append(fut) outer = _GatheringFuture(children, loop=loop) + # Add the awaited-by edges only after all arguments are validated + if cur_task is not None and len(done_futs) < nfuts: + for fut in arg_to_fut.values(): + if not fut.done(): + futures.future_add_to_awaited_by(fut, cur_task) # Run done callbacks after GatheringFuture created so any post-processing # can be performed at this point # optimization: in the special case that *all* futures finished eagerly, diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index f16815474e2d196..ac4856bd7a64c84 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1328,6 +1328,18 @@ async def coro(): self.loop.run_until_complete(self.new_task(self.loop, coro())) + def test_gather_invalid_arg_does_not_add_awaited_by(self): + async def coro(): + task = self.new_task(self.loop, asyncio.sleep(0)) + try: + with self.assertRaises(TypeError): + asyncio.gather(task, None) + self.assertFalse(task._asyncio_awaited_by) + finally: + await task + + self.loop.run_until_complete(self.new_task(self.loop, coro())) + def test_gather_discards_awaited_by_for_pending(self): # gh-157213: a child outliving gather() must lose the awaited-by edge async def fail(): diff --git a/Misc/NEWS.d/next/Library/2026-11-01-19-38-33.gh-issue-161083.PM21lv.rst b/Misc/NEWS.d/next/Library/2026-11-01-19-38-33.gh-issue-161083.PM21lv.rst new file mode 100644 index 000000000000000..98b3ea827276518 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-01-19-38-33.gh-issue-161083.PM21lv.rst @@ -0,0 +1,2 @@ +Fix stale task nodes left by :func:`asyncio.gather` when argument validation +fails. Patch by Timofei Ivankov From e8c0584877e0b0a82ee8e0707336d8df4d68ccb4 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Tue, 10 Nov 2026 06:34:55 -0800 Subject: [PATCH 072/229] Add orsenthil as the CODEOWNER for urllib and httplib modules. (#161750) --- .github/CODEOWNERS | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d6d06555dd26d3d..a95eb943e4a4e7c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -498,6 +498,11 @@ Lib/_markupbase.py @ezio-melotti Lib/test/test_html*.py @ezio-melotti Tools/build/parse_html5_entities.py @ezio-melotti +# HTTP +Doc/library/http* @orsenthil +Lib/http/ @orsenthil +Lib/test/test_http*.py @orsenthil + # IDLE Doc/library/idle.rst @terryjreedy Lib/idlelib/ @terryjreedy @@ -633,6 +638,11 @@ Modules/_typesmodule.c @AA-Turner Lib/unittest/mock.py @cjw296 Lib/test/test_unittest/testmock/ @cjw296 +# Urllib +Doc/library/urllib* @orsenthil +Lib/urllib/ @orsenthil +Lib/test/test_urllib*.py @orsenthil + # Venv **/*venv* @vsajip @FFY00 From 9d4a35c494cb1cfd9c77c619b22bd425695316c2 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Tue, 10 Nov 2026 10:39:30 -0500 Subject: [PATCH 073/229] gh-63293: Reject non-string URL args and deprecate false values in urllib.parse gh-63293: Reject non-string URL args and deprecate false values in urllib.parse Truthy values other than str/bytes previously failed with AttributeError. False values such as 0 and [] were silently treated as empty. Require str, bytes, or None in _coerce_args(): raise TypeError for other truthy values, and emit DeprecationWarning for other false values, scheduled to become TypeError in 3.18. This extends the 3.14 deprecation for parse_qsl()/parse_qs(). Original patch by Jacob Walls. ---- Co-authored-by: Senthil Kumaran --- Doc/deprecations/pending-removal-in-3.18.rst | 7 +++ Doc/library/urllib.parse.rst | 56 +++++++++++++++++++ Doc/whatsnew/3.16.rst | 10 ++++ Lib/test/test_urlparse.py | 23 ++++++++ Lib/urllib/parse.py | 13 ++++- .../2021-06-11-20-00-16.bpo-19094.rMRoIL.rst | 6 ++ 6 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-06-11-20-00-16.bpo-19094.rMRoIL.rst diff --git a/Doc/deprecations/pending-removal-in-3.18.rst b/Doc/deprecations/pending-removal-in-3.18.rst index 19113aab981bbc6..fb10420ab38c52f 100644 --- a/Doc/deprecations/pending-removal-in-3.18.rst +++ b/Doc/deprecations/pending-removal-in-3.18.rst @@ -11,6 +11,13 @@ Pending removal in Python 3.18 C implementation, has been deprecated since Python 3.13. (Contributed by Serhiy Storchaka in :gh:`89902`.) +* :mod:`urllib.parse`: + + * Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, byte-like objects and ``None`` in the URL parsing functions has + been deprecated since Python 3.16. + (Contributed by Jacob Walls in :gh:`63293`.) + * Deprecations defined by :pep:`829`: * ``import`` lines in :file:`{name}.pth` files are silently ignored. diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 26e159306c81070..9a8f8cf9bab012e 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -225,6 +225,16 @@ or on combining URL components into a URL string. .. versionchanged:: 3.15 Added the *missing_as_none* parameter. + .. versionchanged:: 3.16 + Passing an object that is not a string, bytes, or ``None`` as + *urlstring* or *scheme* now raises :exc:`TypeError` instead of + :exc:`AttributeError`. + + .. deprecated:: 3.16 + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, byte-like objects and ``None`` as *urlstring* or *scheme* is + now deprecated. They will raise :exc:`TypeError` in Python 3.18. + .. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser @@ -341,6 +351,15 @@ or on combining URL components into a URL string. .. versionchanged:: 3.15 Added the *keep_empty* parameter. + .. versionchanged:: 3.16 + Passing an object that is not a string, bytes, or ``None`` in *parts* + now raises :exc:`TypeError` instead of :exc:`AttributeError`. + + .. deprecated:: 3.16 + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, byte-like objects and ``None`` in *parts* is now deprecated. + They will raise :exc:`TypeError` in Python 3.18. + .. function:: urlparse(urlstring, scheme=None, allow_fragments=True, *, missing_as_none=False) @@ -358,6 +377,16 @@ or on combining URL components into a URL string. :func:`urlsplit` should generally be used instead of :func:`urlparse`. A separate function is needed to separate the path segments and parameters. + .. versionchanged:: 3.16 + Passing an object that is not a string, bytes, or ``None`` as + *urlstring* or *scheme* now raises :exc:`TypeError` instead of + :exc:`AttributeError`. + + .. deprecated:: 3.16 + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, byte-like objects and ``None`` as *urlstring* or *scheme* is + now deprecated. They will raise :exc:`TypeError` in Python 3.18. + .. function:: urlunparse(parts) urlunparse(parts, *, keep_empty) @@ -379,6 +408,15 @@ or on combining URL components into a URL string. .. versionchanged:: 3.15 Added the *keep_empty* parameter. + .. versionchanged:: 3.16 + Passing an object that is not a string, bytes, or ``None`` in *parts* + now raises :exc:`TypeError` instead of :exc:`AttributeError`. + + .. deprecated:: 3.16 + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, byte-like objects and ``None`` in *parts* is now deprecated. + They will raise :exc:`TypeError` in Python 3.18. + .. function:: urljoin(base, url, allow_fragments=True) @@ -422,6 +460,15 @@ or on combining URL components into a URL string. Behavior updated to match the semantics defined in :rfc:`3986`. + .. versionchanged:: 3.16 + Passing an object that is not a string, bytes, or ``None`` as *base* or + *url* now raises :exc:`TypeError` instead of :exc:`AttributeError`. + + .. deprecated:: 3.16 + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, byte-like objects and ``None`` as *base* or *url* is now + deprecated. They will raise :exc:`TypeError` in Python 3.18. + .. function:: urldefrag(url, *, missing_as_none=False) @@ -452,6 +499,15 @@ or on combining URL components into a URL string. .. versionchanged:: 3.15 Added the *missing_as_none* parameter. + .. versionchanged:: 3.16 + Passing an object that is not a string, bytes, or ``None`` as *url* now + raises :exc:`TypeError` instead of :exc:`AttributeError`. + + .. deprecated:: 3.16 + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, byte-like objects and ``None`` as *url* is now deprecated. + They will raise :exc:`TypeError` in Python 3.18. + .. function:: unwrap(url) Extract the url from a wrapped URL (that is, a string formatted as diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 4af5d3c0bd072e2..84a5ffd186db65d 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1258,6 +1258,16 @@ New deprecations Use :pep:`695` syntax instead. (Contributed by Nikita Sobolev in :gh:`150190`.) +* :mod:`urllib.parse`: + + * Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, byte-like objects and ``None`` is now deprecated in the URL + parsing functions, and is scheduled to raise :exc:`TypeError` in Python + 3.18. This extends the same deprecation made in Python 3.14 for + :func:`~urllib.parse.parse_qsl`, :func:`~urllib.parse.parse_qs`, and + :func:`~urllib.parse.urlencode`. + (Contributed by Jacob Walls in :gh:`63293`.) + .. Add deprecations above alphabetically, not here at the end. .. include:: ../deprecations/pending-removal-in-3.17.rst diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 112cc139d597b6c..37e172f7f2caca3 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -1252,6 +1252,12 @@ def test_mixed_types_rejected(self): with self.assertRaisesRegex(TypeError, "Cannot mix str"): urllib.parse.urljoin(b"http://python.org", "http://python.org") + def test_non_string_true_values_rejected(self): + # True values raise informative TypeErrors + msg = "Expected str, bytes or None, not " + with self.assertRaisesRegex(TypeError, msg): + urllib.parse.urlsplit(1, b'http') + def _check_result_type(self, str_type, str_args): bytes_type = str_type._encoded_counterpart self.assertIs(bytes_type._decoded_counterpart, str_type) @@ -1987,6 +1993,23 @@ def test_to_bytes_deprecation(self): self.assertEqual(str(cm.warning), 'urllib.parse.to_bytes() is deprecated as of 3.8') + def test_falsey_deprecation(self): + cases = [ + (urllib.parse.urljoin, ['http://www.python.org', []]), + (urllib.parse.urljoin, [[], b'docs']), + (urllib.parse.urlparse, [b'www.python.org', {}]), + (urllib.parse.urlparse, [{}, '']), + (urllib.parse.urlsplit, [0, b'http']), + (urllib.parse.urlsplit, [b'http://www.python.org', 0]), + (urllib.parse.urldefrag, [{}]), + (urllib.parse.urlunparse, [[None, b'www.python.org', (), (), (), ()]]), + (urllib.parse.urlunsplit, [['http', 0, '', '', '']]), + ] + for func, args in cases: + with self.subTest(callable=func.__name__, args=args): + with self.assertWarnsRegex(DeprecationWarning, "false value"): + func(*args) + def str_encode(s): if s is None: diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index ee0bceb2928c06f..f6c94060054e48f 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -125,6 +125,13 @@ def _coerce_args(*args): # - encoding function otherwise str_input = None for arg in args: + if not (arg is None or isinstance(arg, str) or hasattr(arg, 'decode')): + if arg: + raise TypeError("Expected str, bytes or None, " + f"not {type(arg).__name__}") + warnings.warn(f"Accepting {type(arg).__name__} objects with false " + f"value in urllib.parse is deprecated as of 3.16", + DeprecationWarning, stacklevel=3) if arg: if str_input is None: str_input = isinstance(arg, str) @@ -712,12 +719,12 @@ def _urlunsplit(scheme, netloc, url, query, fragment): def urljoin(base, url, allow_fragments=True): """Join a base URL and a possibly relative URL to form an absolute interpretation of the latter.""" + base, url, _coerce_result = _coerce_args(base, url) if not base: - return url + return _coerce_result(url) if not url: - return base + return _coerce_result(base) - base, url, _coerce_result = _coerce_args(base, url) bscheme, bnetloc, bpath, bquery, bfragment = \ _urlsplit(base, None, allow_fragments) scheme, netloc, path, query, fragment = \ diff --git a/Misc/NEWS.d/next/Library/2021-06-11-20-00-16.bpo-19094.rMRoIL.rst b/Misc/NEWS.d/next/Library/2021-06-11-20-00-16.bpo-19094.rMRoIL.rst new file mode 100644 index 000000000000000..53cae4448ed9d54 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-06-11-20-00-16.bpo-19094.rMRoIL.rst @@ -0,0 +1,6 @@ +The :mod:`urllib.parse` URL parsing functions now raise :exc:`TypeError` +instead of :exc:`AttributeError` when passed an object which is not a string, +bytes, or ``None``. Accepting objects with false values (like ``0`` and +``[]``) except empty strings, byte-like objects and ``None`` is now +deprecated, extending the deprecation made in Python 3.14 for +:func:`~urllib.parse.parse_qsl` and :func:`~urllib.parse.parse_qs`. From bb8d4fcba98d6aa259e35d401d9748ad231fe106 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:30:53 -0800 Subject: [PATCH 074/229] gh-129724: Intern code object names before merging them with constants (GH-161457) makecode() merges the co_names and co_localsplusnames tuples with the constant cache, so they can become an equal tuple constant of another code object, and _PyCode_New() then interns the items of that tuple in place. If an equal string was already interned in the process, the string of the constant was replaced by it; otherwise the string of the constant itself became the interned one. The constants which are the same object, and so the marshal output, depended on the process state. The parser interns identifiers and the compiler interns the ASCII identifier-like string constants, so this needs a name which is not interned yet and is not an ASCII identifier: a private name of a class with a non-ASCII name, which each compilation mangles into a new string, or an identifier of an AST passed to compile() which was not created by the parser. Intern the names before the merge. --- Lib/test/test_compile.py | 46 +++++++++++++++++++ ...-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst | 4 ++ Python/assemble.c | 26 +++++++++++ 3 files changed, 76 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index df766387dd4c72d..d2069a0ac4214d0 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -2342,6 +2342,29 @@ def g(self, __x): return [__x] """, strings=['_ID_A__x']), + # A constant of an earlier method is equal to the co_names of a + # later one. The mangled name is not an ASCII identifier, so the + # compiler does not intern the equal string constants. + 'mangled_names_non_ascii': Case( + source=""" + class ID_\xe9: + def g(self): + return ("_ID_\xe9__x",), "_ID_\xe9__x" + def f(self): + return self.__x + """, + strings=['_ID_\xe9__x']), + # The same with the names of local variables. + 'mangled_varnames_non_ascii': Case( + source=""" + class ID_\xe9: + def g(self): + return ("_ID_\xe9__x",), "_ID_\xe9__x" + @staticmethod + def f(__x): + return __x + """, + strings=['_ID_\xe9__x']), # The names of the keyword arguments are a constant equal to # the co_names of the function. 'kwnames': Case( @@ -2548,6 +2571,29 @@ def code_constants(code): self.assertEqual(len(names), 1) self.assertTrue(any(const is names for const in consts)) + def test_ast_names_not_interned(self): + # The identifiers of an AST which was not created by the parser are + # not interned yet. A constant of an earlier function is equal to + # the co_names of a later one. + source = textwrap.dedent(""" + def g(self): + return ("gh129724_ast_\xe9",), "gh129724_ast_\xe9" + def f(self): + return self.placeholder + """) + keep = [] + dumps = [] + for attempt in range(2): + tree = ast.parse(source) + attribute, = [node for node in ast.walk(tree) + if isinstance(node, ast.Attribute)] + # Not a literal: a new string object, which is not interned. + attribute.attr = ''.join(['gh129724_ast_', '\xe9']) + code = compile(tree, '', 'exec') + keep.append(code) + dumps.append(marshal.dumps(code)) + self.assertEqual(dumps[0], dumps[1]) + @support.cpython_only def test_slice_strings(self): # The strings of a slice constant are interned like the other diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst new file mode 100644 index 000000000000000..a259af1dcea0f95 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst @@ -0,0 +1,4 @@ +Fix the :mod:`marshal` output of compiled code, and so ``.pyc`` files, +differing for the same source depending on what ran before in the process. +This affected private names in classes with non-ASCII names, and +:func:`compile` of an AST with names not created by :func:`ast.parse`. diff --git a/Python/assemble.c b/Python/assemble.c index 8f68e233ab7b93c..9f4ffb4050d2598 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -4,7 +4,10 @@ #include "pycore_instruction_sequence.h" #include "pycore_opcode_utils.h" // IS_BACKWARDS_JUMP_OPCODE #include "pycore_opcode_metadata.h" // is_pseudo_target, _PyOpcode_Caches +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_symtable.h" // _Py_SourceLocation +#include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal() #include @@ -503,6 +506,27 @@ dict_keys_inorder(PyObject *dict, Py_ssize_t offset) extern void _Py_set_localsplus_info(int, PyObject *, unsigned char, PyObject *, PyObject *); +/* Intern the names in *tuple*, as _PyCode_New() does, before the tuple is + * merged with the constant cache. + * + * The merge can replace the tuple by an equal tuple constant of another + * code object, whose items _PyCode_New() then interns in place. A string + * of that constant would become the interned string or be replaced by it, + * depending on whether an equal string was already interned in the process, + * and so would the output of marshal for the same source. Names which the + * parser did not create, such as mangled private names or the identifiers + * of an AST passed to compile(), may not be interned yet. + */ +static void +intern_names(PyObject *tuple) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(tuple); i++) { + assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(tuple, i))); + _PyUnicode_InternImmortal(interp, &_PyTuple_ITEMS(tuple)[i]); + } +} + static int compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, int flags, PyObject *names, PyObject *kinds) @@ -610,6 +634,7 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_ if (!names) { goto error; } + intern_names(names); if (_PyCompile_ConstCacheMergeOne(const_cache, &names) < 0) { goto error; } @@ -674,6 +699,7 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_ goto error; } + intern_names(localsplusnames); if (_PyCompile_ConstCacheMergeOne(const_cache, &localsplusnames) < 0) { goto error; } From d94fa8bfc23eee5e67e6b8e2d11d7b1c71f173b1 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 10 Nov 2026 08:43:34 -0800 Subject: [PATCH 075/229] GH-151914: Copyedit "What's new in Python 3.16" (#161718) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/whatsnew/3.16.rst | 107 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 84a5ffd186db65d..4a1fb6b6b87e131 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -62,11 +62,48 @@ Summary -- Release highlights .. This section singles out the most important changes in Python 3.16. Brevity is key. +* :ref:`Set operations and Unicode properties in regular expressions + `. +* :ref:`Encrypted Client Hello support in ssl `. +* :ref:`Faster interpreter startup `. +* :ref:`Faster UUID generation `, with + UUIDv4 and UUIDv7 about 15 and 30 times faster in microbenchmarks. +* :ref:`Build support for tvOS, watchOS, and visionOS + `. +* :ref:`Improved error messages `. New features ============ +.. _whatsnew316-improved-error-messages: + +Improved error messages +----------------------- + +* :exc:`SyntaxError` messages now suggest Python alternatives to some + keywords from other programming languages. For example, ``switch x:`` + suggests ``match``, ``delete x`` suggests ``del``, and ``function f():`` + suggests ``def``. + (Contributed by Zang Langyan in :gh:`151128`.) + +* Improve :exc:`SyntaxError` message for ``&&`` and ``||`` operators, + suggesting ``and``/``&`` and ``or``/``|`` respectively. + (Contributed by Aniket, Sergey B Kirpichev, and Nikita Sobolev in + :gh:`146495`.) + +* Suggest the closest valid feature name in the :exc:`SyntaxError` message + when an unknown name is imported from :mod:`__future__`. + (Contributed by Nikita Sobolev and Pablo Galindo Salgado in :gh:`145868`.) + +* :exc:`TypeError` messages for some instance methods called with too many + positional arguments now suggest checking whether the method definition is + missing the ``self`` parameter. (Contributed by Suren Nihalani in + :gh:`152315`.) + +* Add context notes to :exc:`TypeError` raised while hashing a + :class:`tuple`, identifying the index of the item whose hash failed. + (Contributed by Ivan Zorin in :gh:`161433`.) Other language changes @@ -95,11 +132,6 @@ Other language changes they reference) alive indefinitely. (Contributed by Łukasz Langa in :gh:`102960`.) -* :exc:`TypeError` messages for some instance methods called with too many - positional arguments now suggest checking whether the method definition is - missing the ``self`` parameter. (Contributed by Suren Nihalani in - :gh:`152315`.) - * :func:`anext` and :func:`aiter` are now implemented in Python instead of C. The awaitable returned by ``anext(it, default)`` and the ``__anext__()`` of the iterator returned by ``aiter(callable, ...)`` are @@ -204,6 +236,19 @@ concurrent.interpreters :meth:`~concurrent.interpreters.Interpreter.close`. (Contributed by Peter Bierma in :gh:`158370`.) + +copy +---- + +* :func:`copy.replace` now supports :class:`argparse.Namespace`, + :class:`optparse.Values`, :class:`decimal.Context`, + :class:`tarfile.TarInfo`, and CSV dialects (instances of + :class:`csv.Dialect` subclasses and objects returned by + :func:`csv.get_dialect`). + (Contributed by Serhiy Storchaka and Stan Ulbrych in :gh:`155043`, + :gh:`155044`, :gh:`155041`, :gh:`155040`, and :gh:`155033`.) + + csv --- @@ -642,6 +687,8 @@ quopri re -- +.. _whatsnew316-re-set-operations-and-unicode-properties: + * :mod:`re` now supports set operations and nested sets in character classes, as described in `Unicode Technical Standard #18 `__: set difference (``[A--B]``), @@ -715,6 +762,8 @@ sqlite3 ssl --- +.. _whatsnew316-ssl-ech: + * Added support for :ref:`Encrypted Client Hello ` (ECH) with OpenSSL 4.0 or later. Clients can supply an ECHConfigList discovered through DNS, for example using a third-party library such as @@ -1015,6 +1064,35 @@ zlib Optimizations ============= +.. _whatsnew316-interpreter-startup: + +* Interpreter startup is faster, and several common operations have been + optimized. Improvements include less work during garbage collection and + faster construction of :class:`list`, :class:`dict`, :class:`bytes`, and + :class:`bytearray` objects. + Startup performance is tracked on the `Python Speed Center + `__. + (Contributed by Pablo Galindo Salgado, Cody Maloney, and Pieter Eendebak in + :gh:`159137`, :gh:`160858`, :gh:`160913`, and :gh:`87613`.) + +* Speed up iteration over asynchronous generators: values yielded to + ``async for`` no longer create and discard a :exc:`StopIteration` exception + at every nesting level. The ``async_generators`` pyperformance benchmark is + about 30% faster. + (Contributed by Shamil Abdulaev in :gh:`158304`.) + + +ast +--- + +* AST nodes now store attributes inline with shared keys instead of + allocating a separate dictionary for each node. Benchmarks of this + change measured about 16% faster :func:`ast.parse` calls and 52% less + memory used by the resulting ASTs. + (Contributed by Lysandros Nikolaou and Pablo Galindo Salgado + in :gh:`153568`.) + + datetime and time ----------------- @@ -1390,6 +1468,25 @@ Build changes (Contributed by Stan Ulbrych, Florian Mayer and AnnaAr321 in :gh:`156049`.) +* iOS builds can now be configured as free-threaded or debug builds. + + (Contributed by Russell Keith-Magee in :gh:`159385`.) + + +.. _whatsnew316-apple-platforms: + +Build support for tvOS, watchOS, and visionOS +--------------------------------------------- + +Added build support for Apple's tvOS, watchOS, and visionOS. +These platforms are not currently included in the supported +platform tiers defined by :pep:`11`. +See the :source:`tvOS `, +:source:`watchOS `, and +:source:`visionOS ` build instructions. + +(Contributed by Russell Keith-Magee in :gh:`131999`.) + C API changes ============= From 2f0e294d85c905691e360586ebd1d1f7e7a1f5df Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 10 Nov 2026 08:50:50 -0800 Subject: [PATCH 076/229] gh-160369: Reject buffered plaintext after IMAP STARTTLS (#160370) Co-authored-by: Seth Michael Larson Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/imaplib.py | 6 ++ Lib/test/test_imaplib.py | 85 +++++++++++++++++++ ...10-22-21-59-39.gh-issue-160369.imapTLS.rst | 4 + 3 files changed, 95 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2026-10-22-21-59-39.gh-issue-160369.imapTLS.rst diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 977d6bdb1c3d86b..d9ec48cbc21959d 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1176,6 +1176,12 @@ def starttls(self, ssl_context=None): ssl_context = ssl._create_stdlib_context() typ, dat = self._simple_command(name) if typ == 'OK': + if any(self._readbuf): + raise self.abort('unexpected data after STARTTLS response') + # Discard received state before TLS was established + # since data could have been injected by a MITM. + self._readbuf = [] + self.untagged_responses = {} self._close_imaplib_file() self.sock = ssl_context.wrap_socket(self.sock, server_hostname=self.host) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 51e3412324b72d5..906c78f07d9c337 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -353,6 +353,33 @@ def test_substitute_errors(self): self.assertRaises(ValueError, sub, '?s', [[(1,)]]) # not a range pair self.assertRaises(ValueError, sub, '?s', [[(1, 2, 3)]]) # not a range pair + @unittest.skipUnless(ssl, "SSL not available") + def test_starttls_rejects_buffered_plaintext(self): + sock = mock.Mock(spec=socket.socket) + sock.recv.return_value = ( + b'* OK [CAPABILITY IMAP4rev1 STARTTLS] ready\r\n') + with mock.patch.object(imaplib.IMAP4, '_create_socket', + return_value=sock): + client = imaplib.IMAP4('localhost') + self.addCleanup(client.shutdown) + + tag = client.tagpre + str(client.tagnum).encode('ascii') + next_tag = client.tagpre + str(client.tagnum + 1).encode('ascii') + # Return the STARTTLS response and injected plaintext in one recv(), + # so the test does not depend on how TCP packets are split. + sock.recv.side_effect = [ + tag + b' OK Begin TLS\r\n' + b'* CAPABILITY IMAP4rev1 INJECTED\r\n' + + next_tag + b' OK CAPABILITY completed\r\n' + ] + context = mock.Mock() + context.wrap_socket.return_value = sock + with self.assertRaisesRegex(imaplib.IMAP4.abort, + 'unexpected data after STARTTLS response'): + client.starttls(context) + context.wrap_socket.assert_not_called() + self.assertFalse(client._tls_established) + if ssl: class SecureTCPServer(socketserver.TCPServer): @@ -2344,10 +2371,67 @@ def test_file_property_setter_should_not_close_previous_file(self): f.close.assert_not_called() +class StartTLSHandler(SimpleIMAPHandler): + capabilities = 'STARTTLS' + + def cmd_STARTTLS(self, tag, args): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(CERTFILE) + self._send_tagged(tag, 'OK', 'Begin TLS') + self.connection = context.wrap_socket(self.connection, + server_side=True) + self.rfile.close() + self.wfile.close() + self.rfile = self.connection.makefile('rb') + self.wfile = self.connection.makefile('wb', buffering=0) + self.capabilities = 'AUTH=PLAIN' + + def finish(self): + try: + super().finish() + finally: + self.connection.close() + + class NewIMAPTests(NewIMAPTestsMixin, unittest.TestCase): imap_class = imaplib.IMAP4 server_class = socketserver.TCPServer + @unittest.skipUnless(ssl, "SSL not available") + def test_starttls(self): + _, server = self._setup(StartTLSHandler, connect=False) + self.client = client = self.imap_class( + 'localhost', server.server_address[1], + timeout=support.LOOPBACK_TIMEOUT) + context = ssl.create_default_context(cafile=CAFILE) + self.assertEqual(client.capabilities, ('IMAP4REV1', 'STARTTLS')) + self.assertEqual(client.starttls(context), ('OK', [None])) + self.assertIsInstance(client.sock, ssl.SSLSocket) + self.assertEqual(client.capabilities, ('IMAP4REV1', 'AUTH=PLAIN')) + client.logout() + + @unittest.skipUnless(ssl, "SSL not available") + def test_starttls_discards_untagged_responses(self): + # Untagged responses received before STARTTLS could have been + # injected by a MITM, so they must not survive the switch to TLS. + class InjectingStartTLSHandler(StartTLSHandler): + def cmd_STARTTLS(self, tag, args): + self._send_textline('* 3 EXISTS') + self._send_textline(r'* FLAGS (\Injected)') + super().cmd_STARTTLS(tag, args) + + _, server = self._setup(InjectingStartTLSHandler, connect=False) + self.client = client = self.imap_class( + 'localhost', server.server_address[1], + timeout=support.LOOPBACK_TIMEOUT) + context = ssl.create_default_context(cafile=CAFILE) + self.assertEqual(client.starttls(context), ('OK', [None])) + self.assertIsInstance(client.sock, ssl.SSLSocket) + self.assertEqual(client.untagged_responses, {}) + self.assertEqual(client.response('EXISTS'), ('EXISTS', [None])) + self.assertEqual(client.response('FLAGS'), ('FLAGS', [None])) + client.logout() + @unittest.skipUnless(ssl, "SSL not available") class NewIMAPSSLTests(NewIMAPTestsMixin, unittest.TestCase): @@ -2381,6 +2465,7 @@ def test_ssl_verified(self): ssl_context=ssl_context) client.shutdown() + class ThreadedNetworkedTests(unittest.TestCase): server_class = socketserver.TCPServer imap_class = imaplib.IMAP4 diff --git a/Misc/NEWS.d/next/Security/2026-10-22-21-59-39.gh-issue-160369.imapTLS.rst b/Misc/NEWS.d/next/Security/2026-10-22-21-59-39.gh-issue-160369.imapTLS.rst new file mode 100644 index 000000000000000..35c0a7527457935 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-10-22-21-59-39.gh-issue-160369.imapTLS.rst @@ -0,0 +1,4 @@ +.. cve: CVE-2026-115559 + +:meth:`imaplib.IMAP4.starttls` now rejects unexpected plaintext buffered after +the STARTTLS response, preventing it from being processed as TLS-protected data. From d1ff5e51e3fbb30bde31fcd7c2c8ef0b3b1b1cef Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 10 Nov 2026 08:57:00 -0800 Subject: [PATCH 077/229] Python 3.16.0a2 --- Doc/c-api/code.rst | 2 +- Doc/c-api/conversion.rst | 2 +- Doc/c-api/dict.rst | 2 +- Doc/c-api/exceptions.rst | 6 +- Doc/c-api/file.rst | 2 +- Doc/c-api/frame.rst | 2 +- Doc/c-api/profiling.rst | 2 +- Doc/c-api/structures.rst | 2 +- Doc/c-api/synchronization.rst | 2 +- Doc/c-api/tuple.rst | 2 +- Doc/c-api/weakref.rst | 2 +- Doc/library/argparse.rst | 4 +- Doc/library/binascii.rst | 2 +- Doc/library/concurrent.interpreters.rst | 2 +- Doc/library/ctypes.rst | 4 +- Doc/library/datetime.rst | 4 +- Doc/library/fileinput.rst | 2 +- Doc/library/functools.rst | 2 +- Doc/library/gzip.rst | 2 +- Doc/library/os.rst | 2 +- Doc/library/quopri.rst | 4 +- Doc/library/reprlib.rst | 2 +- Doc/library/shutil.rst | 4 +- Doc/library/socket.rst | 2 +- Doc/library/ssl.rst | 58 +- Doc/library/sys.rst | 4 +- Doc/library/tarfile.rst | 6 +- Doc/library/uuid.rst | 2 +- Doc/library/weakref.rst | 10 +- Doc/library/xml.dom.rst | 2 +- Include/patchlevel.h | 4 +- Lib/pydoc_data/module_docs.py | 3 +- Lib/pydoc_data/topics.py | 16 +- Misc/NEWS.d/3.16.0a2.rst | 4357 +++++++++++++++++ ...-07-11-01-53-58.gh-issue-153285.YmSox4.rst | 2 - ...-08-27-23-24-29.gh-issue-155813.LxRQu9.rst | 3 - ...-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst | 1 - ...-09-12-15-33-26.gh-issue-157354.2hw6fe.rst | 2 - ...-09-22-09-13-06.gh-issue-157949.EZ8avp.rst | 2 - ...-10-15-11-20-48.gh-issue-159440.uIz6ms.rst | 3 - ...-10-15-17-19-41.gh-issue-159452.oJ1saO.rst | 2 - ...6-10-15-19-09-31.gh-issue-75382.DSrcra.rst | 4 - ...-10-18-12-00-00.gh-issue-159759.LibStA.rst | 4 - ...6-10-19-13-49-39.gh-issue-54767.IdrGm6.rst | 3 - ...10-19-17-05-00.gh-issue-159819.pyCfgSp.rst | 6 - ...-10-20-00-56-59.gh-issue-153195.35cRE-.rst | 5 - ...-10-20-12-06-22.gh-issue-131999.OWoYGM.rst | 1 - ...-10-20-17-50-00.gh-issue-143632.PrCtlM.rst | 3 - ...-10-20-20-00-00.gh-issue-144942.MdLibs.rst | 3 - ...-10-20-23-00-00.gh-issue-159998.FwDyn1.rst | 3 - ...-10-20-23-30-00.gh-issue-160004.CfgPth.rst | 4 - ...-10-21-14-52-56.gh-issue-128884.Xs5jK-.rst | 2 - ...-00-00.gh-issue-128883.perf-trampoline.rst | 2 - ...-10-22-18-57-02.gh-issue-128884.3zR5Yp.rst | 3 - ...-10-25-15-30-00.gh-issue-160689.bootsp.rst | 5 - ...-10-26-21-00-00.gh-issue-160704.ZRYLKF.rst | 2 - ...6-11-04-14-37-13.gh-issue-75929.nELdwE.rst | 2 - ...-11-04-15-00-00.gh-issue-160924.WQgVVh.rst | 3 - ...-11-06-09-16-30.gh-issue-161467.oSuzx_.rst | 1 - ...-07-09-13-52-17.gh-issue-153410.neaDgY.rst | 3 - ...-08-20-12-00-00.gh-issue-156133.GetLoc.rst | 3 - ...-09-12-18-37-22.gh-issue-157384.uFbdle.rst | 3 - ...-10-19-19-33-35.gh-issue-159739.dee9a0.rst | 1 - ...-10-19-20-38-17.gh-issue-159739.eb9cdc.rst | 2 - ...-10-19-21-05-12.gh-issue-159739.Rk3fTq.rst | 3 - ...-10-21-19-55-00.gh-issue-159739.ctxt01.rst | 2 - ...-11-02-13-15-22.gh-issue-161172.U6kpuN.rst | 2 - ...-11-02-15-17-50.gh-issue-161185.4bpyDC.rst | 3 - ...-11-03-01-24-01.gh-issue-161226.MPrJKp.rst | 2 - ...-11-03-18-27-42.gh-issue-160962.06816I.rst | 2 - ...-11-05-10-51-17.gh-issue-161407.rVoqaQ.rst | 3 - ...-11-05-16-56-57.gh-issue-160109.Fvs4m2.rst | 2 - ...-03-07-22-38-07.gh-issue-116477.AqwRAV.rst | 1 - ...-08-11-12-06-23.gh-issue-122901.B2Huom.rst | 1 - ...-10-27-22-23-57.gh-issue-125873.BG97KU.rst | 1 - ...-08-19-16-10-26.gh-issue-137966.nEgFAt.rst | 2 - ...-03-14-11-20-52.gh-issue-145019.ls2EXo.rst | 2 - ...6-04-12-13-48-26.gh-issue-69605.T9HWOL.rst | 2 - ...6-04-15-21-55-45.gh-issue-69605.NwoP6l.rst | 2 - ...-04-23-08-36-47.gh-issue-148891.AMMHow.rst | 1 - ...-05-21-14-21-06.gh-issue-150075.dLI21Z.rst | 1 - ...-06-07-22-03-20.gh-issue-151060.tdexEE.rst | 2 - ...-06-20-18-00-00.gh-issue-151818.LsApDf.rst | 3 - ...-06-26-00-02-29.gh-issue-149816.JKN-5B.rst | 3 - ...-06-29-23-01-07.gh-issue-152631.LKgCM_.rst | 2 - ...-07-04-18-00-00.gh-issue-153014.gCdBg1.rst | 2 - ...-07-11-15-00-13.gh-issue-153568.atomfp.rst | 2 - ...-07-21-10-42-15.gh-issue-154318.Kq3Wm7.rst | 2 - ...-08-21-14-37-50.gh-issue-127716.KrPth8.rst | 10 - ...-08-27-00-00-00.gh-issue-156399.aB3xYz.rst | 4 - ...-08-27-17-26-21.gh-issue-106045.wzTH7f.rst | 1 - ...-08-30-16-30-00.gh-issue-135385.Wn7Rd4.rst | 3 - ...-09-07-09-52-21.gh-issue-156443.tT40sm.rst | 1 - ...-09-08-17-40-00.gh-issue-157194.17Ffhk.rst | 4 - ...09-13-10-00-00.gh-issue-157361.aiterpy.rst | 5 - ...-09-21-12-44-38.gh-issue-132414.uwxmd5.rst | 4 - ...-09-25-11-05-00.gh-issue-157746.q7Lk3d.rst | 3 - ...-09-28-10-12-25.gh-issue-158304.3pdcnJ.rst | 4 - ...-09-29-14-35-00.gh-issue-158121.Kj4nQ8.rst | 2 - ...-09-30-21-29-54.gh-issue-158529.kZO7mX.rst | 2 - ...-10-01-00-00-00.gh-issue-158525.Xw3K9a.rst | 1 - ...-10-03-13-00-00.gh-issue-158197.Qk7sVd.rst | 5 - ...-10-04-20-04-55.gh-issue-158816.9ulhWb.rst | 3 - ...6-10-05-13-19-49.gh-issue-87613.JLTGBy.rst | 2 - ...00-00-00.gh-issue-158973.warnings-fork.rst | 3 - ...-10-09-16-04-40.gh-issue-159041.PUP8bU.rst | 2 - ...-10-10-20-21-13.gh-issue-159111.XVf2k-.rst | 1 - ...-10-11-06-36-25.gh-issue-159188.Fq7wLm.rst | 5 - ...6-10-12-11-59-37.gh-issue-68241.jVpLIJ.rst | 1 - ...-10-12-16-00-34.gh-issue-159244.ThiobE.rst | 3 - ...-10-12-20-00-00.gh-issue-159136.fnAttr.rst | 8 - ...-10-13-04-03-48.gh-issue-159288.vhqBQG.rst | 3 - ...-10-13-06-23-56.gh-issue-159294.9LTsNA.rst | 3 - ...-10-13-16-38-00.gh-issue-159320._Yhg1s.rst | 2 - ...-10-14-07-36-44.gh-issue-159399.cjuE0N.rst | 3 - ...-10-14-11-40-06.gh-issue-159430.OWpyPm.rst | 2 - ...-10-14-18-45-35.gh-issue-159431.mVTMsA.rst | 4 - ...-10-14-19-16-53.gh-issue-159434.T4AsrQ.rst | 4 - ...-10-14-23-05-00.gh-issue-132042.k7Qs2w.rst | 3 - ...-10-15-10-15-27.gh-issue-159449.xsdzFf.rst | 1 - ...6-10-15-12-10-00.gh-issue-87613.Wr4fvc.rst | 1 - ...-10-15-12-36-09.gh-issue-130279.N9N-sw.rst | 2 - ...6-10-15-12-40-00.gh-issue-87613.Sl1ceV.rst | 1 - ...-10-16-18-25-48.gh-issue-159564.WMRRFA.rst | 3 - ...-10-17-02-47-26.gh-issue-159603.OFagJz.rst | 2 - ...-10-18-00-30-00.gh-issue-159652.Qk3vLm.rst | 6 - ...-10-19-16-38-39.gh-issue-159813.wHfSOc.rst | 2 - ...-10-19-19-33-35.gh-issue-159739.09590b.rst | 1 - ...-10-19-19-33-35.gh-issue-159739.173d49.rst | 2 - ...-10-19-19-33-35.gh-issue-159739.1fc3af.rst | 2 - ...-10-19-19-33-35.gh-issue-159739.a8775b.rst | 2 - ...-10-19-19-33-35.gh-issue-159739.c25c27.rst | 2 - ...-10-19-19-33-35.gh-issue-159739.f81734.rst | 2 - ...-10-19-20-38-17.gh-issue-159739.55fd08.rst | 2 - ...-10-19-20-38-17.gh-issue-159739.78e530.rst | 2 - ...-10-19-20-38-17.gh-issue-159739.b62fc9.rst | 2 - ...-10-19-20-38-17.gh-issue-159739.fc0ed2.rst | 2 - ...10-19-21-30-00.gh-issue-159860.qsbr128.rst | 4 - ...-10-20-00-00-00.gh-issue-159739.848b18.rst | 2 - ...-10-20-00-00-00.gh-issue-159739.a3cc5d.rst | 1 - ...-10-20-00-00-00.gh-issue-159739.bf569b.rst | 2 - ...-10-20-00-00-00.gh-issue-159739.d76e6d.rst | 2 - ...-10-20-00-00-00.gh-issue-159739.da0f04.rst | 2 - ...10-20-10-00-00.gh-issue-158195.UnpinTL.rst | 7 - ...-10-20-20-38-38.gh-issue-159101.vFwrZY.rst | 2 - ...-10-20-22-00-00.gh-issue-159963.pLkHld.rst | 6 - ...-10-21-14-00-00.gh-issue-159493.sfLf1b.rst | 3 - ...18-00-00.gh-issue-160368.sem-clockwait.rst | 3 - ...-10-21-19-55-00.gh-issue-159739.codc01.rst | 2 - ...-10-21-19-55-00.gh-issue-159739.tali01.rst | 2 - ...-10-21-21-26-10.gh-issue-159739.Aw61LR.rst | 1 - ...-10-21-21-26-10.gh-issue-159739.CbTE4N.rst | 1 - ...-10-21-21-26-10.gh-issue-159739.Zw43uX.rst | 1 - ...-10-21-23-18-38.gh-issue-160132.Nj4rQa.rst | 3 - ...-10-22-00-30-02.gh-issue-160179.dQ2e7m.rst | 3 - ...-10-22-00-30-21.gh-issue-160162.lShf32.rst | 3 - ...-10-22-12-30-11.gh-issue-106045.fkMW34.rst | 5 - ...-10-22-14-45-00.gh-issue-160245.mEmO24.rst | 2 - ...-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst | 1 - ...-10-23-19-37-00.gh-issue-160050.aB3x9Q.rst | 1 - ...-10-23-20-15-24.gh-issue-160491.yIQ_Vf.rst | 3 - ...-10-24-19-07-34.gh-issue-153568.6sVdqs.rst | 3 - ...-10-26-12-00-00.gh-issue-160682.Q7mK2x.rst | 2 - ...-10-28-13-47-43.gh-issue-160773.7VFxkA.rst | 2 - ...-11-02-05-50-35.gh-issue-160872.x8NvzN.rst | 1 - ...-11-03-05-52-49.gh-issue-161229.8pOPOI.rst | 2 - ...-11-03-13-44-26.gh-issue-159184.X_sOdY.rst | 2 - ...-11-03-20-28-15.gh-issue-161288.LikC_U.rst | 3 - ...-11-05-10-19-48.gh-issue-161419.wc4DV0.rst | 3 - ...-11-05-21-00-57.gh-issue-161366.ouxAvv.rst | 4 - ...-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst | 4 - ...-11-06-15-15-00.gh-issue-161433.A9b3Kq.rst | 2 - ...-11-06-21-21-04.gh-issue-161242.d4vNgC.rst | 3 - ...-11-07-09-30-00.gh-issue-161420.mIdFre.rst | 2 - ...-11-09-14-30-00.gh-issue-160267.sTfrEe.rst | 4 - ...-09-04-16-01-20.gh-issue-107689.LFLFtG.rst | 1 - ...-08-31-02-28-21.gh-issue-138277.kkORph.rst | 2 - ...-10-16-15-12-36.gh-issue-159517.uzRtAd.rst | 2 - ...-11-06-17-12-40.gh-issue-161477.smXIKC.rst | 3 - .../2021-05-24-14-41-29.bpo-44170._v28mv.rst | 1 - .../2021-06-11-20-00-16.bpo-19094.rMRoIL.rst | 6 - ...3-01-26-09-36-24.gh-issue-61448.SOAkvt.rst | 3 - ...-12-27-15-41-55.gh-issue-128289.-hCzEc.rst | 2 - ...-02-04-09-01-06.gh-issue-129605.wKGNXQ.rst | 1 - ...03-13-20-21-00.gh-issue-115810.qwrb231.rst | 2 - ...5-06-12-15-39-48.gh-issue-89730.Lu35Fu.rst | 2 - ...-09-18-14-13-00.gh-issue-139122.m3lp66.rst | 2 - ...-09-18-18-03-32.gh-issue-139105.1QQ_W4.rst | 1 - ...-10-13-15-43-09.gh-issue-140036.b_59uN.rst | 2 - ...5-10-20-13-08-04.gh-issue-84027.HjBO0B.rst | 1 - ...6-01-06-09-13-53.gh-issue-84644.V_cYP3.rst | 5 - ...-03-23-00-04-13.gh-issue-146065.FIdn8D.rst | 2 - ...-03-28-10-27-46.gh-issue-146169.RBF1xp.rst | 4 - ...-03-28-14-34-18.gh-issue-143376.FvWTwi.rst | 2 - ...-04-27-20-30-16.gh-issue-149083.xcmpSO.rst | 2 - ...-05-03-22-52-07.gh-issue-149309.gXhk9Y.rst | 1 - ...-05-09-12-58-04.gh-issue-149527.hX36Yh.rst | 3 - ...-05-21-16-15-01.gh-issue-105578.oq-0pg.rst | 2 - ...-06-18-11-26-00.gh-issue-151633.uJzHdc.rst | 2 - ...-06-27-08-31-30.gh-issue-152373.3cd0aa.rst | 2 - ...-06-28-14-37-01.gh-issue-152490.A8qLbF.rst | 1 - ...6-06-30-08-39-07.gh-issue-62222.Zq7x2P.rst | 6 - ...-07-04-12-52-22.gh-issue-149816.XQOutB.rst | 3 - ...-07-11-16-00-00.gh-issue-153605.Rk7Xz3.rst | 2 - ...-07-13-00-10-00.gh-issue-153679.RJ2WGB.rst | 3 - ...6-07-16-21-30-00.gh-issue-74876.wR3fXk.rst | 8 - ...-07-21-00-02-11.gh-issue-148228.D3rP8m.rst | 3 - ...-08-04-16-38-59.gh-issue-155152.qN1EEN.rst | 1 - ...-08-05-10-13-23.gh-issue-153711.9g9dVz.rst | 3 - ...-08-09-14-06-27.gh-issue-155426.zZVPMd.rst | 4 - ...-08-09-14-26-48.gh-issue-155428.wi0_fI.rst | 3 - ...-08-15-11-30-42.gh-issue-155782.JUmDP4.rst | 2 - ...-08-15-11-49-30.gh-issue-155830.cY5xgK.rst | 2 - ...-08-15-12-08-33.gh-issue-155833.Pi2Yjp.rst | 2 - ...-08-20-09-56-45.gh-issue-156078.-E7gZc.rst | 2 - ....gh-issue-156057.tarfile-wrap-comp-err.rst | 5 - ...-08-23-16-33-00.gh-issue-151126.GUvtG7.rst | 2 - ...-08-29-13-41-56.gh-issue-154675.cYvE-Z.rst | 2 - ...-08-29-14-28-46.gh-issue-156568.mkxLY8.rst | 2 - ...-09-01-00-00-00.gh-issue-156784.8gvtt4.rst | 3 - ...-09-05-17-29-50.gh-issue-160837.W6kHva.rst | 2 - ...9-17-10-30-00.gh-issue-158252.strptime.rst | 4 - ...6-09-17-20-36-01.gh-issue-157290.NfFmt.rst | 4 - ...6-09-18-00-00-00.gh-issue-49960.MDT3jE.rst | 3 - ...-09-20-12-00-00.gh-issue-157856.wq4xuF.rst | 3 - ...-09-20-21-24-05.gh-issue-157880.aE7wIu.rst | 2 - ...-09-21-14-41-07.gh-issue-157899.s6vBQC.rst | 2 - ...-09-21-21-07-12.gh-issue-157928.ZPlNYC.rst | 1 - ...6-09-23-09-21-25.gh-issue-89977.fmia18.rst | 3 - ...-09-23-19-59-03.gh-issue-158004.p68swW.rst | 2 - ...9-23-23-34-30.gh-issue-144812.ujptbVOI.rst | 3 - ...-09-24-14-59-08.gh-issue-158095.lcKOT4.rst | 2 - ...-09-25-01-29-55.gh-issue-158126.nWmFnq.rst | 3 - ...-09-25-17-04-45.gh-issue-158159.lbEalG.rst | 3 - ...-09-26-05-18-21.gh-issue-144650.zqHLOW.rst | 2 - ...-09-27-14-00-00.gh-issue-158277.aBcDeF.rst | 3 - ...-09-28-00-00-00.gh-issue-158370.ctxmgr.rst | 2 - ...-09-29-19-34-30.gh-issue-158192.6wPjiU.rst | 1 - ...-10-05-08-35-10.gh-issue-130698.iibO9V.rst | 2 - ...-10-05-14-58-12.gh-issue-158839.wYd6fI.rst | 5 - ...-10-07-10-59-27.gh-issue-155233.YGuQGK.rst | 1 - ...-10-07-17-30-00.gh-issue-158287.xY9zQp.rst | 1 - ...-10-08-11-58-52.gh-issue-158628.NJxwuw.rst | 1 - ...-10-08-21-02-38.gh-issue-158998.DZYJI7.rst | 1 - ...-10-10-22-14-22.gh-issue-150751.JInuGL.rst | 5 - ...-10-10-22-29-13.gh-issue-159049.eJUiKL.rst | 3 - ...-10-11-08-42-00.gh-issue-159184.Tq2nSe.rst | 3 - ...6-10-11-13-08-07.gh-issue-88037.CzKcxh.rst | 3 - ...-10-11-13-40-44.gh-issue-159168.z3KjB8.rst | 2 - ...-10-12-00-00-00.gh-issue-159044.Tr8kXv.rst | 4 - ...-10-12-11-36-48.gh-issue-158856.jLR-Lv.rst | 1 - ...6-10-12-12-00-00.gh-issue-72113.njkm8v.rst | 2 - ...-10-12-13-05-00.gh-issue-158501.n7Qp3k.rst | 4 - ...-10-12-18-21-06.gh-issue-159240.a1b2c3.rst | 6 - ...-10-12-21-20-00.gh-issue-159225.T8jlor.rst | 1 - ...-10-13-12-00-00.gh-issue-159304.mFbYq2.rst | 2 - ...-10-13-19-10-00.gh-issue-159322.hP6k3A.rst | 2 - ...-10-14-08-50-00.gh-issue-159398.k2Xd7q.rst | 2 - ...-10-14-10-26-10.gh-issue-159387.xMbQbB.rst | 2 - ...-10-14-11-41-00.gh-issue-159430.ZEW-mx.rst | 2 - ...6-10-14-14-47-05.gh-issue-81954.KvM0gY.rst | 2 - ...-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst | 2 - ...-10-15-18-38-22.gh-issue-159500.OjAQ8W.rst | 4 - ...-10-16-01-56-15.gh-issue-159503.WgYiow.rst | 3 - ...-10-16-10-25-22.gh-issue-159458.q02EnY.rst | 1 - ...26-10-16-13-00-00.gh-issue-159435.OMNI.rst | 3 - ...-10-16-18-23-03.gh-issue-159558.ctRelk.rst | 4 - ...-10-17-15-22-16.gh-issue-159620.Ynsu5L.rst | 3 - ...17-19-33-13.gh-issue-159643.EJfe0IXetV.rst | 3 - ...0-17-19-41-12.gh-issue-157210.o9hycSeJ.rst | 5 - ...0-18-08-30-10.gh-issue-155976.feSum4_j.rst | 3 - ...-20-00.gh-issue-159465.partial-escapes.rst | 2 - ...0-18-16-09-24.gh-issue-159684.VHVYjVuE.rst | 3 - ...026-10-19-00-00-00.gh-issue-149015.bdb.rst | 2 - ...-10-19-12-00-00.gh-issue-159714.j59fix.rst | 2 - ...6-10-19-12-00-00.gh-issue-78327.Qk3hzA.rst | 2 - ...-10-19-16-00-00.gh-issue-139122.bYtEs1.rst | 3 - ...6-10-19-16-26-57.gh-issue-71990.K3FlJ9.rst | 2 - ...-10-19-16-49-09.gh-issue-159706.ushGrx.rst | 3 - ...-10-19-19-33-35.gh-issue-159739.a1c3c7.rst | 2 - ...-10-19-19-33-35.gh-issue-159739.b5fbca.rst | 1 - ...-10-19-20-38-17.gh-issue-159739.08caa7.rst | 2 - ...-10-19-20-38-17.gh-issue-159739.243aa8.rst | 2 - ...-10-19-20-38-17.gh-issue-159739.5a6e94.rst | 2 - ...-10-19-20-38-17.gh-issue-159739.a81270.rst | 1 - ...-10-19-20-38-17.gh-issue-159739.bc65da.rst | 2 - ...-10-19-20-38-17.gh-issue-159739.ff088c.rst | 1 - ...-10-19-21-20-28.gh-issue-159786.fThehQ.rst | 6 - ...-10-19-23-25-29.gh-issue-159811.P6yy0z.rst | 3 - ...-10-19-23-25-29.gh-issue-159811.cMB4DO.rst | 3 - ...-10-20-00-00-00.gh-issue-159739.0c4ae9.rst | 2 - ...-10-20-00-00-00.gh-issue-159739.1e4599.rst | 2 - ...-10-20-00-00-00.gh-issue-159739.35942a.rst | 1 - ...-10-20-00-00-00.gh-issue-159739.3660a6.rst | 2 - ...-10-20-00-00-00.gh-issue-159739.4c9783.rst | 2 - ...-10-20-00-00-00.gh-issue-159739.5d4269.rst | 1 - ...-10-20-00-00-00.gh-issue-159739.9ef89c.rst | 2 - ...-10-20-00-00-00.gh-issue-159739.eb8987.rst | 2 - ...-10-20-00-12-05.gh-issue-159690.vZgLP6.rst | 3 - ...-10-20-12-30-00.gh-issue-159864.4FZ6MR.rst | 2 - ...-10-20-16-00-00.gh-issue-159868.qLhYkP.rst | 3 - ...-10-20-17-29-14.gh-issue-138641.FN6qM0.rst | 1 - ...-10-21-15-10-00.gh-issue-160125.pdbpkg.rst | 3 - ...-10-21-15-42-29.gh-issue-159875.p6j3yW.rst | 7 - ...6-10-21-19-38-33.gh-issue-89721.uMpxRw.rst | 3 - ...-10-21-19-40-00.gh-issue-100609.THXgRC.rst | 5 - ...-10-21-19-55-00.gh-issue-159739.sque01.rst | 2 - ...-10-21-19-55-00.gh-issue-159739.tloc01.rst | 2 - ...-10-21-20-05-00.gh-issue-160104.BUAL5C.rst | 6 - ...-10-21-21-26-10.gh-issue-159739.JIqZRY.rst | 1 - ...-10-21-21-26-10.gh-issue-159739.b4ueJ7.rst | 1 - ...-10-21-21-26-10.gh-issue-159739.ghGctt.rst | 1 - ...-10-21-21-26-10.gh-issue-159739.k8uuGe.rst | 3 - ...-10-21-21-26-10.gh-issue-159739.rVWF2a.rst | 1 - ...-10-21-21-26-10.gh-issue-159739.yBc6Ei.rst | 1 - ...-10-22-00-29-50.gh-issue-160176.tkNst1.rst | 4 - ...-10-22-00-29-55.gh-issue-160185.BPz40E.rst | 5 - ...-10-22-00-30-04.gh-issue-160187.9dcj2B.rst | 4 - ...-10-22-00-32-51.gh-issue-160181.nnEeg1.rst | 3 - ...-10-22-00-33-42.gh-issue-160171.11l4om.rst | 4 - ...-10-22-03-55-33.gh-issue-159739.a8bf30.rst | 2 - ...-10-22-03-55-33.gh-issue-159739.b4c371.rst | 2 - ...-10-22-03-55-33.gh-issue-159739.ce26e1.rst | 4 - ...-10-22-05-19-56.gh-issue-161090.pxZ1IA.rst | 3 - ...-10-22-05-20-02.gh-issue-161092.OtC5l6.rst | 4 - ...0-22-05-20-02.gh-issue-161102.gqDOIM7X.rst | 4 - ...-10-22-05-20-06.gh-issue-161100.pLgnlA.rst | 3 - ...-10-22-05-20-11.gh-issue-161110.miKrSQ.rst | 4 - ...-10-22-05-20-18.gh-issue-161108.xOVsRA.rst | 4 - ...-10-22-05-20-27.gh-issue-161098.voMVnd.rst | 3 - ...-10-22-05-20-44.gh-issue-161106.t0ux5q.rst | 3 - ...-10-22-05-20-52.gh-issue-161096.vkhfyy.rst | 4 - ...-10-22-05-21-00.gh-issue-161094.MDjCNm.rst | 4 - ...-10-22-05-23-40.gh-issue-161088._-HEWz.rst | 3 - ...-10-22-05-24-21.gh-issue-161112.BFrZRw.rst | 3 - ...-10-22-05-24-34.gh-issue-161086.FGPjqA.rst | 2 - ...-10-22-08-04-26.gh-issue-160297.hL4kQz.rst | 4 - ...-10-22-08-06-39.gh-issue-160513.q7RkZd.rst | 5 - ...-10-22-08-09-29.gh-issue-160406.q7Rk2d.rst | 4 - ...-10-22-08-15-56.gh-issue-160324.sIoRlz.rst | 3 - ...-10-22-08-17-51.gh-issue-160316.sRe9pQ.rst | 5 - ...-10-22-08-18-31.gh-issue-160522.cJkSt8.rst | 4 - ...-10-22-08-34-22.gh-issue-160310.q7Zm2K.rst | 4 - ...-10-22-16-18-42.gh-issue-159739.TioR3a.rst | 4 - ...-10-22-16-23-01.gh-issue-159739.CjkR5a.rst | 6 - ...-10-22-16-23-01.gh-issue-159739.XmlH3a.rst | 3 - ...-10-22-16-30-00.gh-issue-159739.mFWntK.rst | 3 - ...-10-22-16-30-00.gh-issue-159739.tRq417.rst | 3 - ...-10-22-18-30-00.gh-issue-160314.q7RkZd.rst | 4 - ...-10-22-18-30-00.gh-issue-160326.nAn7Tq.rst | 5 - ...-10-22-18-30-00.gh-issue-160520.sSdTmo.rst | 4 - ...-10-22-22-51-40.gh-issue-160386.wYyboz.rst | 11 - ...6-10-23-21-52-40.gh-issue-60299.Af6IzL.rst | 4 - ...-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst | 4 - ...-10-23-23-21-21.gh-issue-160549.q7ZdMt.rst | 4 - ...-10-24-01-53-42.gh-issue-160433.DSTlog.rst | 2 - ...-10-24-07-58-00.gh-issue-160561.b3Kq1z.rst | 1 - ...-10-24-22-32-23.gh-issue-160596.RKll-Z.rst | 2 - ...-10-26-16-15-29.gh-issue-160427.9G4EBb.rst | 2 - ...-10-26-16-17-45.gh-issue-160708.w7z33g.rst | 3 - ...-10-26-21-37-42.gh-issue-158500.zxyuq5.rst | 4 - ...-10-27-15-26-53.gh-issue-160781.4VSeQK.rst | 3 - ...-10-27-15-27-32.gh-issue-160781.kARuHu.rst | 2 - ...-10-27-15-42-16.gh-issue-160781.VGLB9S.rst | 2 - ...-10-28-07-26-47.gh-issue-160804.R5HeTQ.rst | 3 - ...-10-28-07-34-05.gh-issue-160806.ZZRkhi.rst | 3 - ...-10-28-08-12-33.gh-issue-160811.wQ3xT7.rst | 2 - ...6-10-28-12-00-00.gh-issue-53852.Pr0xy4.rst | 3 - ...10-28-13-23-28.gh-issue-59515.wt31yNN4.rst | 4 - ...-28-16-40-00.gh-issue-66333.pdb-sigint.rst | 3 - ...-10-28-20-36-44.gh-issue-109564.WqoyNE.rst | 4 - ...10-29-19-42-53.gh-issue-53089.ewaIBvY6.rst | 5 - ...6-10-29-20-52-57.gh-issue-50720.Kd1VMR.rst | 4 - ...-10-29-21-36-08.gh-issue-160929.mK8n2Q.rst | 3 - ...-10-29-21-43-53.gh-issue-160429.eWapMs.rst | 3 - ...-10-29-23-25-31.gh-issue-160782.1ZwCOr.rst | 2 - ...-10-30-10-36-30.gh-issue-160782.n6JCUU.rst | 2 - ...-10-30-12-00-00.gh-issue-137549.Tm0Bxg.rst | 3 - ...-10-31-21-34-00.gh-issue-161039.k7_pTD.rst | 1 - ...-11-01-19-38-33.gh-issue-161083.PM21lv.rst | 2 - ...-11-01-22-07-24.gh-issue-161124.c2YmcV.rst | 2 - ...-11-02-13-11-30.gh-issue-161166.zfFDlW.rst | 2 - ...-11-02-13-26-36.gh-issue-161179.H8Yyaw.rst | 2 - ...-11-02-16-28-11.gh-issue-161178.D3-23R.rst | 2 - ...-11-02-21-10-00.gh-issue-161212.pip263.rst | 1 - ...-11-03-10-33-33.gh-issue-161233._9VrfL.rst | 2 - ...-11-03-12-00-00.gh-issue-161076.nOdLy1.rst | 5 - ...6-11-03-12-00-00.gh-issue-93821.aCcRsT.rst | 4 - ...-11-03-18-56-28.gh-issue-161259.iyZz6y.rst | 3 - ...-11-03-21-26-09.gh-issue-161295.IJqoJM.rst | 2 - ...-11-05-10-05-00.gh-issue-161387.def456.rst | 2 - ...-11-05-11-56-50.gh-issue-161389.EIl2E6.rst | 2 - ...-11-05-11-59-58.gh-issue-161414.Xnxe8A.rst | 2 - ...-11-05-12-00-24.gh-issue-161415.NJhrAR.rst | 3 - ...-11-06-00-00-00.gh-issue-142782.qU7nE4.rst | 3 - ...6-11-06-05-42-11.gh-issue-71199.qJN4cV.rst | 3 - ...-11-06-11-59-26.gh-issue-161475.82LWf4.rst | 2 - ...-08-31-16-47-33.gh-issue-157953.KxCFSN.rst | 1 - ...-10-06-12-30-00.gh-issue-158928.Zd9Ybv.rst | 2 - ...-10-15-03-47-08.gh-issue-159428.XHRKwY.rst | 2 - ...-10-20-13-15-00.gh-issue-159894.8Kp4mN.rst | 1 - ...-10-21-16-55-07.gh-issue-160040.9PjFjC.rst | 12 - ...-10-21-19-58-53.gh-issue-160097.-opfAL.rst | 5 - ...-10-22-08-04-33.gh-issue-160300.q7Zk2B.rst | 3 - ...-10-22-08-05-32.gh-issue-160420.tZ9dRv.rst | 4 - ...-10-22-08-07-52.gh-issue-161323.q7Drv2.rst | 5 - ...-10-22-08-08-23.gh-issue-160506.rN4dLq.rst | 4 - ...-10-22-08-08-57.gh-issue-160499.pf9H9g.rst | 5 - ...-10-22-11-09-30.gh-issue-160279.9BtxeF.rst | 2 - ...10-22-21-59-39.gh-issue-160369.imapTLS.rst | 4 - ...-10-27-16-25-40.gh-issue-160898.epi53X.rst | 4 - ...-11-02-18-30-09.gh-issue-161192.2ScSRO.rst | 3 - ...-11-06-16-22-47.gh-issue-161503.FkIl87.rst | 4 - ...11-07-01-43-49.gh-issue-161569.cookiej.rst | 7 - ...-11-07-15-56-47.gh-issue-161705.OjDK2d.rst | 6 - ...-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst | 3 - ...-10-07-12-00-58.gh-issue-158975.yR8q2L.rst | 3 - ...-10-13-13-34-00.gh-issue-159293.Lr8q2N.rst | 2 - ...-10-19-17-05-00.gh-issue-107862.qH3sTa.rst | 2 - ...-10-28-20-40-00.gh-issue-158646.k2Xw7q.rst | 3 - ...-11-03-13-12-50.gh-issue-161263.Voe8VM.rst | 2 - ...-11-05-16-20-23.gh-issue-161430.M7b8N6.rst | 2 - ...-11-06-23-15-23.gh-issue-161517.LkAlDt.rst | 3 - ...-11-07-01-45-00.gh-issue-160334.F2n8qA.rst | 2 - ...03-25-18-00-00.gh-issue-131725.qwfh321.rst | 2 - ...-10-14-09-48-26.gh-issue-159385.0aWOA9.rst | 1 - ...-10-20-12-50-00.gh-issue-114258.slotNx.rst | 3 - ...6-10-21-11-53-40.gh-issue-87613.tmKBlg.rst | 2 - ...-10-21-15-26-12.gh-issue-160135.UP-DlO.rst | 1 - ...-11-04-16-34-48.gh-issue-161367.bO5MGy.rst | 3 - ...-09-14-16-49-44.gh-issue-157418.r2IggA.rst | 3 - ...-10-15-18-45-00.gh-issue-159118.wmiMTA.rst | 1 - ...-10-20-21-37-34.gh-issue-159880.eKBiLr.rst | 3 - ...-10-22-14-04-17.gh-issue-160239.A1b2C3.rst | 1 - ...-10-19-19-33-35.gh-issue-159739.aaf575.rst | 1 - README.rst | 4 +- 436 files changed, 4445 insertions(+), 1185 deletions(-) create mode 100644 Misc/NEWS.d/3.16.0a2.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-07-11-01-53-58.gh-issue-153285.YmSox4.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-08-27-23-24-29.gh-issue-155813.LxRQu9.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-09-12-15-33-26.gh-issue-157354.2hw6fe.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-09-22-09-13-06.gh-issue-157949.EZ8avp.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-15-11-20-48.gh-issue-159440.uIz6ms.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-15-17-19-41.gh-issue-159452.oJ1saO.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-15-19-09-31.gh-issue-75382.DSrcra.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-18-12-00-00.gh-issue-159759.LibStA.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-19-13-49-39.gh-issue-54767.IdrGm6.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-19-17-05-00.gh-issue-159819.pyCfgSp.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-20-00-56-59.gh-issue-153195.35cRE-.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-20-12-06-22.gh-issue-131999.OWoYGM.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-20-17-50-00.gh-issue-143632.PrCtlM.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-20-20-00-00.gh-issue-144942.MdLibs.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-20-23-00-00.gh-issue-159998.FwDyn1.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-20-23-30-00.gh-issue-160004.CfgPth.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-21-14-52-56.gh-issue-128884.Xs5jK-.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-21-18-00-00.gh-issue-128883.perf-trampoline.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-22-18-57-02.gh-issue-128884.3zR5Yp.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-25-15-30-00.gh-issue-160689.bootsp.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-10-26-21-00-00.gh-issue-160704.ZRYLKF.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-11-04-14-37-13.gh-issue-75929.nELdwE.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-11-04-15-00-00.gh-issue-160924.WQgVVh.rst delete mode 100644 Misc/NEWS.d/next/Build/2026-11-06-09-16-30.gh-issue-161467.oSuzx_.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-07-09-13-52-17.gh-issue-153410.neaDgY.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-08-20-12-00-00.gh-issue-156133.GetLoc.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-09-12-18-37-22.gh-issue-157384.uFbdle.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-10-19-19-33-35.gh-issue-159739.dee9a0.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-10-19-20-38-17.gh-issue-159739.eb9cdc.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-10-19-21-05-12.gh-issue-159739.Rk3fTq.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-10-21-19-55-00.gh-issue-159739.ctxt01.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-11-02-13-15-22.gh-issue-161172.U6kpuN.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-11-02-15-17-50.gh-issue-161185.4bpyDC.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-11-03-01-24-01.gh-issue-161226.MPrJKp.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-11-03-18-27-42.gh-issue-160962.06816I.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-11-05-10-51-17.gh-issue-161407.rVoqaQ.rst delete mode 100644 Misc/NEWS.d/next/C_API/2026-11-05-16-56-57.gh-issue-160109.Fvs4m2.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-03-07-22-38-07.gh-issue-116477.AqwRAV.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-08-11-12-06-23.gh-issue-122901.B2Huom.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-22-23-57.gh-issue-125873.BG97KU.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-08-19-16-10-26.gh-issue-137966.nEgFAt.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-03-14-11-20-52.gh-issue-145019.ls2EXo.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-04-12-13-48-26.gh-issue-69605.T9HWOL.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-04-15-21-55-45.gh-issue-69605.NwoP6l.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-04-23-08-36-47.gh-issue-148891.AMMHow.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-21-14-21-06.gh-issue-150075.dLI21Z.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-06-07-22-03-20.gh-issue-151060.tdexEE.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-06-20-18-00-00.gh-issue-151818.LsApDf.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-00-02-29.gh-issue-149816.JKN-5B.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-23-01-07.gh-issue-152631.LKgCM_.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-18-00-00.gh-issue-153014.gCdBg1.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-00-13.gh-issue-153568.atomfp.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-21-10-42-15.gh-issue-154318.Kq3Wm7.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-14-37-50.gh-issue-127716.KrPth8.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-27-00-00-00.gh-issue-156399.aB3xYz.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-27-17-26-21.gh-issue-106045.wzTH7f.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-30-16-30-00.gh-issue-135385.Wn7Rd4.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-07-09-52-21.gh-issue-156443.tT40sm.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-08-17-40-00.gh-issue-157194.17Ffhk.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-10-00-00.gh-issue-157361.aiterpy.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-21-12-44-38.gh-issue-132414.uwxmd5.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-25-11-05-00.gh-issue-157746.q7Lk3d.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-28-10-12-25.gh-issue-158304.3pdcnJ.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-29-14-35-00.gh-issue-158121.Kj4nQ8.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-21-29-54.gh-issue-158529.kZO7mX.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-00-00-00.gh-issue-158525.Xw3K9a.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-13-00-00.gh-issue-158197.Qk7sVd.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-20-04-55.gh-issue-158816.9ulhWb.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-05-13-19-49.gh-issue-87613.JLTGBy.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-07-00-00-00.gh-issue-158973.warnings-fork.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-10-20-21-13.gh-issue-159111.XVf2k-.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-06-36-25.gh-issue-159188.Fq7wLm.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-11-59-37.gh-issue-68241.jVpLIJ.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-16-00-34.gh-issue-159244.ThiobE.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-20-00-00.gh-issue-159136.fnAttr.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-04-03-48.gh-issue-159288.vhqBQG.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-06-23-56.gh-issue-159294.9LTsNA.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-16-38-00.gh-issue-159320._Yhg1s.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-07-36-44.gh-issue-159399.cjuE0N.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-11-40-06.gh-issue-159430.OWpyPm.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-18-45-35.gh-issue-159431.mVTMsA.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-19-16-53.gh-issue-159434.T4AsrQ.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-23-05-00.gh-issue-132042.k7Qs2w.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-10-15-27.gh-issue-159449.xsdzFf.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-10-00.gh-issue-87613.Wr4fvc.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-36-09.gh-issue-130279.N9N-sw.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-40-00.gh-issue-87613.Sl1ceV.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-16-18-25-48.gh-issue-159564.WMRRFA.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-17-02-47-26.gh-issue-159603.OFagJz.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-18-00-30-00.gh-issue-159652.Qk3vLm.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-16-38-39.gh-issue-159813.wHfSOc.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.09590b.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.173d49.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.1fc3af.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.a8775b.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.c25c27.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.f81734.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.55fd08.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.78e530.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.b62fc9.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.fc0ed2.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-21-30-00.gh-issue-159860.qsbr128.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.848b18.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.a3cc5d.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.bf569b.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.d76e6d.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.da0f04.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-10-00-00.gh-issue-158195.UnpinTL.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-20-38-38.gh-issue-159101.vFwrZY.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-22-00-00.gh-issue-159963.pLkHld.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-14-00-00.gh-issue-159493.sfLf1b.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-18-00-00.gh-issue-160368.sem-clockwait.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-19-55-00.gh-issue-159739.codc01.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-19-55-00.gh-issue-159739.tali01.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.Aw61LR.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.CbTE4N.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.Zw43uX.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-23-18-38.gh-issue-160132.Nj4rQa.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-00-30-02.gh-issue-160179.dQ2e7m.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-00-30-21.gh-issue-160162.lShf32.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-12-30-11.gh-issue-106045.fkMW34.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-14-45-00.gh-issue-160245.mEmO24.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-37-00.gh-issue-160050.aB3x9Q.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-20-15-24.gh-issue-160491.yIQ_Vf.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-12-00-00.gh-issue-160682.Q7mK2x.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-28-13-47-43.gh-issue-160773.7VFxkA.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-02-05-50-35.gh-issue-160872.x8NvzN.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-05-52-49.gh-issue-161229.8pOPOI.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-13-44-26.gh-issue-159184.X_sOdY.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-20-28-15.gh-issue-161288.LikC_U.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-10-19-48.gh-issue-161419.wc4DV0.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-21-00-57.gh-issue-161366.ouxAvv.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-15-15-00.gh-issue-161433.A9b3Kq.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-21-21-04.gh-issue-161242.d4vNgC.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-07-09-30-00.gh-issue-161420.mIdFre.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-09-04-16-01-20.gh-issue-107689.LFLFtG.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2025-08-31-02-28-21.gh-issue-138277.kkORph.rst delete mode 100644 Misc/NEWS.d/next/IDLE/2026-10-16-15-12-36.gh-issue-159517.uzRtAd.rst delete mode 100644 Misc/NEWS.d/next/IDLE/2026-11-06-17-12-40.gh-issue-161477.smXIKC.rst delete mode 100644 Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst delete mode 100644 Misc/NEWS.d/next/Library/2021-06-11-20-00-16.bpo-19094.rMRoIL.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-26-09-36-24.gh-issue-61448.SOAkvt.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-12-27-15-41-55.gh-issue-128289.-hCzEc.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-02-04-09-01-06.gh-issue-129605.wKGNXQ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-03-13-20-21-00.gh-issue-115810.qwrb231.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-12-15-39-48.gh-issue-89730.Lu35Fu.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-18-14-13-00.gh-issue-139122.m3lp66.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-18-18-03-32.gh-issue-139105.1QQ_W4.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-13-15-43-09.gh-issue-140036.b_59uN.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-20-13-08-04.gh-issue-84027.HjBO0B.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-01-06-09-13-53.gh-issue-84644.V_cYP3.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-03-23-00-04-13.gh-issue-146065.FIdn8D.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-03-28-10-27-46.gh-issue-146169.RBF1xp.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-03-28-14-34-18.gh-issue-143376.FvWTwi.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-04-27-20-30-16.gh-issue-149083.xcmpSO.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-05-03-22-52-07.gh-issue-149309.gXhk9Y.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-05-09-12-58-04.gh-issue-149527.hX36Yh.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-05-21-16-15-01.gh-issue-105578.oq-0pg.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-06-18-11-26-00.gh-issue-151633.uJzHdc.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-06-27-08-31-30.gh-issue-152373.3cd0aa.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-06-28-14-37-01.gh-issue-152490.A8qLbF.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-06-30-08-39-07.gh-issue-62222.Zq7x2P.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-07-04-12-52-22.gh-issue-149816.XQOutB.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-07-11-16-00-00.gh-issue-153605.Rk7Xz3.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-07-13-00-10-00.gh-issue-153679.RJ2WGB.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-07-16-21-30-00.gh-issue-74876.wR3fXk.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-07-21-00-02-11.gh-issue-148228.D3rP8m.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-04-16-38-59.gh-issue-155152.qN1EEN.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-05-10-13-23.gh-issue-153711.9g9dVz.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-09-14-06-27.gh-issue-155426.zZVPMd.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-09-14-26-48.gh-issue-155428.wi0_fI.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-15-11-30-42.gh-issue-155782.JUmDP4.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-15-11-49-30.gh-issue-155830.cY5xgK.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-15-12-08-33.gh-issue-155833.Pi2Yjp.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-20-09-56-45.gh-issue-156078.-E7gZc.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-21-06-00-00.gh-issue-156057.tarfile-wrap-comp-err.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-23-16-33-00.gh-issue-151126.GUvtG7.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-29-13-41-56.gh-issue-154675.cYvE-Z.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-08-29-14-28-46.gh-issue-156568.mkxLY8.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-01-00-00-00.gh-issue-156784.8gvtt4.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-05-17-29-50.gh-issue-160837.W6kHva.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-17-10-30-00.gh-issue-158252.strptime.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-17-20-36-01.gh-issue-157290.NfFmt.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-18-00-00-00.gh-issue-49960.MDT3jE.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-20-12-00-00.gh-issue-157856.wq4xuF.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-20-21-24-05.gh-issue-157880.aE7wIu.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-21-14-41-07.gh-issue-157899.s6vBQC.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-21-21-07-12.gh-issue-157928.ZPlNYC.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-23-09-21-25.gh-issue-89977.fmia18.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-23-19-59-03.gh-issue-158004.p68swW.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-23-23-34-30.gh-issue-144812.ujptbVOI.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-24-14-59-08.gh-issue-158095.lcKOT4.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-25-01-29-55.gh-issue-158126.nWmFnq.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-25-17-04-45.gh-issue-158159.lbEalG.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-26-05-18-21.gh-issue-144650.zqHLOW.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-27-14-00-00.gh-issue-158277.aBcDeF.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-28-00-00-00.gh-issue-158370.ctxmgr.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-09-29-19-34-30.gh-issue-158192.6wPjiU.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-05-08-35-10.gh-issue-130698.iibO9V.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-05-14-58-12.gh-issue-158839.wYd6fI.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-07-10-59-27.gh-issue-155233.YGuQGK.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-07-17-30-00.gh-issue-158287.xY9zQp.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-08-11-58-52.gh-issue-158628.NJxwuw.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-08-21-02-38.gh-issue-158998.DZYJI7.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-10-22-14-22.gh-issue-150751.JInuGL.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-10-22-29-13.gh-issue-159049.eJUiKL.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-11-08-42-00.gh-issue-159184.Tq2nSe.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-11-13-08-07.gh-issue-88037.CzKcxh.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-11-13-40-44.gh-issue-159168.z3KjB8.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-12-00-00-00.gh-issue-159044.Tr8kXv.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-12-11-36-48.gh-issue-158856.jLR-Lv.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-12-12-00-00.gh-issue-72113.njkm8v.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-12-13-05-00.gh-issue-158501.n7Qp3k.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-12-18-21-06.gh-issue-159240.a1b2c3.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-12-21-20-00.gh-issue-159225.T8jlor.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-13-12-00-00.gh-issue-159304.mFbYq2.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-13-19-10-00.gh-issue-159322.hP6k3A.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-14-08-50-00.gh-issue-159398.k2Xd7q.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-14-10-26-10.gh-issue-159387.xMbQbB.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-14-11-41-00.gh-issue-159430.ZEW-mx.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-14-14-47-05.gh-issue-81954.KvM0gY.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-15-18-38-22.gh-issue-159500.OjAQ8W.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-16-10-25-22.gh-issue-159458.q02EnY.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-16-13-00-00.gh-issue-159435.OMNI.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-16-18-23-03.gh-issue-159558.ctRelk.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-17-15-22-16.gh-issue-159620.Ynsu5L.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-17-19-33-13.gh-issue-159643.EJfe0IXetV.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-17-19-41-12.gh-issue-157210.o9hycSeJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-18-08-30-10.gh-issue-155976.feSum4_j.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-18-12-20-00.gh-issue-159465.partial-escapes.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-18-16-09-24.gh-issue-159684.VHVYjVuE.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-00-00-00.gh-issue-149015.bdb.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-12-00-00.gh-issue-159714.j59fix.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-12-00-00.gh-issue-78327.Qk3hzA.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-16-00-00.gh-issue-139122.bYtEs1.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-16-26-57.gh-issue-71990.K3FlJ9.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-16-49-09.gh-issue-159706.ushGrx.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-19-33-35.gh-issue-159739.a1c3c7.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-19-33-35.gh-issue-159739.b5fbca.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.08caa7.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.243aa8.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.5a6e94.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.a81270.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.bc65da.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.ff088c.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-21-20-28.gh-issue-159786.fThehQ.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-23-25-29.gh-issue-159811.P6yy0z.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-19-23-25-29.gh-issue-159811.cMB4DO.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.0c4ae9.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.1e4599.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.35942a.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.3660a6.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.4c9783.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.5d4269.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.9ef89c.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.eb8987.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-00-12-05.gh-issue-159690.vZgLP6.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-12-30-00.gh-issue-159864.4FZ6MR.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-16-00-00.gh-issue-159868.qLhYkP.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-20-17-29-14.gh-issue-138641.FN6qM0.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-15-10-00.gh-issue-160125.pdbpkg.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-15-42-29.gh-issue-159875.p6j3yW.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-19-38-33.gh-issue-89721.uMpxRw.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-19-40-00.gh-issue-100609.THXgRC.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-19-55-00.gh-issue-159739.sque01.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-19-55-00.gh-issue-159739.tloc01.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-20-05-00.gh-issue-160104.BUAL5C.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.JIqZRY.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.b4ueJ7.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.ghGctt.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.k8uuGe.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.rVWF2a.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.yBc6Ei.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-00-29-50.gh-issue-160176.tkNst1.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-00-30-04.gh-issue-160187.9dcj2B.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-00-33-42.gh-issue-160171.11l4om.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.b4c371.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.ce26e1.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-19-56.gh-issue-161090.pxZ1IA.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-20-02.gh-issue-161092.OtC5l6.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-20-02.gh-issue-161102.gqDOIM7X.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-20-06.gh-issue-161100.pLgnlA.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-20-11.gh-issue-161110.miKrSQ.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-20-18.gh-issue-161108.xOVsRA.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-20-27.gh-issue-161098.voMVnd.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-20-44.gh-issue-161106.t0ux5q.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-20-52.gh-issue-161096.vkhfyy.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-21-00.gh-issue-161094.MDjCNm.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-23-40.gh-issue-161088._-HEWz.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-24-21.gh-issue-161112.BFrZRw.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-24-34.gh-issue-161086.FGPjqA.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-08-04-26.gh-issue-160297.hL4kQz.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-08-06-39.gh-issue-160513.q7RkZd.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-08-09-29.gh-issue-160406.q7Rk2d.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-08-17-51.gh-issue-160316.sRe9pQ.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-08-18-31.gh-issue-160522.cJkSt8.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-08-34-22.gh-issue-160310.q7Zm2K.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-16-18-42.gh-issue-159739.TioR3a.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.XmlH3a.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-16-30-00.gh-issue-159739.mFWntK.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-16-30-00.gh-issue-159739.tRq417.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160326.nAn7Tq.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160520.sSdTmo.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-22-22-51-40.gh-issue-160386.wYyboz.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-23-21-52-40.gh-issue-60299.Af6IzL.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-23-23-21-21.gh-issue-160549.q7ZdMt.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-24-01-53-42.gh-issue-160433.DSTlog.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-24-07-58-00.gh-issue-160561.b3Kq1z.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-24-22-32-23.gh-issue-160596.RKll-Z.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-26-16-15-29.gh-issue-160427.9G4EBb.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-26-16-17-45.gh-issue-160708.w7z33g.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-26-21-37-42.gh-issue-158500.zxyuq5.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-27-15-26-53.gh-issue-160781.4VSeQK.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-27-15-27-32.gh-issue-160781.kARuHu.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-27-15-42-16.gh-issue-160781.VGLB9S.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-28-07-26-47.gh-issue-160804.R5HeTQ.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-28-07-34-05.gh-issue-160806.ZZRkhi.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-28-08-12-33.gh-issue-160811.wQ3xT7.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-28-12-00-00.gh-issue-53852.Pr0xy4.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-28-13-23-28.gh-issue-59515.wt31yNN4.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-28-16-40-00.gh-issue-66333.pdb-sigint.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-28-20-36-44.gh-issue-109564.WqoyNE.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-29-19-42-53.gh-issue-53089.ewaIBvY6.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-29-20-52-57.gh-issue-50720.Kd1VMR.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-29-21-36-08.gh-issue-160929.mK8n2Q.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-29-21-43-53.gh-issue-160429.eWapMs.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-29-23-25-31.gh-issue-160782.1ZwCOr.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-30-10-36-30.gh-issue-160782.n6JCUU.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-30-12-00-00.gh-issue-137549.Tm0Bxg.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-10-31-21-34-00.gh-issue-161039.k7_pTD.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-01-19-38-33.gh-issue-161083.PM21lv.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-01-22-07-24.gh-issue-161124.c2YmcV.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-02-13-11-30.gh-issue-161166.zfFDlW.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-02-13-26-36.gh-issue-161179.H8Yyaw.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-02-16-28-11.gh-issue-161178.D3-23R.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-02-21-10-00.gh-issue-161212.pip263.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-03-10-33-33.gh-issue-161233._9VrfL.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-03-12-00-00.gh-issue-161076.nOdLy1.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-03-12-00-00.gh-issue-93821.aCcRsT.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-03-18-56-28.gh-issue-161259.iyZz6y.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-03-21-26-09.gh-issue-161295.IJqoJM.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-05-10-05-00.gh-issue-161387.def456.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-05-11-56-50.gh-issue-161389.EIl2E6.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-05-11-59-58.gh-issue-161414.Xnxe8A.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-05-12-00-24.gh-issue-161415.NJhrAR.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-06-00-00-00.gh-issue-142782.qU7nE4.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-06-05-42-11.gh-issue-71199.qJN4cV.rst delete mode 100644 Misc/NEWS.d/next/Library/2026-11-06-11-59-26.gh-issue-161475.82LWf4.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCFSN.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-06-12-30-00.gh-issue-158928.Zd9Ybv.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-15-03-47-08.gh-issue-159428.XHRKwY.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-20-13-15-00.gh-issue-159894.8Kp4mN.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-21-16-55-07.gh-issue-160040.9PjFjC.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-21-19-58-53.gh-issue-160097.-opfAL.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-22-08-04-33.gh-issue-160300.q7Zk2B.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-22-08-05-32.gh-issue-160420.tZ9dRv.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-22-08-07-52.gh-issue-161323.q7Drv2.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-22-08-08-23.gh-issue-160506.rN4dLq.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-22-08-08-57.gh-issue-160499.pf9H9g.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-22-11-09-30.gh-issue-160279.9BtxeF.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-22-21-59-39.gh-issue-160369.imapTLS.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-10-27-16-25-40.gh-issue-160898.epi53X.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-11-02-18-30-09.gh-issue-161192.2ScSRO.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-11-06-16-22-47.gh-issue-161503.FkIl87.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-11-07-01-43-49.gh-issue-161569.cookiej.rst delete mode 100644 Misc/NEWS.d/next/Security/2026-11-07-15-56-47.gh-issue-161705.OjDK2d.rst delete mode 100644 Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst delete mode 100644 Misc/NEWS.d/next/Tests/2026-10-07-12-00-58.gh-issue-158975.yR8q2L.rst delete mode 100644 Misc/NEWS.d/next/Tests/2026-10-13-13-34-00.gh-issue-159293.Lr8q2N.rst delete mode 100644 Misc/NEWS.d/next/Tests/2026-10-19-17-05-00.gh-issue-107862.qH3sTa.rst delete mode 100644 Misc/NEWS.d/next/Tests/2026-10-28-20-40-00.gh-issue-158646.k2Xw7q.rst delete mode 100644 Misc/NEWS.d/next/Tests/2026-11-03-13-12-50.gh-issue-161263.Voe8VM.rst delete mode 100644 Misc/NEWS.d/next/Tests/2026-11-05-16-20-23.gh-issue-161430.M7b8N6.rst delete mode 100644 Misc/NEWS.d/next/Tests/2026-11-06-23-15-23.gh-issue-161517.LkAlDt.rst delete mode 100644 Misc/NEWS.d/next/Tests/2026-11-07-01-45-00.gh-issue-160334.F2n8qA.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-03-25-18-00-00.gh-issue-131725.qwfh321.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2026-10-14-09-48-26.gh-issue-159385.0aWOA9.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2026-10-20-12-50-00.gh-issue-114258.slotNx.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2026-10-21-11-53-40.gh-issue-87613.tmKBlg.rst delete mode 100644 Misc/NEWS.d/next/WASI/2026-10-21-15-26-12.gh-issue-160135.UP-DlO.rst delete mode 100644 Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst delete mode 100644 Misc/NEWS.d/next/Windows/2026-09-14-16-49-44.gh-issue-157418.r2IggA.rst delete mode 100644 Misc/NEWS.d/next/Windows/2026-10-15-18-45-00.gh-issue-159118.wmiMTA.rst delete mode 100644 Misc/NEWS.d/next/Windows/2026-10-20-21-37-34.gh-issue-159880.eKBiLr.rst delete mode 100644 Misc/NEWS.d/next/Windows/2026-10-22-14-04-17.gh-issue-160239.A1b2C3.rst delete mode 100644 Misc/NEWS.d/next/macOS/2026-10-19-19-33-35.gh-issue-159739.aaf575.rst diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 48d9b83f437ff73..efd0bc79cbd780c 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -50,7 +50,7 @@ bound into a function. free variables of a code object. The names are in the same order as the values returned by :c:func:`PyUnstable_InterpreterFrame_GetLocal`. - .. versionadded:: next + .. versionadded:: 3.16 .. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index 0b176cdb118234e..84cb777692ce3e5 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -277,4 +277,4 @@ macros above: Values used internally by the ``Py_IS*`` classification macros above. Exposed for backwards compatibility only. - .. soft-deprecated:: next + .. soft-deprecated:: 3.16 diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index f375af2925b5fd8..6db8dfca8a80db7 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -527,7 +527,7 @@ Dictionary objects Works with O(1) complexity. - .. versionadded:: next + .. versionadded:: 3.16 .. c:function:: int PyDict_AddWatcher(PyDict_WatchCallback callback) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index ba4e2a418b2a49f..dd67222bebc3db0 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -1350,19 +1350,19 @@ to :c:data:`PyExc_OSError`. .. c:var:: PyObject *PyExc_EnvironmentError - .. soft-deprecated:: next + .. soft-deprecated:: 3.16 Use :c:data:`PyExc_OSError` instead. .. c:var:: PyObject *PyExc_IOError - .. soft-deprecated:: next + .. soft-deprecated:: 3.16 Use :c:data:`PyExc_OSError` instead. .. c:var:: PyObject *PyExc_WindowsError .. availability:: Windows - .. soft-deprecated:: next + .. soft-deprecated:: 3.16 Use :c:data:`PyExc_OSError` instead. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index a6bfa8358e3aacc..3f5b3bdbb26f4e7 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -188,4 +188,4 @@ by mistake. They are documented solely for completeness; use other The string literal ``"b"``, used to open files in binary mode. Exposed for backwards compatibility only. - .. soft-deprecated:: next + .. soft-deprecated:: 3.16 diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index ea6846306273552..a086c60905402ea 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -258,4 +258,4 @@ Unless using :pep:`523`, you will not need this. * If the slot is unset or hidden, store ``NULL`` in *result* and return ``0``. * On error, raise an exception, store ``NULL`` in *result* and return ``-1``. - .. versionadded:: next + .. versionadded:: 3.16 diff --git a/Doc/c-api/profiling.rst b/Doc/c-api/profiling.rst index 8393858c827e2dd..c87c03e96f83ae8 100644 --- a/Doc/c-api/profiling.rst +++ b/Doc/c-api/profiling.rst @@ -251,4 +251,4 @@ Performance statistics This constant is intended for internal use only and is not useful to extension modules. - .. soft-deprecated:: next + .. soft-deprecated:: 3.16 diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 2f2c8e06f668b97..19b9265eccf7460 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -792,7 +792,7 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: A member flag that has no effect, retained solely for backwards compatibility. This is only available with ``#include "structmember.h"``. - .. soft-deprecated:: next + .. soft-deprecated:: 3.16 Omit this flag instead. diff --git a/Doc/c-api/synchronization.rst b/Doc/c-api/synchronization.rst index a74a92ca7cfa24e..26712549f285f1a 100644 --- a/Doc/c-api/synchronization.rst +++ b/Doc/c-api/synchronization.rst @@ -269,7 +269,7 @@ These APIs are obsolete since Python 3.13 with the introduction of A :term:`soft deprecated` alias to :c:expr:`long long`, the integer type used for the timeout argument of :c:func:`PyThread_acquire_lock_timed`. - .. soft-deprecated:: next + .. soft-deprecated:: 3.16 .. c:var:: const long long PY_TIMEOUT_MAX diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 10929b233d68400..2105571eee5ff07 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -161,7 +161,7 @@ Tuple Objects :c:func:`PyTuple_SetItem`; except if *newsize* is zero in which case it returns the immutable empty tuple. - .. versionchanged:: next + .. versionchanged:: 3.16 The function now fails if the tuple hash value was already computed. diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 83ca45ec251e2be..40c9f1151112c87 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -114,7 +114,7 @@ as much as it can. This function is intended for internal use only. Do not access or modify the weak reference list through the returned pointer. - .. soft-deprecated:: next + .. soft-deprecated:: 3.16 .. c:function:: void PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *object) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index ac4f12de6fb663a..ffcf2ff37be53bf 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -2088,7 +2088,7 @@ Argument groups to create instances. Only the methods documented here are part of the public API; other attributes and methods are implementation details. - .. versionadded:: next + .. versionadded:: 3.16 The private ``_ArgumentGroup`` name remains as an alias. @@ -2165,7 +2165,7 @@ Mutual exclusion Only the ``add_argument`` method documented here is part of the public API; other attributes and methods are implementation details. - .. versionadded:: next + .. versionadded:: 3.16 The private ``_MutuallyExclusiveGroup`` name remains as an alias. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 60d2b878aa455af..a725df99fdda2ab 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -291,7 +291,7 @@ The :mod:`!binascii` module defines the following functions: If the optional argument *strip_ws* is true, trailing whitespace is stripped from each line, as required by :rfc:`2045`. - .. versionchanged:: next + .. versionchanged:: 3.16 Added the *strip_ws* parameter. diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index 058e59b32fd15a6..db6b8c444e02878 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -250,7 +250,7 @@ Interpreter objects with interpreters.create() as interp: interp.exec('print("hello")') - .. versionchanged:: next + .. versionchanged:: 3.16 Added support for using :class:`!Interpreter` objects in a :term:`context manager`. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 6f3524e0253bcac..b62bd94893c285f 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1117,7 +1117,7 @@ an object passed for a pointer argument (see ``c_int`` instance is passed by reference, and one whose :attr:`!_as_parameter_` is a ``c_int`` array is accepted as well. -.. versionchanged:: next +.. versionchanged:: 3.16 Objects with an :attr:`!_as_parameter_` attribute now receive the same implicit conversions for pointer arguments as the object they wrap. @@ -1358,7 +1358,7 @@ the simple types: same requirement C itself imposes. Returning a pointer to memory owned by a temporary Python object will leave the C caller with a dangling pointer. -.. versionchanged:: next +.. versionchanged:: 3.16 Callback functions can now return structures and unions. .. _ctypes-accessing-values-exported-from-dlls: diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 552626cbb8ac88e..20d6f0cb9d4abff 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2480,7 +2480,7 @@ where historical changes have been made to civil time. The *name* passed to the constructor, or ``None`` if no name was passed. - .. versionadded:: next + .. versionadded:: 3.16 Class attributes: @@ -2724,7 +2724,7 @@ differences between platforms in handling of unsupported format specifiers. ``%D``, ``%F``, ``%n``, ``%t``, and ``%:z`` were added for :meth:`~.datetime.strptime`. -.. versionadded:: next +.. versionadded:: 3.16 ``%Nf`` was added. diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 24132ee39f130c4..b175591ddea2b6f 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -195,7 +195,7 @@ backup file, :exc:`FileExistsError` is raised. If the output file cannot be created, the backup file is kept. In-place filtering of files in a directory that other, untrusted users can modify is not supported. -.. versionchanged:: next +.. versionchanged:: 3.16 The output file is now created exclusively, and the backup file is no longer deleted if the output file cannot be created. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 1bbe30158749c48..2ce07976c395311 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -339,7 +339,7 @@ The :mod:`!functools` module defines the following functions: .. versionadded:: 3.14 - .. versionchanged:: next + .. versionchanged:: 3.16 ``Placeholder`` is now a :class:`sentinel`. .. function:: partial(func, /, *args, **keywords) diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 3a050873b79b9d7..cd55deccbe34c7a 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -307,7 +307,7 @@ Once executed the :mod:`!gzip` module keeps the input file(s). Add a new command line interface with a usage. By default, when you will execute the CLI, the default compression level is 6. -.. versionchanged:: next +.. versionchanged:: 3.16 An existing output file is removed and created anew instead of being opened for writing, so a symbolic link at that name is replaced rather diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 76403d80955b476..7f4b698eac38cd8 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2838,7 +2838,7 @@ features: substitution path (which typically includes ``\\?\`` prefix) rather than the optional "print name" field that was previously returned. - .. versionchanged:: next + .. versionchanged:: 3.16 Accepts file descriptors pointing to symbolic links as *path* on Linux, Android, macOS and iOS. diff --git a/Doc/library/quopri.rst b/Doc/library/quopri.rst index 60b4d1c182d6414..ee21ad65543a3a7 100644 --- a/Doc/library/quopri.rst +++ b/Doc/library/quopri.rst @@ -31,7 +31,7 @@ sending a graphics file. If the optional argument *strip_ws* is true, trailing whitespace is stripped from each line, as required by :rfc:`2045`. - .. versionchanged:: next + .. versionchanged:: 3.16 Added the *strip_ws* parameter. @@ -53,7 +53,7 @@ sending a graphics file. Like :func:`decode`, except that it accepts a source :class:`bytes` and returns the corresponding decoded :class:`bytes`. - .. versionchanged:: next + .. versionchanged:: 3.16 Added the *strip_ws* parameter. diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst index 75d0df481c79b41..563ad2b3195e954 100644 --- a/Doc/library/reprlib.rst +++ b/Doc/library/reprlib.rst @@ -40,7 +40,7 @@ This module provides a class, an instance, and a function: .. versionchanged:: 3.12 Allow attributes to be set via keyword arguments. - .. versionchanged:: next + .. versionchanged:: 3.16 :class:`!Repr` instances now have a string representation that includes their attribute values. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 2d3edd8e5aebfca..6104ad3d3a801b1 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -786,7 +786,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. .. versionchanged:: 3.12 Added the *filter* argument. - .. versionchanged:: next + .. versionchanged:: 3.16 Added the "whl" format. .. function:: register_unpack_format(name, extensions, function[, extra_args[, description]]) @@ -837,7 +837,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. You can register new formats or provide your own unpacker for any existing formats, by using :func:`register_unpack_format`. - .. versionchanged:: next + .. versionchanged:: 3.16 Added the *whl* format. diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 5a1cd95da527ea8..5aaad9a1bcc5ba1 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1065,7 +1065,7 @@ The :mod:`!socket` module also offers various network-related services: for IPv6 multicast addresses, string representing an address will not contain ``%scope_id`` part. - .. versionchanged:: next + .. versionchanged:: 3.16 Null characters are now rejected in the *host* and *port* parameters. .. _Happy Eyeballs: https://en.wikipedia.org/wiki/Happy_Eyeballs diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 020a8ab7301b6b7..d1c099bd77e47ea 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -341,7 +341,7 @@ Exceptions The server's ECH retry configuration as an ECHConfigList, or ``b""`` if the server did not provide one. - .. versionadded:: next + .. versionadded:: 3.16 .. exception:: CertificateError @@ -446,7 +446,7 @@ Certificate handling .. versionchanged:: 3.10 The *timeout* parameter was added. - .. versionchanged:: next + .. versionchanged:: 3.16 The function now raises :exc:`SSLCertVerificationError` if *ca_certs* is specified but the certificate hostname does not match the server hostname. @@ -906,7 +906,7 @@ Constants Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. data:: OP_ECH_TRIALDECRYPT @@ -915,7 +915,7 @@ Constants Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. data:: OP_ECH_IGNORE_CID @@ -923,7 +923,7 @@ Constants Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. data:: OP_ECH_GREASE_RETRY_CONFIG @@ -931,7 +931,7 @@ Constants Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. data:: HAS_ALPN @@ -1033,7 +1033,7 @@ Constants Whether the OpenSSL library has built-in support for Encrypted Client Hello (ECH). This requires OpenSSL 4.0 or later with ECH enabled. - .. versionadded:: next + .. versionadded:: 3.16 .. data:: CHANNEL_BINDING_TYPES @@ -1110,33 +1110,33 @@ Constants Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: ECHStatus.ECH_STATUS_SUCCESS ECH was accepted without a certificate verification error. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: ECHStatus.ECH_STATUS_FAILED An internal or protocol error occurred, or ECH was rejected without a retry configuration. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: ECHStatus.ECH_STATUS_FAILED_ECH ECH was rejected and the server supplied an authenticated retry configuration. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: ECHStatus.ECH_STATUS_BAD_NAME ECH was accepted, but verification of the peer's certificate failed. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: ECHStatus.ECH_STATUS_FAILED_ECH_BAD_NAME @@ -1144,45 +1144,45 @@ Constants verification for the outer server name failed. The retry configuration must not be used. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: ECHStatus.ECH_STATUS_NOT_TRIED ECH is configured but has not been attempted (such as before the handshake starts). - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: ECHStatus.ECH_STATUS_NOT_CONFIGURED No ECH configuration is set for this connection. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: ECHStatus.ECH_STATUS_GREASE ECH GREASE is enabled or was used, but no retry configuration was received. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: ECHStatus.ECH_STATUS_GREASE_ECH The client sent a GREASE ECH extension and received a retry configuration. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: ECHStatus.ECH_STATUS_BACKEND The server received a ClientHello marked as an inner ClientHello, as used by a backend server in an ECH split deployment. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: ECHStatus.ECH_STATUS_BAD_CALL Invalid arguments were supplied to OpenSSL's ECH status API. - .. versionadded:: next + .. versionadded:: 3.16 .. data:: Purpose.SERVER_AUTH @@ -1538,7 +1538,7 @@ SSL sockets also have the following additional methods and attributes: Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. method:: SSLSocket.get_ech_status() @@ -1550,7 +1550,7 @@ SSL sockets also have the following additional methods and attributes: Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. method:: SSLSocket.set_ech_config(ech_config) @@ -1565,7 +1565,7 @@ SSL sockets also have the following additional methods and attributes: Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. method:: SSLSocket.selected_alpn_protocol() @@ -1647,7 +1647,7 @@ SSL sockets also have the following additional methods and attributes: Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: SSLSocket.server_side @@ -1849,7 +1849,7 @@ to speed up repeated connections from the same clients. Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. method:: SSLContext.load_verify_locations(cafile=None, capath=None, cadata=None) @@ -2078,7 +2078,7 @@ to speed up repeated connections from the same clients. Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. method:: SSLContext.set_npn_protocols(npn_protocols) @@ -2116,7 +2116,7 @@ to speed up repeated connections from the same clients. Only available when :data:`HAS_ECH` is ``True``. - .. versionadded:: next + .. versionadded:: 3.16 .. attribute:: SSLContext.sni_callback @@ -2284,7 +2284,7 @@ to speed up repeated connections from the same clients. The method returns an instance of :attr:`SSLContext.sslsocket_class` instead of hard-coded :class:`SSLSocket`. - .. versionchanged:: next + .. versionchanged:: 3.16 *ech_config* argument was added. .. attribute:: SSLContext.sslsocket_class @@ -2326,7 +2326,7 @@ to speed up repeated connections from the same clients. *server_hostname* was accepted, and verified the certificate chain but never the peer's identity. - .. versionchanged:: next + .. versionchanged:: 3.16 *ech_config* argument was added. .. attribute:: SSLContext.sslobject_class @@ -3302,7 +3302,7 @@ of TLS/SSL. Some new TLS 1.3 features are not yet available. Encrypted Client Hello ^^^^^^^^^^^^^^^^^^^^^^ -.. versionadded:: next +.. versionadded:: 3.16 Encrypted Client Hello (ECH) encrypts values in the ClientHello record sent when establishing a TLS connection. It requires an ECHConfigList containing algorithm diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index f08a06d3edae9ba..c3485204fb8b4b9 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -438,7 +438,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. availability:: WASI. - .. versionadded:: next + .. versionadded:: 3.16 .. data:: pycache_prefix @@ -1666,7 +1666,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only :envvar:`PYTHONSAFEPATH` environment variable is used, :data:`sys._path0` is ``None`` and :data:`sys.path` is left unchanged. - .. versionadded:: next + .. versionadded:: 3.16 .. data:: path_hooks diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 092332812296ccf..3131107be97189e 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -1149,11 +1149,11 @@ reused in custom filters: links (symbolic and hard) to arbitrary files on the filesystem, potentially granting unintended access to these files. - .. versionchanged:: next + .. versionchanged:: 3.16 Filenames containing ``..`` components are now normalized. - .. versionchanged:: next + .. versionchanged:: 3.16 Filenames with a drive that are not absolute are now refused. @@ -1204,7 +1204,7 @@ reused in custom filters: Link targets are now normalized. - .. versionchanged:: next + .. versionchanged:: 3.16 Link targets with a drive that are not absolute are now refused. diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index 2d266c2d433c604..6e21b6f99f877d4 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -69,7 +69,7 @@ which relays any information about the UUID's safety, using this enumeration: variant and version number set according to :rfc:`9562`, overriding bits in the given *hex*, *bytes*, *bytes_le*, *fields*, or *int*. - .. versionchanged:: next + .. versionchanged:: 3.16 Passing an object other than :class:`bytes` as *bytes* or *bytes_le* now consistently raises :exc:`TypeError`. Previously, some invalid inputs raised :exc:`AssertionError` or :exc:`ValueError`. diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 56c06ae609a2154..e51a9bdcb870d56 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -149,7 +149,7 @@ See :ref:`__slots__ documentation ` for details. .. versionchanged:: 3.16 Raise :exc:`!TypeError` if *callback* is not callable or ``None``. - .. versionchanged:: next + .. versionchanged:: 3.16 Weak references can now be pickled. @@ -224,7 +224,7 @@ See :ref:`__slots__ documentation ` for details. .. versionchanged:: 3.9 Added support for ``|`` and ``|=`` operators, as specified in :pep:`584`. - .. versionchanged:: next + .. versionchanged:: 3.16 :class:`!WeakKeyDictionary` can now be pickled. :class:`WeakKeyDictionary` objects have an additional method that @@ -252,7 +252,7 @@ than needed. .. versionchanged:: 3.9 Added support for ``|`` and ``|=`` operators, as specified in :pep:`584`. - .. versionchanged:: next + .. versionchanged:: 3.16 :class:`!WeakValueDictionary` can now be pickled. :class:`WeakValueDictionary` objects have the following additional methods. @@ -279,7 +279,7 @@ These have the same issues as the :meth:`WeakKeyDictionary.keyrefs` method. A copy or a deep copy shares the elements with the original; user-defined attributes are copied or deep-copied. - .. versionchanged:: next + .. versionchanged:: 3.16 :class:`!WeakSet` can now be pickled and copied. @@ -318,7 +318,7 @@ These have the same issues as the :meth:`WeakKeyDictionary.keyrefs` method. .. versionadded:: 3.4 - .. versionchanged:: next + .. versionchanged:: 3.16 :class:`!WeakMethod` can now be pickled and copied. .. class:: finalize(obj, func, /, *args, **kwargs) diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index 208eb4ad414f706..05166214c7fc31f 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -759,7 +759,7 @@ inherits properties from :class:`Node`. If several elements have the same ID, the first one in document order is returned. - .. versionchanged:: next + .. versionchanged:: 3.16 Previously, which of the elements with the same ID was returned was unspecified and could change between calls. diff --git a/Include/patchlevel.h b/Include/patchlevel.h index c6c7dde1797e946..bde6ca17c3ff0c5 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -24,10 +24,10 @@ #define PY_MINOR_VERSION 16 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.16.0a1+dev" +#define PY_VERSION "3.16.0a2" /*--end constants--*/ diff --git a/Lib/pydoc_data/module_docs.py b/Lib/pydoc_data/module_docs.py index f5ead1624929240..5a9a92410d30c2d 100644 --- a/Lib/pydoc_data/module_docs.py +++ b/Lib/pydoc_data/module_docs.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue Oct 13 08:05:49 2026 +# Autogenerated by Sphinx on Tue Nov 10 08:57:12 2026 # as part of the release process. module_docs = { @@ -48,6 +48,7 @@ 'crypt': 'crypt#module-crypt', 'csv': 'csv#module-csv', 'ctypes': 'ctypes#module-ctypes', + 'ctypes.wintypes': 'ctypes#module-ctypes.wintypes', 'curses': 'curses#module-curses', 'curses.ascii': 'curses.ascii#module-curses.ascii', 'curses.panel': 'curses.panel#module-curses.panel', diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 8a237ef4d5e927d..7119825ac14e5b9 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Sat Nov 7 11:32:29 2026 +# Autogenerated by Sphinx on Tue Nov 10 08:57:12 2026 # as part of the release process. topics = { @@ -2404,7 +2404,7 @@ class namespace that define a "__set_name__()" method; The last two expressions apply to totally ordered collections (e.g. to sequences, but not to sets or mappings). See also the - "@~functools.total_ordering" decorator. + "@total_ordering" decorator. * The "hash()" result should be consistent with equality. Objects that are equal should either have the same hash value, or be marked as @@ -5019,10 +5019,11 @@ class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=Fa originate in a module that matches one of these patterns. [1] By default, Pdb sets a handler for the SIGINT signal (which is sent - when the user presses "Ctrl"-"C" on the console) when you give a - "continue" command. This allows you to break into the debugger - again by pressing "Ctrl"-"C". If you want Pdb not to touch the - SIGINT handler, set *nosigint* to true. + when the user presses "Ctrl"-"C" on the console) when you resume + execution with "continue", "next", "until", or "return". This + allows you to break into the debugger again by pressing "Ctrl"-"C". + If you want Pdb not to touch the SIGINT handler, set *nosigint* to + true. The *readrc* argument defaults to true and controls whether Pdb will load .pdbrc files from the filesystem. @@ -5065,6 +5066,9 @@ class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=Fa "pdb.set_trace()" will always stop the program at calling frame, ignoring the *skip* pattern (if any). + Changed in version 3.16: SIGINT handling also applies to "next", + "until", and "return". + run(statement, globals=None, locals=None) runeval(expression, globals=None, locals=None) runcall(function, *args, **kwds) diff --git a/Misc/NEWS.d/3.16.0a2.rst b/Misc/NEWS.d/3.16.0a2.rst new file mode 100644 index 000000000000000..629953ebd574298 --- /dev/null +++ b/Misc/NEWS.d/3.16.0a2.rst @@ -0,0 +1,4357 @@ +.. cve: CVE-2026-115791 +.. date: 2026-11-07-15-56-47 +.. gh-issue: 161705 +.. nonce: OjDK2d +.. release date: 2026-11-10 +.. section: Security + +Remove ``sys.path[0]`` in :mod:`gzip`, :mod:`tarfile` and :mod:`zipfile` +command line interface to prevent shadowing standard library modules from +extracted files. Issue reported by Jelle Zijlstra, patch by Gregory P. Smith +and Victor Stinner. + +.. + +.. cve: CVE-2026-124249 +.. date: 2026-11-07-01-43-49 +.. gh-issue: 161569 +.. nonce: cookiej +.. section: Security + +Fix :class:`http.cookiejar.CookieJar` deleting a stored cookie when a +response from an unrelated host sets an already expired cookie with a +``Domain`` or ``Path`` attribute pointing at that cookie. The cookie policy +is now consulted before an expired cookie is allowed to delete an existing +one. + +.. + +.. cve: CVE-2026-19552 +.. date: 2026-11-06-16-22-47 +.. gh-issue: 161503 +.. nonce: FkIl87 +.. section: Security + +:func:`ssl.get_server_certificate` now verifies that the server hostname +matches its certificate when the caller specifies *ca_certs*. + +.. + +.. date: 2026-11-02-18-30-09 +.. gh-issue: 161192 +.. nonce: 2ScSRO +.. section: Security + +Update bundled `libexpat `_ to version 2.10.0 +for the fixes to :cve:`2026-113351`, :cve:`2026-115506`, :cve:`2026-117610` +and :cve:`2026-118370`. + +.. + +.. cve: CVE-2026-116566 +.. date: 2026-10-27-16-25-40 +.. gh-issue: 160898 +.. nonce: epi53X +.. section: Security + +:func:`socket.getaddrinfo` now rejects null bytes within *host* and *port* +parameters as this previously caused the values to be silently truncated. + +.. + +.. cve: CVE-2026-115559 +.. date: 2026-10-22-21-59-39 +.. gh-issue: 160369 +.. nonce: imapTLS +.. section: Security + +:meth:`imaplib.IMAP4.starttls` now rejects unexpected plaintext buffered +after the STARTTLS response, preventing it from being processed as +TLS-protected data. + +.. + +.. date: 2026-10-22-11-09-30 +.. gh-issue: 160279 +.. nonce: 9BtxeF +.. section: Security + +Prevent :mod:`venv` from importing modules from the root of an existing +virtual environment when installing or upgrading pip. + +.. + +.. date: 2026-10-22-08-08-57 +.. gh-issue: 160499 +.. nonce: pf9H9g +.. section: Security + +Fix unbounded memory growth in :mod:`asyncio` TLS connections when a peer +stops reading but keeps sending records that make OpenSSL generate its own +replies (such as TLS 1.2 renegotiation alerts). Reading from the peer is +now paused when this output accumulates, instead of only when application +data does. + +.. + +.. date: 2026-10-22-08-08-23 +.. gh-issue: 160506 +.. nonce: rN4dLq +.. section: Security + +Fix :func:`os.path.realpath` on Windows returning the wrong path for a +broken relative symbolic link reached through a symbolic link to a +directory. This allowed :mod:`tarfile` extraction with the ``'data'`` or +``'tar'`` filter to create a file outside the destination directory. + +.. + +.. date: 2026-10-22-08-07-52 +.. gh-issue: 161323 +.. nonce: q7Drv2 +.. section: Security + +On Windows, the ``'tar'`` and ``'data'`` :mod:`tarfile` extraction filters +now refuse member names and link targets that have a drive but are not +absolute, such as ``C:foo``, with :exc:`~tarfile.AbsolutePathError` or +:exc:`~tarfile.AbsoluteLinkError`. Such members could be extracted outside +the destination directory. Fixes :cve:`2026-107784`. + +.. + +.. date: 2026-10-22-08-05-32 +.. gh-issue: 160420 +.. nonce: tZ9dRv +.. section: Security + +Fix :class:`zoneinfo.ZoneInfo` on Windows accepting keys in which a +component after a ``/`` starts with a drive letter, such as +``America/C:\file``. Such a key could open a file outside the ``tzdata`` +package; it now raises :exc:`ValueError`. + +.. + +.. date: 2026-10-22-08-04-33 +.. gh-issue: 160300 +.. nonce: q7Zk2B +.. section: Security + +Fix :class:`http.cookiejar.DefaultCookiePolicy` accepting and returning +cookies with a negative ``Version`` attribute without checking their domain +against the request host. Such cookies are now ignored. + +.. + +.. cve: CVE-2026-114583 +.. date: 2026-10-21-19-58-53 +.. gh-issue: 160097 +.. nonce: -opfAL +.. section: Security + +:func:`tarfile.tar_filter` now rejects link members whose path resolves to +the destination directory itself, as :func:`tarfile.data_filter` already +did. + +.. + +.. cve: CVE-2026-115557 +.. date: 2026-10-21-16-55-07 +.. gh-issue: 160040 +.. nonce: 9PjFjC +.. section: Security + +:mod:`ssl` now rejects TLS certificate chains that contain any of a set of +compromised eMudhra intermediate CA keys. This is an extraordinary measure +and a last line of defense: removing trust is primarily the job of the +platform root store, and the built-in denylist only blocks connections while +that update has not yet reached the OS, a container image, or a bundled +trust store in various Python packages. Entries are added to the denylist +only under a very high threshold. The denylist matches on the public key +(the SHA-256 digest of the :rfc:`5280` ``SubjectPublicKeyInfo``), so it also +covers the cross-signed certificates that reuse one of the distrusted keys. + +.. + +.. date: 2026-10-20-13-15-00 +.. gh-issue: 159894 +.. nonce: 8Kp4mN +.. section: Security + +Bump the version of :pypi:`pip` bundled in :mod:`ensurepip` to version +26.2.2. + +.. + +.. date: 2026-10-15-03-47-08 +.. gh-issue: 159428 +.. nonce: XHRKwY +.. section: Security + +:mod:`mimetypes` no longer reads the files listed in +:data:`mimetypes.knownfiles` by default on Windows, and uses only the +registry. + +.. + +.. date: 2026-10-06-12-30-00 +.. gh-issue: 158928 +.. nonce: Zd9Ybv +.. section: Security + +Fix a buffer overflow when initializing a :class:`bytearray` from an +iterator. + +.. + +.. date: 2026-08-31-16-47-33 +.. gh-issue: 157953 +.. nonce: KxCFSN +.. section: Security + +Update bundled `libexpat `_ to version 2.9.0. + +.. + +.. date: 2026-11-09-14-30-00 +.. gh-issue: 160267 +.. nonce: sTfrEe +.. section: Core and Builtins + +Fix a crash when :mod:`profiling.sampling` profiles a process that creates +and destroys threads or subinterpreters quickly. Memory of deleted thread +and interpreter states is now reused for new states instead of being freed +until the runtime is finalized. + +.. + +.. date: 2026-11-07-09-30-00 +.. gh-issue: 161420 +.. nonce: mIdFre +.. section: Core and Builtins + +Restore ``mimalloc``'s previous behavior of processing delayed frees from +other threads on every generic malloc call, reverting the change introduced +in v2.2.2. + +.. + +.. date: 2026-11-06-21-21-04 +.. gh-issue: 161242 +.. nonce: d4vNgC +.. section: Core and Builtins + +Speed up :exc:`AttributeError`, :exc:`ImportError` and :exc:`NameError` by +enabling :ref:`vectorcall`. Unexpected keyword arguments are now reported by +name. + +.. + +.. date: 2026-11-06-15-15-00 +.. gh-issue: 161433 +.. nonce: A9b3Kq +.. section: Core and Builtins + +Add context notes to :exc:`TypeError` raised while hashing a :class:`tuple`, +identifying the index of the item whose hash failed. + +.. + +.. date: 2026-11-06-02-30-00 +.. gh-issue: 129724 +.. nonce: Mn8sLq +.. section: Core and Builtins + +Fix the :mod:`marshal` output of compiled code, and so ``.pyc`` files, +differing for the same source depending on what ran before in the process. +This affected private names in classes with non-ASCII names, and +:func:`compile` of an AST with names not created by :func:`ast.parse`. + +.. + +.. date: 2026-11-05-21-00-57 +.. gh-issue: 161366 +.. nonce: ouxAvv +.. section: Core and Builtins + +Fix a deadlock in the garbage collector of the :term:`free-threaded build` +when :data:`gc.DEBUG_COLLECTABLE` is set and another thread is writing to +:data:`sys.stderr`. The debug output is now written after the other threads +are resumed. + +.. + +.. date: 2026-11-05-10-19-48 +.. gh-issue: 161419 +.. nonce: wc4DV0 +.. section: Core and Builtins + +Fix a crash in the :term:`free-threaded build` where a weak reference could +revive an object whose finalizer had already run during deallocation, +causing it to be deallocated twice. + +.. + +.. date: 2026-11-03-20-28-15 +.. gh-issue: 161288 +.. nonce: LikC_U +.. section: Core and Builtins + +Fix a crash in the perf trampoline (:option:`-X perf <-X>`) on builds with +clang's ``-fsanitize=function``, which ``-fsanitize=undefined`` enables +since clang 17. + +.. + +.. date: 2026-11-03-13-44-26 +.. gh-issue: 159184 +.. nonce: X_sOdY +.. section: Core and Builtins + +Fix a leak of NaN constants in code objects created by :func:`marshal.loads` +on the free-threaded build. + +.. + +.. date: 2026-11-03-05-52-49 +.. gh-issue: 161229 +.. nonce: 8pOPOI +.. section: Core and Builtins + +:func:`!__lazy_import__` now treats ``None`` for *globals* and *locals* the +same as omitting them, matching its signature. + +.. + +.. date: 2026-11-02-05-50-35 +.. gh-issue: 160872 +.. nonce: x8NvzN +.. section: Core and Builtins + +Optimize division and remainder operations on large :class:`int` values. + +.. + +.. date: 2026-10-28-13-47-43 +.. gh-issue: 160773 +.. nonce: 7VFxkA +.. section: Core and Builtins + +Fix a crash in :func:`str` of an :exc:`OSError` when formatting one of its +attributes modifies the others. + +.. + +.. date: 2026-10-26-12-00-00 +.. gh-issue: 160682 +.. nonce: Q7mK2x +.. section: Core and Builtins + +Fix a bug in the :term:`free-threaded build` where garbage collection could +prematurely collect the function object of a running method. + +.. + +.. date: 2026-10-24-19-07-34 +.. gh-issue: 153568 +.. nonce: 6sVdqs +.. section: Core and Builtins + +Speed up :func:`ast.parse` by ~15.8% and reduce the memory used by ~51.6% by +storing node attributes as inline values instead of in a per-node +dictionary. + +.. + +.. date: 2026-10-23-20-15-24 +.. gh-issue: 160491 +.. nonce: yIQ_Vf +.. section: Core and Builtins + +Fix the location reported for invalid escape sequences in f-strings and +t-strings. The :exc:`SyntaxError` now points at the literal part that +contains the escape, instead of at the closing quote. + +.. + +.. date: 2026-10-23-19-37-00 +.. gh-issue: 160050 +.. nonce: aB3x9Q +.. section: Core and Builtins + +Use asymptotically faster :func:`divmod` implementation for integer modulo +of large integers. + +.. + +.. date: 2026-10-22-17-37-27 +.. gh-issue: 160268 +.. nonce: gJ2iDx +.. section: Core and Builtins + +Fix a crash in :mod:`tokenize` when a mismatched closing bracket follows the +opening brace of an f-string or t-string replacement field, as in +``f'{]a'``. It now raises the same "unmatched" error as the parser. + +.. + +.. date: 2026-10-22-14-45-00 +.. gh-issue: 160245 +.. nonce: mEmO24 +.. section: Core and Builtins + +Reduce the size of parser memo entries on 64-bit platforms by reordering +fields to remove padding. Patch by Shamil Abdulaev. + +.. + +.. date: 2026-10-22-12-30-11 +.. gh-issue: 106045 +.. nonce: fkMW34 +.. section: Core and Builtins + +Fix :mod:`venv` path discovery when the base executable resolves to a +location outside the installation pointed to by the ``home`` key in +``pyvenv.cfg``. The standard library is now found from ``home``, and +extension modules from the resolved executable's installation, instead of +falling back to the compiled-in prefix. + +.. + +.. date: 2026-10-22-00-30-21 +.. gh-issue: 160162 +.. nonce: lShf32 +.. section: Core and Builtins + +Fix a crash on 32-bit platforms when left shifting an :class:`int` by a +count close to ``sys.maxsize * sys.int_info.bits_per_digit``. +:exc:`OverflowError` is now raised, as for other too large counts. + +.. + +.. date: 2026-10-22-00-30-02 +.. gh-issue: 160179 +.. nonce: dQ2e7m +.. section: Core and Builtins + +Fix :meth:`!bytearray.__init__` leaving uninitialized memory in the object +when it fails on a :class:`list` or :class:`tuple` containing an integer not +in ``range(256)``. The object is now empty after the failed call. + +.. + +.. date: 2026-10-21-23-18-38 +.. gh-issue: 160132 +.. nonce: Nj4rQa +.. section: Core and Builtins + +Reduce memory usage during compilation by reordering the fields of the +internal basic block structure to reduce padding. Patch by Shamil Abdulaev. + +.. + +.. date: 2026-10-21-21-26-10 +.. gh-issue: 159739 +.. nonce: Zw43uX +.. section: Core and Builtins + +Raise :exc:`MemoryError` instead of returning an incorrect result if +:func:`round` runs out of memory while rounding a float. + +.. + +.. date: 2026-10-21-21-26-10 +.. gh-issue: 159739 +.. nonce: CbTE4N +.. section: Core and Builtins + +Fix a possible crash when assigning metadata to :class:`classmethod` or +:class:`staticmethod` objects. + +.. + +.. date: 2026-10-21-21-26-10 +.. gh-issue: 159739 +.. nonce: Aw61LR +.. section: Core and Builtins + +Fix a race in ``list.__sizeof__()`` when a list is concurrently resized on a +free-threaded build. + +.. + +.. date: 2026-10-21-19-55-00 +.. gh-issue: 159739 +.. nonce: tali01 +.. section: Core and Builtins + +Fix a reference leak when the value of a type alias is evaluated recursively +or concurrently. + +.. + +.. date: 2026-10-21-19-55-00 +.. gh-issue: 159739 +.. nonce: codc01 +.. section: Core and Builtins + +Fix a crash in free-threaded builds when :func:`codecs.unregister` is called +concurrently. + +.. + +.. date: 2026-10-21-18-00-00 +.. gh-issue: 160368 +.. nonce: sem-clockwait +.. section: Core and Builtins + +On Linux, allow timed semaphore waits to use the monotonic clock when +``sem_clockwait`` is available at runtime, even if Python was built with +older libc headers. + +.. + +.. date: 2026-10-21-14-00-00 +.. gh-issue: 159493 +.. nonce: sfLf1b +.. section: Core and Builtins + +Superinstructions are now formed after the compiler decides which +:opcode:`LOAD_FAST` instructions can borrow their reference. This fixes +reference count contention slowdown on the free-threaded build for +single-line loops and comprehensions. + +.. + +.. date: 2026-10-20-22-00-00 +.. gh-issue: 159963 +.. nonce: pLkHld +.. section: Core and Builtins + +Fix a regression in Python 3.15.0 where threads importing submodules of +packages for the first time could raise ``_DeadlockError`` or a spurious +"partially initialized module" :exc:`ImportError`, or block one another, +with no circular import involved. :func:`importlib.import_module` now +returns the partially initialized module in a cross-thread circular import, +as the :keyword:`import` statement does, instead of raising +``_DeadlockError``. + +.. + +.. date: 2026-10-20-20-38-38 +.. gh-issue: 159101 +.. nonce: vFwrZY +.. section: Core and Builtins + +Fix use-after-free when :meth:`~object.__del__` changes the base class. +Patch by Sergey Miryanov. + +.. + +.. date: 2026-10-20-10-00-00 +.. gh-issue: 158195 +.. nonce: UnpinTL +.. section: Core and Builtins + +Fix a deadlock in the free-threaded build between a thread that stops the +world while running Python code under the type lock (for example a custom +metaclass ``mro()`` or the ``__eq__`` of a :class:`str` subclass used as a +class ``__dict__`` key) and a thread that assigns to a special method or to +``__bases__`` of a class. The type lock is no longer held while waiting for +a stop-the-world; the prepared slot, MRO and bases updates are validated +with the world stopped and recomputed if they went stale. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: da0f04 +.. section: Core and Builtins + +Fix reference leaks when free-threaded garbage collection aborts early +because of an allocation error. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: d76e6d +.. section: Core and Builtins + +Prevent free-threaded builds from interning and immortalizing complex +subclasses in code object constants. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: bf569b +.. section: Core and Builtins + +Fix a reference leak in binary, ternary, and comparison operations on weak +reference proxies when another operand is a dead proxy. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: a3cc5d +.. section: Core and Builtins + +Fix a crash when constructing an error for invalid generic alias arguments. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: 848b18 +.. section: Core and Builtins + +Allow the garbage collector to break reference cycles involving property +accessors or names. + +.. + +.. date: 2026-10-19-21-30-00 +.. gh-issue: 159860 +.. nonce: qsbr128 +.. section: Core and Builtins + +Fix false sharing between the per-thread QSBR states in the +:term:`free-threaded build` on AArch64 CPUs with 128-byte cache lines, such +as Apple Silicon. CPU-bound threads that share no objects now scale like +processes there. + +.. + +.. date: 2026-10-19-20-38-17 +.. gh-issue: 159739 +.. nonce: fc0ed2 +.. section: Core and Builtins + +Avoid a crash if allocation fails while creating a +:class:`string.templatelib.Template` iterator. + +.. + +.. date: 2026-10-19-20-38-17 +.. gh-issue: 159739 +.. nonce: b62fc9 +.. section: Core and Builtins + +Propagate errors during generic class construction and avoid a crash after +failed generic alias substitution. + +.. + +.. date: 2026-10-19-20-38-17 +.. gh-issue: 159739 +.. nonce: 78e530 +.. section: Core and Builtins + +Raise :exc:`OverflowError` when the absolute value of a dynamic printf-style +formatting width is too large. + +.. + +.. date: 2026-10-19-20-38-17 +.. gh-issue: 159739 +.. nonce: 55fd08 +.. section: Core and Builtins + +Fix a reference leak in :func:`gc.get_objects` and :func:`gc.get_referrers` +when an internal allocation fails in the free-threaded build. + +.. + +.. date: 2026-10-19-19-33-35 +.. gh-issue: 159739 +.. nonce: f81734 +.. section: Core and Builtins + +Fix a crash in :func:`warnings.warn` when ``skip_file_prefixes`` is an empty +tuple subclass. Propagate allocation errors when creating the fallback +filename. + +.. + +.. date: 2026-10-19-19-33-35 +.. gh-issue: 159739 +.. nonce: c25c27 +.. section: Core and Builtins + +Fix memory leaks during interpreter startup when a path configuration file +is empty or too large, or when an audit hook rejects a command. + +.. + +.. date: 2026-10-19-19-33-35 +.. gh-issue: 159739 +.. nonce: a8775b +.. section: Core and Builtins + +Avoid a crash when an interpreter configuration string cannot be encoded as +UTF-8. + +.. + +.. date: 2026-10-19-19-33-35 +.. gh-issue: 159739 +.. nonce: 1fc3af +.. section: Core and Builtins + +Preserve the original :exc:`SyntaxError` if allocating optional error +metadata fails. + +.. + +.. date: 2026-10-19-19-33-35 +.. gh-issue: 159739 +.. nonce: 173d49 +.. section: Core and Builtins + +Avoid a crash if formatting an integer with the ``c`` presentation type runs +out of memory. + +.. + +.. date: 2026-10-19-19-33-35 +.. gh-issue: 159739 +.. nonce: 09590b +.. section: Core and Builtins + +Avoid a crash when a module filename cannot be encoded as UTF-8. + +.. + +.. date: 2026-10-19-16-38-39 +.. gh-issue: 159813 +.. nonce: wHfSOc +.. section: Core and Builtins + +Add a fast path for :const:`None` in :c:func:`PyUnicodeWriter_WriteRepr`. +Patch by Donghee Na. + +.. + +.. date: 2026-10-18-00-30-00 +.. gh-issue: 159652 +.. nonce: Qk3vLm +.. section: Core and Builtins + +Fix a hang when an import fails with :exc:`RecursionError` because the C +stack is nearly exhausted: releasing the import locks could itself fail and +leave the global import lock or a module lock held forever, so that later +imports in other threads blocked indefinitely. The import system now raises +:exc:`RecursionError` before taking an import lock when there is not enough +stack left to release it. + +.. + +.. date: 2026-10-17-02-47-26 +.. gh-issue: 159603 +.. nonce: OFagJz +.. section: Core and Builtins + +On FreeBSD, call ``pthread_attr_get_np()`` to get the stack size (and guard +size) of the current thread. Patch by Victor Stinner. + +.. + +.. date: 2026-10-16-18-25-48 +.. gh-issue: 159564 +.. nonce: WMRRFA +.. section: Core and Builtins + +Improve the performance and thread scaling of dereferencing weak references +in the :term:`free-threaded build`. Dereferencing a weak reference no +longer takes a lock. + +.. + +.. date: 2026-10-15-12-40-00 +.. gh-issue: 87613 +.. nonce: Sl1ceV +.. section: Core and Builtins + +Speed up calling :class:`slice` by adding a vectorcall implementation of the +constructor generated by Argument Clinic. Patch by Pieter Eendebak. + +.. + +.. date: 2026-10-15-12-36-09 +.. gh-issue: 130279 +.. nonce: N9N-sw +.. section: Core and Builtins + +Fix bug where compiler did not set the right exception handler for some jump +instructions. + +.. + +.. date: 2026-10-15-12-10-00 +.. gh-issue: 87613 +.. nonce: Wr4fvc +.. section: Core and Builtins + +Speed up creating :class:`weakref.ref` objects by adding a vectorcall +implementation of the constructor generated by Argument Clinic. Patch by +Pieter Eendebak. + +.. + +.. date: 2026-10-15-10-15-27 +.. gh-issue: 159449 +.. nonce: xsdzFf +.. section: Core and Builtins + +Fix a free-threaded asyncio race leading to segfaults. + +.. + +.. date: 2026-10-14-23-05-00 +.. gh-issue: 132042 +.. nonce: k7Qs2w +.. section: Core and Builtins + +Speed up class creation by resolving the special method slots of a new class +with one pass over each class dict along the MRO instead of one lookup per +slot name and per class. + +.. + +.. date: 2026-10-14-19-16-53 +.. gh-issue: 159434 +.. nonce: T4AsrQ +.. section: Core and Builtins + +Fix reading a function's :attr:`~function.__annotations__` using borrowed +references to :attr:`~function.__annotate__` and the internal annotations +tuple while running code that can replace them. If ``__annotations__`` is +set while ``__annotate__`` runs, the newly set value is now kept. + +.. + +.. date: 2026-10-14-18-45-35 +.. gh-issue: 159431 +.. nonce: mVTMsA +.. section: Core and Builtins + +Fix an assertion failure, or an immortal object becoming mortal, on the +:term:`free threaded ` build when an object passed to the +internal delayed decref mechanism is immortal or becomes immortal before the +delayed decref is processed by the garbage collector. + +.. + +.. date: 2026-10-14-11-40-06 +.. gh-issue: 159430 +.. nonce: OWpyPm +.. section: Core and Builtins + +Report the correct exception if looking up :meth:`~object.__bytes__` raises +an exception while performing %-formatting of :class:`bytes`. + +.. + +.. date: 2026-10-14-07-36-44 +.. gh-issue: 159399 +.. nonce: cjuE0N +.. section: Core and Builtins + +Fix a crash in :func:`codecs.encode` and :func:`codecs.decode` in the +:term:`free-threaded build` when another thread concurrently unregisters a +codec search function. Patch by Christian Aurich Zanettini Martins. + +.. + +.. date: 2026-10-13-16-38-00 +.. gh-issue: 159320 +.. nonce: _Yhg1s +.. section: Core and Builtins + +Add dynamic detection of stack sizes on Android, allowing safer deep +recursion. + +.. + +.. date: 2026-10-13-06-23-56 +.. gh-issue: 159294 +.. nonce: 9LTsNA +.. section: Core and Builtins + +Fix a crash in a debug build when creating an :exc:`ExceptionGroup` or +:exc:`BaseExceptionGroup` fails with :exc:`MemoryError`. Patch by Christian +Aurich Zanettini Martins. + +.. + +.. date: 2026-10-13-04-03-48 +.. gh-issue: 159288 +.. nonce: vhqBQG +.. section: Core and Builtins + +Fix a crash when walking a very long chain of lazy import placeholders, as +created by an aliased dotted import such as ``lazy import a.b.c as d``. +:exc:`RecursionError` is now raised instead. + +.. + +.. date: 2026-10-12-20-00-00 +.. gh-issue: 159136 +.. nonce: fnAttr +.. section: Core and Builtins + +Setting ``__name__``, ``__qualname__``, ``__doc__`` or ``__module__`` on a +function no longer stops all threads in the free-threaded build. The setters +now use the per-object lock and defer freeing the old value with QSBR, the +same approach used for generator ``__name__``. This fixes a large slowdown +in multi-threaded code using :func:`functools.wraps` since 3.14.7. Readers +of these attributes now use atomic loads, and argument error reporting reads +``__qualname__`` after keyword comparisons, which may execute Python code +and replace it. Previously this could use a freed object. + +.. + +.. date: 2026-10-12-16-00-34 +.. gh-issue: 159244 +.. nonce: ThiobE +.. section: Core and Builtins + +Fix a crash in :class:`dict` and :meth:`dict.update` in the +:term:`free-threaded build` when another thread concurrently mutates a list +used as a key/value pair. Patch by Christian Aurich Zanettini Martins. + +.. + +.. date: 2026-10-12-11-59-37 +.. gh-issue: 68241 +.. nonce: jVpLIJ +.. section: Core and Builtins + +Add EXIT_FAILURE and EXIT_SUCCESS in the sys module. + +.. + +.. date: 2026-10-11-06-36-25 +.. gh-issue: 159188 +.. nonce: Fq7wLm +.. section: Core and Builtins + +In the :term:`free-threaded build`, :func:`compile` no longer immortalizes +the constants of the code objects when it is given an AST object, as was +already the case for a source string. The output of :mod:`marshal` for such +code no longer depends on the code objects which were created before in the +process. + +.. + +.. date: 2026-10-10-20-21-13 +.. gh-issue: 159111 +.. nonce: XVf2k- +.. section: Core and Builtins + +Update mimalloc version from 2.1.2 to 2.5.2. Patch by Donghee Na. + +.. + +.. date: 2026-10-09-16-04-40 +.. gh-issue: 159041 +.. nonce: PUP8bU +.. section: Core and Builtins + +Fix a use-after-free in :mod:`asyncio` task bookkeeping when a finalizer +creates a task during thread shutdown. + +.. + +.. date: 2026-10-07-00-00-00 +.. gh-issue: 158973 +.. nonce: warnings-fork +.. section: Core and Builtins + +Reset the warnings lock in the child after :func:`os.fork`, before clearing +other threads' states, to avoid deadlocks when their finalizers issue +warnings. Preserve the lock's recursion depth if the forking thread owns it. + +.. + +.. date: 2026-10-05-13-19-49 +.. gh-issue: 87613 +.. nonce: JLTGBy +.. section: Core and Builtins + +Speed up :class:`bytes` and :class:`bytearray` construction by adding +:ref:`vectorcall` support. + +.. + +.. date: 2026-10-04-20-04-55 +.. gh-issue: 158816 +.. nonce: 9ulhWb +.. section: Core and Builtins + +Speed up deleting items from split dictionaries (for example the +materialized ``__dict__`` of an instance with inline values) by skipping a +redundant hash table probe. + +.. + +.. date: 2026-10-03-13-00-00 +.. gh-issue: 158197 +.. nonce: Qk7sVd +.. section: Core and Builtins + +Fix crashes and debug-build assertion failures in the :term:`free-threaded +build` when iterating over a split dictionary while another thread deletes +entries or clears it. Publish insertion-order updates atomically so readers +cannot observe an increased size before the corresponding order entry is +initialized. + +.. + +.. date: 2026-10-01-00-00-00 +.. gh-issue: 158525 +.. nonce: Xw3K9a +.. section: Core and Builtins + +Fix slow ``dict.copy()`` for dicts with many deleted entries in the +free-threaded build. + +.. + +.. date: 2026-09-30-21-29-54 +.. gh-issue: 158529 +.. nonce: kZO7mX +.. section: Core and Builtins + +Fixed crash on Android x86_64, API levels 27-28, caused by mimalloc calling +system calls directly. + +.. + +.. date: 2026-09-29-14-35-00 +.. gh-issue: 158121 +.. nonce: Kj4nQ8 +.. section: Core and Builtins + +Fix missing :mod:`sys.monitoring` ``INSTRUCTION`` events when a ``LINE`` +callback returns :data:`sys.monitoring.DISABLE`. + +.. + +.. date: 2026-09-28-10-12-25 +.. gh-issue: 158304 +.. nonce: 3pdcnJ +.. section: Core and Builtins + +Speed up iteration over asynchronous generators: values yielded to ``async +for`` no longer create and discard a :exc:`StopIteration` exception at every +nesting level. The ``async_generators`` pyperformance benchmark is about 30% +faster. Patch by Shamil Abdulaev. + +.. + +.. date: 2026-09-25-11-05-00 +.. gh-issue: 157746 +.. nonce: q7Lk3d +.. section: Core and Builtins + +Reduce the overhead of :c:func:`PyErr_CheckSignals` when no signal is +pending: the cheap flag checks are now done before the main-thread check and +the remote debugger configuration lookup. + +.. + +.. date: 2026-09-21-12-44-38 +.. gh-issue: 132414 +.. nonce: uwxmd5 +.. section: Core and Builtins + +Quote the invalid option in the error message for an unknown command-line +option: ``Invalid option '-z'`` instead of ``Unknown option: -z``. A stray +space, as passed by a shebang line like ``#!/usr/bin/python3 -q -i``, is now +reported as ``Invalid option '- '``. + +.. + +.. date: 2026-09-13-10-00-00 +.. gh-issue: 157361 +.. nonce: aiterpy +.. section: Core and Builtins + +Implement :func:`aiter` in Python instead of C, in the frozen +``_pybuiltins`` module. The async iterator returned by ``aiter(callable, +...)`` now has a plain coroutine ``__anext__()``, so introspection tools +such as :func:`asyncio.print_call_graph` can see through it into the awaited +callable result. + +.. + +.. date: 2026-09-08-17-40-00 +.. gh-issue: 157194 +.. nonce: 17Ffhk +.. section: Core and Builtins + +Fix a data race in the free-threaded build, reported by ThreadSanitizer, +between the specializer reading an instruction opcode and another thread +writing it. Only reachable when thread-local bytecode is disabled with ``-X +tlbc=0``, which makes threads share one copy of the bytecode. + +.. + +.. date: 2026-09-07-09-52-21 +.. gh-issue: 156443 +.. nonce: tT40sm +.. section: Core and Builtins + +Speed up :func:`float` conversion and true division of large :class:`int` +values. + +.. + +.. date: 2026-08-30-16-30-00 +.. gh-issue: 135385 +.. nonce: Wn7Rd4 +.. section: Core and Builtins + +Reduce the memory consumption of instances of classes which define +``__slots__`` and also have a ``__dict__``. Space for the attributes stored +in slots is no longer reserved in the inline values of every instance. + +.. + +.. date: 2026-08-27-17-26-21 +.. gh-issue: 106045 +.. nonce: wzTH7f +.. section: Core and Builtins + +Fix standard library discovery in virtual environments created from a Python +executable symlink located outside its base installation. + +.. + +.. date: 2026-08-27-00-00-00 +.. gh-issue: 156399 +.. nonce: aB3xYz +.. section: Core and Builtins + +Fix a regression where attribute access on a :class:`types.ModuleType` +subclass could bypass a data descriptor defined on the subclass. The +``LOAD_ATTR_MODULE`` specialization guarded on ``tp_getattro``, which +subclasses inherit, and then read the module dictionary directly. + +.. + +.. date: 2026-08-21-14-37-50 +.. gh-issue: 127716 +.. nonce: KrPth8 +.. section: Core and Builtins + +On the free-threaded build, :class:`memoryview` lifetime and bookkeeping +operations are now thread safe: an operation that reads the buffer holds the +managed buffer for its duration, so a concurrent :meth:`~memoryview.release` +can no longer free the buffer under it; slicing a memoryview from several +threads no longer loses buffer registrations and raises ``ValueError: +operation forbidden on released memoryview object``; and :func:`hash` of a +memoryview is no longer a data race. As a result :meth:`~memoryview.release` +may now return while an operation started by another thread is still +running, see :ref:`thread-safety-memoryview`. + +.. + +.. date: 2026-07-21-10-42-15 +.. gh-issue: 154318 +.. nonce: Kq3Wm7 +.. section: Core and Builtins + +Fix a crash when hashing a deeply nested :class:`tuple` or +:class:`frozendict`. A :exc:`RecursionError` is now raised instead of +overflowing the C stack. + +.. + +.. date: 2026-07-11-15-00-13 +.. gh-issue: 153568 +.. nonce: atomfp +.. section: Core and Builtins + +Speed up parsing of simple expressions by recognizing single-token atoms +without descending the full precedence rule chain. + +.. + +.. date: 2026-07-04-18-00-00 +.. gh-issue: 153014 +.. nonce: gCdBg1 +.. section: Core and Builtins + +Fix a data race on the garbage collector debug flag in free-threading +builds: :func:`gc.set_debug` and :func:`gc.get_debug` now access it +atomically. + +.. + +.. date: 2026-06-29-23-01-07 +.. gh-issue: 152631 +.. nonce: LKgCM_ +.. section: Core and Builtins + +Fix :func:`ast.parse` ability to parse lazy imports with *feature_version* +older than 3.15, now it properly raises :exc:`SyntaxError`. + +.. + +.. date: 2026-06-26-00-02-29 +.. gh-issue: 149816 +.. nonce: JKN-5B +.. section: Core and Builtins + +Fix a race in free-threaded builds where storing an instance attribute could +use a stale borrowed reference to an instance ``__dict__`` after another +thread replaced it. + +.. + +.. date: 2026-06-20-18-00-00 +.. gh-issue: 151818 +.. nonce: LsApDf +.. section: Core and Builtins + +Fix a double-free crash in ``list.append()`` (the specialized +``CALL_LIST_APPEND`` instruction) when growing the list fails with +:exc:`MemoryError`. + +.. + +.. date: 2026-06-07-22-03-20 +.. gh-issue: 151060 +.. nonce: tdexEE +.. section: Core and Builtins + +Fix a crash or memory corruption in :func:`sum` when summing a +:class:`float` subclass instance into a complex running total. + +.. + +.. date: 2026-05-21-14-21-06 +.. gh-issue: 150075 +.. nonce: dLI21Z +.. section: Core and Builtins + +tar.addfile() now sets member offsets in TarInfo objects + +.. + +.. date: 2026-04-23-08-36-47 +.. gh-issue: 148891 +.. nonce: AMMHow +.. section: Core and Builtins + +Fix a crash in :func:`contextvars.Context.run` where a GC invocation could +free the current HAMT during traverse. + +.. + +.. date: 2026-04-15-21-55-45 +.. gh-issue: 69605 +.. nonce: NwoP6l +.. section: Core and Builtins + +Fix :term:`REPL` completions inserted incorrectly when there was spaces in +import statements. + +.. + +.. date: 2026-04-12-13-48-26 +.. gh-issue: 69605 +.. nonce: T9HWOL +.. section: Core and Builtins + +Insert ``import`` after ``from x.y.z `` in the :term:`REPL` +auto-completion. + +.. + +.. date: 2026-03-14-11-20-52 +.. gh-issue: 145019 +.. nonce: ls2EXo +.. section: Core and Builtins + +Improve :exc:`SyntaxError` when :keyword:`match` alternative patterns bind +different names. Patch by Bénédikt Tran. + +.. + +.. date: 2025-08-19-16-10-26 +.. gh-issue: 137966 +.. nonce: nEgFAt +.. section: Core and Builtins + +Add support for additional special names to the built-in :func:`help` +function. Patch by Adam Hartz. + +.. + +.. date: 2024-10-27-22-23-57 +.. gh-issue: 125873 +.. nonce: BG97KU +.. section: Core and Builtins + +Improved error message when an environment variable is misconfigured. + +.. + +.. date: 2024-08-11-12-06-23 +.. gh-issue: 122901 +.. nonce: B2Huom +.. section: Core and Builtins + +Speed up creating :class:`str` objects from non-ASCII character data. + +.. + +.. date: 2024-03-07-22-38-07 +.. gh-issue: 116477 +.. nonce: AqwRAV +.. section: Core and Builtins + +Speed up the :class:`range` object constructor in the case of a single +argument. + +.. + +.. date: 2026-11-06-11-59-26 +.. gh-issue: 161475 +.. nonce: 82LWf4 +.. section: Library + +Fix :mod:`ctypes` structures containing arrays of floating-point type +returned by value on 64-bit Linux SPARC. + +.. + +.. date: 2026-11-06-05-42-11 +.. gh-issue: 71199 +.. nonce: qJN4cV +.. section: Library + +Change :mod:`os` module types such as :class:`~os.DirEntry` to report ``os`` +as their module on every platform. :ref:`Pickled ` objects +using the platform-specific qualified name, such as ``nt.times_result``, +still load. + +.. + +.. date: 2026-11-06-00-00-00 +.. gh-issue: 142782 +.. nonce: qU7nE4 +.. section: Library + +Fix a use-after-free in :class:`zoneinfo.ZoneInfo` when a key's ``__eq__`` +clears the cache during a lookup. Strong cache keys are now compared by +string contents. + +.. + +.. date: 2026-11-05-12-00-24 +.. gh-issue: 161415 +.. nonce: NJhrAR +.. section: Library + +Remove ``FILES`` and ``unicode`` from the related help topics listed by +:func:`help` for the ``TYPES`` and ``UNICODE`` topics. Neither exists in +Python 3, so asking for help on them failed. + +.. + +.. date: 2026-11-05-11-59-58 +.. gh-issue: 161414 +.. nonce: Xnxe8A +.. section: Library + +Remove the backquote from the symbols listed by :func:`help`, asking for +help on it reported that no documentation was found. + +.. + +.. date: 2026-11-05-11-56-50 +.. gh-issue: 161389 +.. nonce: EIl2E6 +.. section: Library + +Fixes a crash in :class:`decimal.Context` when passing invalid signaling +dicts as *flags* and *traps*. + +.. + +.. date: 2026-11-05-10-05-00 +.. gh-issue: 161387 +.. nonce: def456 +.. section: Library + +Fix a crash in :mod:`decimal` when the internal ``decimal_context`` +:class:`contextvars.ContextVar` holds a value that is not a context. + +.. + +.. date: 2026-11-03-21-26-09 +.. gh-issue: 161295 +.. nonce: IJqoJM +.. section: Library + +:class:`reprlib.Repr` objects now have a meaningful :meth:`~object.__repr__` +describing the instance attributes. + +.. + +.. date: 2026-11-03-18-56-28 +.. gh-issue: 161259 +.. nonce: iyZz6y +.. section: Library + +Improved the accuracy of :func:`statistics.correlation` for very small +inputs, where the product of the sums of squares is subnormal rather than +zero. + +.. + +.. date: 2026-11-03-12-00-00 +.. gh-issue: 93821 +.. nonce: aCcRsT +.. section: Library + +Fix :class:`asyncio.ProactorEventLoop` servers that stopped accepting new +connections on Windows when a client reset its connection before it was +accepted (``WinError 64``, ``ERROR_NETNAME_DELETED``). The listening socket +is now kept open and the server continues to accept connections. + +.. + +.. date: 2026-11-03-12-00-00 +.. gh-issue: 161076 +.. nonce: nOdLy1 +.. section: Library + +Fix :mod:`asyncio` leaving a half-created transport registered with the +event loop and the server when setting ``TCP_NODELAY`` on a socket fails. +Such a failure could make :meth:`asyncio.Server.wait_closed` hang and +prevent new connections from being served. Errors from +``setsockopt(TCP_NODELAY)`` are now ignored. + +.. + +.. date: 2026-11-03-10-33-33 +.. gh-issue: 161233 +.. nonce: _9VrfL +.. section: Library + +Fix unraisable :exc:`TypeError` when ``None`` was passed as a *timer* to +:class:`cProfile.Profile `. + +.. + +.. date: 2026-11-02-21-10-00 +.. gh-issue: 161212 +.. nonce: pip263 +.. section: Library + +Bump the version of :pypi:`pip` bundled in :mod:`ensurepip` to version 26.3. + +.. + +.. date: 2026-11-02-16-28-11 +.. gh-issue: 161178 +.. nonce: D3-23R +.. section: Library + +Fix :meth:`asyncio.Task.print_stack` to include frames from async generator +operations. Patch by Timofei Ivankov + +.. + +.. date: 2026-11-02-13-26-36 +.. gh-issue: 161179 +.. nonce: H8Yyaw +.. section: Library + +Fix the function names in the error messages of :func:`os.spawnve`, +:func:`os.preadv` and :meth:`tkinter.Wm.wm_attributes`. + +.. + +.. date: 2026-11-02-13-11-30 +.. gh-issue: 161166 +.. nonce: zfFDlW +.. section: Library + +Fix :meth:`asyncio.Task.print_stack` displaying an incorrect exception line +after a ``finally`` block. Patch by Timofei Ivankov. + +.. + +.. date: 2026-11-01-22-07-24 +.. gh-issue: 161124 +.. nonce: c2YmcV +.. section: Library + +Fix :func:`asyncio.print_call_graph` reporting event loop frames when +*depth* exceeds the task stack. Patch by Timofei Ivankov. + +.. + +.. date: 2026-11-01-19-38-33 +.. gh-issue: 161083 +.. nonce: PM21lv +.. section: Library + +Fix stale task nodes left by :func:`asyncio.gather` when argument validation +fails. Patch by Timofei Ivankov + +.. + +.. date: 2026-10-31-21-34-00 +.. gh-issue: 161039 +.. nonce: k7_pTD +.. section: Library + +Fix the :mod:`pydoc` pager not detecting ``less`` 706 and newer. + +.. + +.. date: 2026-10-30-12-00-00 +.. gh-issue: 137549 +.. nonce: Tm0Bxg +.. section: Library + +:func:`asyncio.timeout` now inserts :exc:`TimeoutError` into the context +chain of sub-exceptions of a :exc:`BaseExceptionGroup`, as it already did +for :exc:`ExceptionGroup`. + +.. + +.. date: 2026-10-30-10-36-30 +.. gh-issue: 160782 +.. nonce: n6JCUU +.. section: Library + +:mod:`venv`: a symbolic link or junction in place of a directory inside the +environment now raises :exc:`ValueError` instead of being followed. + +.. + +.. date: 2026-10-29-23-25-31 +.. gh-issue: 160782 +.. nonce: 1ZwCOr +.. section: Library + +:mod:`venv`: a backslash before a quote in the prompt or the path no longer +breaks ``activate.fish``. + +.. + +.. date: 2026-10-29-21-43-53 +.. gh-issue: 160429 +.. nonce: eWapMs +.. section: Library + +Fix :class:`asyncio.ProactorEventLoop` socket transports leaving the socket +open, and :meth:`asyncio.Server.wait_closed` hanging, when the peer resets +the connection while the transport is being closed. + +.. + +.. date: 2026-10-29-21-36-08 +.. gh-issue: 160929 +.. nonce: mK8n2Q +.. section: Library + +Fix reverse union with :class:`~collections.ChainMap` to preserve its lookup +semantics for defaulting mappings such as :class:`~collections.Counter` and +:class:`~collections.defaultdict`. + +.. + +.. date: 2026-10-29-20-52-57 +.. gh-issue: 50720 +.. nonce: Kd1VMR +.. section: Library + +:exc:`urllib.error.URLError` now sets the :attr:`~OSError.errno` and +:attr:`~OSError.strerror` attributes from +:attr:`~urllib.error.URLError.reason` when it is an :exc:`OSError`. On +Windows, :attr:`~OSError.winerror` is set as well. + +.. + +.. date: 2026-10-29-19-42-53 +.. gh-issue: 53089 +.. nonce: ewaIBvY6 +.. section: Library + +:class:`urllib.request.HTTPDigestAuthHandler` now sends ``uri="/"`` in the +Digest ``Authorization`` header for URLs with an empty path, such as +``http://example.com``, matching the request-target sent by +:mod:`http.client`. Previously it sent ``uri=""``, which servers could +reject. Patch by Senthil Kumaran. + +.. + +.. date: 2026-10-28-20-36-44 +.. gh-issue: 109564 +.. nonce: WqoyNE +.. section: Library + +Fix a race in :mod:`asyncio` where closing a server after it accepted a +connection, but before the connection's transport was created, reported an +:exc:`AssertionError` to the exception handler. Such connections are now +closed quietly. + +.. + +.. date: 2026-10-28-16-40-00 +.. gh-issue: 66333 +.. nonce: pdb-sigint +.. section: Library + +Allow Ctrl-C to interrupt execution resumed with the :mod:`pdb` ``next``, +``until``, and ``return`` commands without unwinding the debugged program, +as with ``continue``. + +.. + +.. date: 2026-10-28-13-23-28 +.. gh-issue: 59515 +.. nonce: wt31yNN4 +.. section: Library + +:class:`urllib.request.HTTPDigestAuthHandler` and +:class:`urllib.request.ProxyDigestAuthHandler` now parse all +``WWW-Authenticate`` (or ``Proxy-Authenticate``) headers and accept multiple +challenges per header: the first Digest challenge is used. + +.. + +.. date: 2026-10-28-12-00-00 +.. gh-issue: 53852 +.. nonce: Pr0xy4 +.. section: Library + +:class:`urllib.request.ProxyDigestAuthHandler` now raises +:exc:`~urllib.error.HTTPError` with status code 407 (Proxy Authentication +Required) instead of 401 when proxy authentication fails. + +.. + +.. date: 2026-10-28-08-12-33 +.. gh-issue: 160811 +.. nonce: wQ3xT7 +.. section: Library + +Fix :meth:`io.TextIOWrapper.tell` reading past the end of the buffer after +:meth:`~io.TextIOWrapper.seek`. + +.. + +.. date: 2026-10-28-07-34-05 +.. gh-issue: 160806 +.. nonce: ZZRkhi +.. section: Library + +Fix the pure-Python implementation of :class:`datetime.timedelta` losing +sub-second precision with large float arguments. It now gives the same +results as the C implementation. + +.. + +.. date: 2026-10-28-07-26-47 +.. gh-issue: 160804 +.. nonce: R5HeTQ +.. section: Library + +:mod:`asyncio` call graph introspection now includes frames of generator +based awaitables, such as generators returned by ``__await__`` and +:func:`types.coroutine` generators. + +.. + +.. date: 2026-10-27-15-42-16 +.. gh-issue: 160781 +.. nonce: VGLB9S +.. section: Library + +Fix :class:`io.BytesIO` implementation of :meth:`io.IOBase.truncate` +rejecting sizes larger than ``2**31 - 1`` on Windows. + +.. + +.. date: 2026-10-27-15-27-32 +.. gh-issue: 160781 +.. nonce: kARuHu +.. section: Library + +Fix :class:`io.StringIO` implementation of :meth:`io.IOBase.truncate` +rejecting sizes larger than ``2**31 - 1`` on Windows. + +.. + +.. date: 2026-10-27-15-26-53 +.. gh-issue: 160781 +.. nonce: 4VSeQK +.. section: Library + +Fix :class:`io.BytesIO` raising :exc:`SystemError` instead of +:exc:`OverflowError` when writing at a position close to +:data:`sys.maxsize`. + +.. + +.. date: 2026-10-26-21-37-42 +.. gh-issue: 158500 +.. nonce: zxyuq5 +.. section: Library + +:meth:`datetime.date.fromisoformat` now rejects strings with trailing +characters after a basic-format date, such as ``'2020010112'``. The +pure-Python :meth:`datetime.datetime.fromisoformat` now rejects a date +followed by a bare separator, such as ``'2020-01-01T'``, matching the C +implementation. + +.. + +.. date: 2026-10-26-16-17-45 +.. gh-issue: 160708 +.. nonce: w7z33g +.. section: Library + +Fix out-of-bounds memory access in :func:`bool` on :mod:`ctypes` simple +objects enlarged by :func:`ctypes.resize`, and when passing such objects to +a foreign function. + +.. + +.. date: 2026-10-26-16-15-29 +.. gh-issue: 160427 +.. nonce: 9G4EBb +.. section: Library + +Fix :attr:`~io.TextIOBase.buffer` of a detached :class:`io.TextIOWrapper` +raising :exc:`ValueError` instead of returning ``None``. + +.. + +.. date: 2026-10-24-22-32-23 +.. gh-issue: 160596 +.. nonce: RKll-Z +.. section: Library + +Fix the error message for invalid data passed to +:meth:`!asyncio.proactor_events.sendto`. + +.. + +.. date: 2026-10-24-07-58-00 +.. gh-issue: 160561 +.. nonce: b3Kq1z +.. section: Library + +Fix a descriptor leak in :func:`socket.fromfd` when the socket constructor +rejects its arguments. The duplicated descriptor was never closed. + +.. + +.. date: 2026-10-24-01-53-42 +.. gh-issue: 160433 +.. nonce: DSTlog +.. section: Library + +Fix incorrect backup names around daylight saving time changes in +:class:`logging.handlers.TimedRotatingFileHandler`, including delayed +rollovers. + +.. + +.. date: 2026-10-23-23-21-21 +.. gh-issue: 160549 +.. nonce: q7ZdMt +.. section: Library + +Fix a crash when a :class:`compression.zstd.ZstdCompressor` that uses worker +threads and a :class:`~compression.zstd.ZstdDict` is deallocated, or the +interpreter exits, while data passed to +:meth:`~compression.zstd.ZstdCompressor.compress` is still being compressed. + +.. + +.. date: 2026-10-23-22-34-55 +.. gh-issue: 160535 +.. nonce: Zk3dGc +.. section: Library + +Fix a crash in :mod:`compression.zstd` when the garbage collector frees a +:class:`~compression.zstd.ZstdCompressor` that uses worker threads together +with its :class:`~compression.zstd.ZstdDict` before the compression of +already submitted data has finished. + +.. + +.. date: 2026-10-23-21-52-40 +.. gh-issue: 60299 +.. nonce: Af6IzL +.. section: Library + +Fix :class:`urllib.request.ProxyDigestAuthHandler` failing to authenticate. +It looked up credentials using the target request URL instead of the proxy's +authority, so passwords registered via ``add_password()`` against the proxy +never matched. Patch by Senthil Kumaran. + +.. + +.. date: 2026-10-22-22-51-40 +.. gh-issue: 160386 +.. nonce: wYyboz +.. section: Library + +Applied changes to ``importlib.metadata`` from `importlib_metadata 9.1 +`_: +distribution discovery now prefers the greater of several metadata +directories supplied by one path, so stale metadata left by an unclean +upgrade no longer shadows the version in use; a missing required metadata +field now raises a ``KeyError`` naming both the field and the distribution; +``EntryPoint`` value validation is deferred until the value is used, so one +malformed entry point no longer prevents others from being selected; +``importlib.metadata.diagnose`` no longer crashes on a metadata directory +with no metadata file or no declared name; and ``SimplePath.joinpath`` and +``SimplePath.__truediv__`` take ``other`` positional-only. + +.. + +.. date: 2026-10-22-18-30-00 +.. gh-issue: 160520 +.. nonce: sSdTmo +.. section: Library + +Fix :mod:`asyncio` SSL transports not applying *ssl_shutdown_timeout* when +the peer has sent EOF but does not read the data still queued for it. +Closing such a connection could wait indefinitely; it is now aborted with +:exc:`TimeoutError` once the timeout expires. + +.. + +.. date: 2026-10-22-18-30-00 +.. gh-issue: 160326 +.. nonce: nAn7Tq +.. section: Library + +Raise :exc:`ValueError` for a NaN *timeout* in +:meth:`threading.Condition.wait` and the blocking calls built on it, such as +:meth:`threading.Event.wait`, :meth:`threading.Semaphore.acquire` and +:meth:`queue.Queue.get`, instead of returning at once or looping without +ever timing out. + +.. + +.. date: 2026-10-22-18-30-00 +.. gh-issue: 160314 +.. nonce: q7RkZd +.. section: Library + +Raise an error in :class:`lzma.LZMAFile`, :func:`lzma.decompress`, +:class:`bz2.BZ2File` and :func:`bz2.decompress` for a damaged ``.xz`` or +bzip2 stream that follows another stream, instead of silently ignoring it +and everything after it as trailing data. + +.. + +.. date: 2026-10-22-16-30-00 +.. gh-issue: 159739 +.. nonce: tRq417 +.. section: Library + +Fix crashes in :mod:`sqlite3` when a row or text factory is replaced during +a query, when cursor metadata is read during execution in a free-threaded +build, or when allocating cursor metadata fails. + +.. + +.. date: 2026-10-22-16-30-00 +.. gh-issue: 159739 +.. nonce: mFWntK +.. section: Library + +Fix crashes in :mod:`profiling.tracing` when a custom timer reinitializes +the profiler or its result clears or disables the profiler during conversion +or cleanup. + +.. + +.. date: 2026-10-22-16-23-01 +.. gh-issue: 159739 +.. nonce: XmlH3a +.. section: Library + +Fix crashes in :mod:`xml.parsers.expat` when a character data callback +disables buffering or clears another handler during a flush, or when +interning an attribute name changes +:attr:`~xml.parsers.expat.xmlparser.ordered_attributes`. + +.. + +.. date: 2026-10-22-16-23-01 +.. gh-issue: 159739 +.. nonce: CjkR5a +.. section: Library + +Fix the ``hz`` codec sometimes encoding ``~`` as a single ``~`` in text that +mixes ASCII and Chinese characters, and the incremental ``iso2022_*`` codecs +mishandling an escape sequence split across calls or keeping the previous +character set after :meth:`!reset`. Also fix possible crashes in the CJK +incremental and stream codecs when their ``errors`` attribute or their +class's ``codec`` attribute is replaced while they are in use. + +.. + +.. date: 2026-10-22-16-18-42 +.. gh-issue: 159739 +.. nonce: TioR3a +.. section: Library + +Fix crashes in :class:`io.TextIOWrapper` and +:class:`io.IncrementalNewlineDecoder` when a callback changes the active +codec or decoded buffer. :class:`io.StringIO` now raises :exc:`ValueError` +if converting the size passed to ``truncate()`` closes the stream. + +.. + +.. date: 2026-10-22-08-34-22 +.. gh-issue: 160310 +.. nonce: q7Zm2K +.. section: Library + +Fix quadratic time complexity of :func:`gzip.decompress`, +:func:`bz2.decompress`, :func:`lzma.decompress` and +:func:`compression.zstd.decompress` on data consisting of many concatenated +members. + +.. + +.. date: 2026-10-22-08-18-31 +.. gh-issue: 160522 +.. nonce: cJkSt8 +.. section: Library + +Fix the incremental encoders and stream writers of the stateful CJK codecs +(``iso2022_*`` and ``hz``) keeping a changed shift state after an +:meth:`!encode` or :meth:`!write` call failed. The next call could omit the +escape sequence, writing multibyte characters in ASCII mode. + +.. + +.. date: 2026-10-22-08-17-51 +.. gh-issue: 160316 +.. nonce: sRe9pQ +.. section: Library + +Fix :meth:`re.Pattern.match`, :meth:`re.Pattern.prefixmatch`, +:meth:`re.Pattern.fullmatch` and the ``match()`` and ``prefixmatch()`` +methods of scanner objects returning an empty match, or reading past the end +of the string, when *pos* is greater than *endpos*. They now return +``None``, as documented. + +.. + +.. date: 2026-10-22-08-15-56 +.. gh-issue: 160324 +.. nonce: sIoRlz +.. section: Library + +Fix a crash or garbage data when an :class:`io.StringIO` object is used +again after one of its methods failed with :exc:`MemoryError`. The object +now keeps its contents. + +.. + +.. date: 2026-10-22-08-09-29 +.. gh-issue: 160406 +.. nonce: q7Rk2d +.. section: Library + +Fix in-place filtering in :mod:`fileinput` deleting the file when it cannot +be opened or the output file cannot be created: the backup file is now kept. +The output file is now created exclusively, so :exc:`FileExistsError` is +raised if a file or symbolic link appears under the name of the input file. + +.. + +.. date: 2026-10-22-08-06-39 +.. gh-issue: 160513 +.. nonce: q7RkZd +.. section: Library + +Fix :meth:`asyncio.loop.start_tls` and +:meth:`asyncio.StreamWriter.start_tls` not calling the protocol's +:meth:`~asyncio.BaseProtocol.connection_lost` when the connection is lost, +the handshake times out or the call is cancelled during the TLS handshake. +:meth:`asyncio.StreamWriter.wait_closed` no longer waits forever after such +a failure. + +.. + +.. date: 2026-10-22-08-04-26 +.. gh-issue: 160297 +.. nonce: hL4kQz +.. section: Library + +Fix :meth:`tarfile.TarFile.extractall` and :meth:`tarfile.TarFile.extract` +changing the mode, modification time and owner of a file outside the +destination directory when a hard link member is extracted as a copy of a +symbolic link member, for example with the ``'tar'`` filter. + +.. + +.. date: 2026-10-22-05-24-34 +.. gh-issue: 161086 +.. nonce: FGPjqA +.. section: Library + +Fix the :mod:`gzip` command-line interface ignoring :option:`gzip --fast` +and :option:`gzip --best` when compressing files given as arguments. + +.. + +.. date: 2026-10-22-05-24-21 +.. gh-issue: 161112 +.. nonce: BFrZRw +.. section: Library + +Fix :func:`shutil.make_archive` adding the zip archive being created to +itself when it is written inside the directory being archived or a symbolic +link in that directory refers to it. + +.. + +.. date: 2026-10-22-05-23-40 +.. gh-issue: 161088 +.. nonce: _-HEWz +.. section: Library + +Fix the :mod:`gzip` command-line interface writing through a symbolic link +at the output file name. Like :program:`gzip`, it now replaces an existing +output file with a new one that has the permission bits of the input file. + +.. + +.. date: 2026-10-22-05-21-00 +.. gh-issue: 161094 +.. nonce: MDjCNm +.. section: Library + +Fix :func:`zipfile.is_zipfile` raising :exc:`ValueError` instead of +returning ``False`` for a file-like object, such as :class:`io.BytesIO`, +whose end of central directory record places the central directory before +the start of the file. + +.. + +.. date: 2026-10-22-05-20-52 +.. gh-issue: 161096 +.. nonce: vkhfyy +.. section: Library + +Fix :meth:`zoneinfo.ZoneInfo.from_file` accepting TZif data with a negative +or oversized abbreviation block, which could use hundreds of times the file +size in memory; it now raises :exc:`ValueError`. Abbreviation indexes above +127 now also give the correct time zone abbreviation. + +.. + +.. date: 2026-10-22-05-20-44 +.. gh-issue: 161106 +.. nonce: t0ux5q +.. section: Library + +Fix :func:`plistlib.dump` and :func:`plistlib.dumps` with +:data:`~plistlib.FMT_BINARY` holding on to memory proportional to the size +of large arrays and dictionaries after the call returns. + +.. + +.. date: 2026-10-22-05-20-27 +.. gh-issue: 161098 +.. nonce: voMVnd +.. section: Library + +Fix :class:`multiprocessing.shared_memory.ShareableList` leaving its newly +created shared memory block behind when a value cannot be stored, such as an +integer outside the signed 64-bit range. + +.. + +.. date: 2026-10-22-05-20-18 +.. gh-issue: 161108 +.. nonce: xOVsRA +.. section: Library + +Fix quadratic time complexity of :meth:`!clear` and :meth:`!popitem` on +:class:`configparser.ConfigParser` objects and their section proxies. +Clearing a section no longer fails part way with +:exc:`~configparser.InterpolationError` on a value it removes. + +.. + +.. date: 2026-10-22-05-20-11 +.. gh-issue: 161110 +.. nonce: miKrSQ +.. section: Library + +Fix :meth:`~xml.dom.Document.getElementById` in :mod:`xml.dom.minidom` so +that it returns the first element in document order when several elements +have the same ID. Previously, the result could differ between calls on an +unmodified document. + +.. + +.. date: 2026-10-22-05-20-06 +.. gh-issue: 161100 +.. nonce: pLgnlA +.. section: Library + +Fix :attr:`!ipaddress.IPv6Interface.is_unspecified` returning ``False`` for +the IPv4-mapped address ``::ffff:0.0.0.0``. It now gives the same result as +:attr:`ipaddress.IPv6Address.is_unspecified`. + +.. + +.. date: 2026-10-22-05-20-02 +.. gh-issue: 161102 +.. nonce: gqDOIM7X +.. section: Library + +Fix the incremental decoder of the :mod:`idna ` codec +decoding some dots twice when its input starts with empty labels, for +example ``b".example.com"``. Reading a run of dots through +:class:`io.TextIOWrapper` no longer produces quadratically more text than +the input holds. + +.. + +.. date: 2026-10-22-05-20-02 +.. gh-issue: 161092 +.. nonce: OtC5l6 +.. section: Library + +Fix the ``uu`` codec decoder taking any line that starts with ``begin``, +such as ``beginning of the report``, as the header of the encoded data. Like +the :mod:`email` package, it now requires ``begin`` followed by an octal +mode and skips other lines as leading text. + +.. + +.. date: 2026-10-22-05-19-56 +.. gh-issue: 161090 +.. nonce: pxZ1IA +.. section: Library + +Fix the ``punycode`` codec accepting a delimiter (``-``) that is not +preceded by any basic code point. Such input, for example ``b'-80a1acny'``, +now raises :exc:`UnicodeDecodeError`, as required by :rfc:`3492`. + +.. + +.. date: 2026-10-22-03-55-33 +.. gh-issue: 159739 +.. nonce: ce26e1 +.. section: Library + +Fix :mod:`time` functions returning a negative year or failing to accept +their own output for large timestamps on platforms that support them. Avoid +reporting an incorrect year in errors raised by :mod:`datetime` for those +timestamps. + +.. + +.. date: 2026-10-22-03-55-33 +.. gh-issue: 159739 +.. nonce: b4c371 +.. section: Library + +Fix a crash when a :mod:`ctypes` structure subclass cannot be created due to +an allocation failure. + +.. + +.. date: 2026-10-22-03-55-33 +.. gh-issue: 159739 +.. nonce: a8bf30 +.. section: Library + +Fix clearing a :class:`pickle.Unpickler` memo so that the unpickler can be +reused for independent records. + +.. + +.. date: 2026-10-22-00-33-42 +.. gh-issue: 160171 +.. nonce: 11l4om +.. section: Library + +On the server side, :meth:`ssl.SSLSocket.get_unverified_chain` now returns +the client's certificate when the session was resumed from a session ticket, +instead of an empty list, and returns an empty list instead of raising +:exc:`AttributeError` when the client sent no certificate. + +.. + +.. date: 2026-10-22-00-32-51 +.. gh-issue: 160181 +.. nonce: nnEeg1 +.. section: Library + +Fix :meth:`io.BytesIO.getvalue` sometimes returning a :class:`bytes` object +with the hash of earlier contents after the buffer was written to, which +made :class:`dict` and :class:`set` lookups with it fail. + +.. + +.. date: 2026-10-22-00-30-04 +.. gh-issue: 160187 +.. nonce: 9dcj2B +.. section: Library + +Fix the ISO-2022 decoders (``iso2022_jp``, ``iso2022_kr`` and related) +accepting escape sequences that designate a character set to the wrong code +element (G0, G1 or G2). Such sequences now raise :exc:`UnicodeDecodeError` +instead of producing wrong text or :exc:`RuntimeError`. + +.. + +.. date: 2026-10-22-00-29-55 +.. gh-issue: 160185 +.. nonce: BPz40E +.. section: Library + +Fix :meth:`codecs.StreamWriter.reset` of the stateful CJK codecs (``hz`` and +the ``iso2022_*`` family) not writing the sequence that returns the output +to ASCII. ``seek(0)`` of :class:`codecs.StreamWriter`, +:class:`codecs.StreamReaderWriter` and :class:`codecs.StreamRecoder` now +resets the writer before moving the stream position. + +.. + +.. date: 2026-10-22-00-29-50 +.. gh-issue: 160176 +.. nonce: tkNst1 +.. section: Library + +Fix a crash in :mod:`tkinter` when a deeply nested or recursive list or +tuple is passed to Tcl, or a deeply nested Tcl list is converted to Python. +These now raise :exc:`ValueError` (beyond 1000 levels) and +:exc:`RecursionError` respectively. + +.. + +.. date: 2026-10-21-21-26-10 +.. gh-issue: 159739 +.. nonce: yBc6Ei +.. section: Library + +Fix a possible crash in :class:`abc.ABCMeta` instance checks when a +subclass-checking callback changes the instance’s class. + +.. + +.. date: 2026-10-21-21-26-10 +.. gh-issue: 159739 +.. nonce: rVWF2a +.. section: Library + +Fix the ``truncate`` method on :class:`io.BytesIO` when converting its size +argument closes the stream or exports its buffer. + +.. + +.. date: 2026-10-21-21-26-10 +.. gh-issue: 159739 +.. nonce: k8uuGe +.. section: Library + +Raise :exc:`TypeError` if a string subclass returns a non-string during +newline translation in :class:`io.TextIOWrapper`. Recheck the stream state +if the replacement callback closes, detaches or reinitializes the stream. + +.. + +.. date: 2026-10-21-21-26-10 +.. gh-issue: 159739 +.. nonce: ghGctt +.. section: Library + +Fix :class:`contextvars.ContextVar` returning a stale value after a +reentrant finalizer changes it during :meth:`~contextvars.ContextVar.reset`. + +.. + +.. date: 2026-10-21-21-26-10 +.. gh-issue: 159739 +.. nonce: b4ueJ7 +.. section: Library + +Detect recursive ``_as_parameter_`` chains for :mod:`ctypes` calls without +argument types. + +.. + +.. date: 2026-10-21-21-26-10 +.. gh-issue: 159739 +.. nonce: JIqZRY +.. section: Library + +Fix a possible crash when a list of :mod:`sqlite3` parameters is modified +concurrently on a free-threaded build. + +.. + +.. date: 2026-10-21-20-05-00 +.. gh-issue: 160104 +.. nonce: BUAL5C +.. section: Library + +Synchronized :class:`zipfile.Path` with :pypi:`zipp` 4.1.2. +:meth:`zipfile.Path.iterdir` now raises :exc:`NotADirectoryError` instead of +:exc:`ValueError` when the path does not reference a directory. Character +classes in :meth:`!zipfile.Path.glob` and :meth:`!zipfile.Path.rglob` +patterns now follow shell-style semantics, treating a leading ``!`` as +negation and ``^`` as a literal, and no longer match path separators. + +.. + +.. date: 2026-10-21-19-55-00 +.. gh-issue: 159739 +.. nonce: tloc01 +.. section: Library + +Avoid a crash when a :class:`threading.local` subclass finalizer accesses +attributes after construction fails. + +.. + +.. date: 2026-10-21-19-55-00 +.. gh-issue: 159739 +.. nonce: sque01 +.. section: Library + +Avoid a crash when a :class:`queue.SimpleQueue` subclass finalizer adds an +item after construction fails. + +.. + +.. date: 2026-10-21-19-40-00 +.. gh-issue: 100609 +.. nonce: THXgRC +.. section: Library + +``str()`` of a :class:`zipfile.Path` whose underlying zipfile has no +filename, such as one constructed from an :class:`io.BytesIO`, now renders +that filename as ``:zipfile:`` instead of raising :exc:`TypeError`. The root +of a zipfile is also no longer rendered with a trailing slash. Synchronized +with :pypi:`zipp` 3.23.1. + +.. + +.. date: 2026-10-21-19-38-33 +.. gh-issue: 89721 +.. nonce: uMpxRw +.. section: Library + +Add ``copy_dir_stat`` parameter to :func:`shutil.copytree`: set it to false +to skip copying permissions and times of directories with +:func:`shutil.copystat`, e.g. when copying a read-only tree. + +.. + +.. date: 2026-10-21-15-42-29 +.. gh-issue: 159875 +.. nonce: p6j3yW +.. section: Library + +Fix :exc:`AttributeError` from the methods of an :mod:`asyncio` SSL +transport, such as :meth:`~asyncio.WriteTransport.write` and +:meth:`~asyncio.BaseTransport.set_protocol`, when the transport is closed +after its connection was lost. This also affected the fallback of +:meth:`loop.sendfile() ` over SSL when the peer went +away mid-transfer. The SSL context is still released once the connection is +lost. + +.. + +.. date: 2026-10-21-15-10-00 +.. gh-issue: 160125 +.. nonce: pdbpkg +.. section: Library + +Support setting breakpoints with the :mod:`pdb` ``break`` and ``tbreak`` +commands using ``package.module.function`` names when the module has not +been imported in the current namespace. + +.. + +.. date: 2026-10-20-17-29-14 +.. gh-issue: 138641 +.. nonce: FN6qM0 +.. section: Library + +Ignore :func:`pdb.set_trace` when called directly at the :mod:`pdb` prompt. + +.. + +.. date: 2026-10-20-16-00-00 +.. gh-issue: 159868 +.. nonce: qLhYkP +.. section: Library + +Allow :meth:`!collections.UserString.startswith` and +:meth:`!collections.UserString.endswith` to accept a +:class:`~collections.UserString` instance as the prefix or suffix argument. + +.. + +.. date: 2026-10-20-12-30-00 +.. gh-issue: 159864 +.. nonce: 4FZ6MR +.. section: Library + +Fix a crash in :func:`syslog.openlog` under the free-threaded build when +:data:`sys.argv` is modified concurrently. + +.. + +.. date: 2026-10-20-00-12-05 +.. gh-issue: 159690 +.. nonce: vZgLP6 +.. section: Library + +Fix :class:`logging.handlers.TimedRotatingFileHandler` permanently stopping +rollover after a rollover failed or its backup file already existed, for +example at a DST change. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: eb8987 +.. section: Library + +Avoid undefined behavior when :func:`struct.pack_into` copies a bytearray +into overlapping output. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: 9ef89c +.. section: Library + +Fix allocation error handling in the fallback implementation of +``getaddrinfo()``. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: 5d4269 +.. section: Library + +Fix a double free if remote unwinder initialization runs out of memory. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: 4c9783 +.. section: Library + +Fix :meth:`io.IOBase.readline` returning more bytes than requested when a +stream supports :meth:`!peek` and returns short reads. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: 3660a6 +.. section: Library + +Fix multibyte encoders silently consuming a non-BMP character when looking +up a two-character mapping. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: 35942a +.. section: Library + +Improve performance when parsing email address groups with many members. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: 1e4599 +.. section: Library + +Fix off-by-one output buffer checks in the fallback implementation of +``getnameinfo()``. + +.. + +.. date: 2026-10-20-00-00-00 +.. gh-issue: 159739 +.. nonce: 0c4ae9 +.. section: Library + +Fix reference cycles involving the types of grapheme iterators and segments +created by :func:`unicodedata.iter_graphemes`. + +.. + +.. date: 2026-10-19-23-25-29 +.. gh-issue: 159811 +.. nonce: cMB4DO +.. section: Library + +Fix a crash in :mod:`hashlib` when the OpenSSL routine used by +:meth:`hash.update() ` or by hash object constructors +fails on data larger than 2048 bytes. Patch by Bénédikt Tran. + +.. + +.. date: 2026-10-19-23-25-29 +.. gh-issue: 159811 +.. nonce: P6yy0z +.. section: Library + +Fix a crash in :mod:`hmac` when the OpenSSL routine used by +:meth:`hmac.HMAC.update` or by :func:`hmac.new` fails on data larger than +2048 bytes. Patch by Bénédikt Tran. + +.. + +.. date: 2026-10-19-21-20-28 +.. gh-issue: 159786 +.. nonce: fThehQ +.. section: Library + +Fix the error handling of the HACL* routine used by :func:`hmac.new` and +:meth:`hmac.HMAC.update`. If the maximum input length of the underlying hash +function is exceeded, an :exc:`OverflowError` is now raised on all builds, +instead of crashing on DEBUG builds and being silently ignored otherwise. +Code correctness. Realistically this can only happen at implausible sizes. +Patch by Bénédikt Tran. + +.. + +.. date: 2026-10-19-20-38-17 +.. gh-issue: 159739 +.. nonce: ff088c +.. section: Library + +Fix copying :class:`collections.deque` subclasses with a large maximum +length. + +.. + +.. date: 2026-10-19-20-38-17 +.. gh-issue: 159739 +.. nonce: bc65da +.. section: Library + +Allow cyclic garbage collection to reclaim group records returned by +:mod:`grp`. + +.. + +.. date: 2026-10-19-20-38-17 +.. gh-issue: 159739 +.. nonce: a81270 +.. section: Library + +Fix :class:`io.BytesIO` reads into overlapping views of the same buffer. + +.. + +.. date: 2026-10-19-20-38-17 +.. gh-issue: 159739 +.. nonce: 5a6e94 +.. section: Library + +Avoid a crash when memory allocation fails while creating a +:class:`collections.OrderedDict` items iterator. + +.. + +.. date: 2026-10-19-20-38-17 +.. gh-issue: 159739 +.. nonce: 243aa8 +.. section: Library + +Avoid reading past the end of an encoding name when the ``surrogatepass`` +error handler receives a truncated UTF-16 or UTF-32 name. + +.. + +.. date: 2026-10-19-20-38-17 +.. gh-issue: 159739 +.. nonce: 08caa7 +.. section: Library + +Fix incorrect :mod:`ctypes` scalar values after :func:`ctypes.resize` +enlarges the object. + +.. + +.. date: 2026-10-19-19-33-35 +.. gh-issue: 159739 +.. nonce: b5fbca +.. section: Library + +Fix a memory leak when Unicode normalization cannot grow its temporary +buffer. + +.. + +.. date: 2026-10-19-19-33-35 +.. gh-issue: 159739 +.. nonce: a1c3c7 +.. section: Library + +Fix resource leaks in :mod:`ssl` when certificate conversion or key log +setup fails. + +.. + +.. date: 2026-10-19-16-49-09 +.. gh-issue: 159706 +.. nonce: ushGrx +.. section: Library + +Fix the pure-Python :class:`zoneinfo.ZoneInfo` setting ``fold=0`` in +:meth:`~datetime.tzinfo.fromutc` for times in the fold created by the only +transition of a zone with a single transition. + +.. + +.. date: 2026-10-19-16-26-57 +.. gh-issue: 71990 +.. nonce: K3FlJ9 +.. section: Library + +:mod:`ctypes`: objects with an :attr:`!_as_parameter_` attribute are now +implicitly converted when passed for a :func:`~ctypes.POINTER` argument. + +.. + +.. date: 2026-10-19-16-00-00 +.. gh-issue: 139122 +.. nonce: bYtEs1 +.. section: Library + +:class:`uuid.UUID` now consistently raises :exc:`TypeError` when the *bytes* +or *bytes_le* argument is not a :class:`bytes` object. Previously, some +invalid inputs raised :exc:`AssertionError` or :exc:`ValueError`. + +.. + +.. date: 2026-10-19-12-00-00 +.. gh-issue: 78327 +.. nonce: Qk3hzA +.. section: Library + +:mod:`ctypes` data types now set :meth:`~object.__hash__` to ``None``, so +:class:`collections.abc.Hashable` checks report them correctly. + +.. + +.. date: 2026-10-19-12-00-00 +.. gh-issue: 159714 +.. nonce: j59fix +.. section: Library + +Fix :class:`zoneinfo.ZoneInfo` interpreting the Julian day rule ``J59`` as +February 29 in leap years instead of February 28. + +.. + +.. date: 2026-10-19-00-00-00 +.. gh-issue: 149015 +.. nonce: bdb +.. section: Library + +Fix :meth:`bdb.Bdb.clear_all_file_breaks` leaving breakpoints behind when +multiple breakpoints are set on the same line. + +.. + +.. date: 2026-10-18-16-09-24 +.. gh-issue: 159684 +.. nonce: VHVYjVuE +.. section: Library + +Update :mod:`configparser` and its tests to modern linter standards: +printf-style and :meth:`str.format` call sites are now f-strings, and a +handful of constructs have simpler spellings. No behavior change. + +.. + +.. date: 2026-10-18-12-20-00 +.. gh-issue: 159465 +.. nonce: partial-escapes +.. section: Library + +Preserve :exc:`SyntaxError` for malformed escapes in unfinished f-strings +and t-strings passed to :func:`codeop.compile_command`. + +.. + +.. date: 2026-10-18-08-30-10 +.. gh-issue: 155976 +.. nonce: feSum4_j +.. section: Library + +:func:`curses.set_term` and :meth:`curses.screen.use` now update +:data:`curses.LINES`, :data:`curses.COLS`, :data:`curses.COLORS` and +:data:`curses.COLOR_PAIRS` for the current screen. + +.. + +.. date: 2026-10-17-19-41-12 +.. gh-issue: 157210 +.. nonce: o9hycSeJ +.. section: Library + +Fix entering a combining character in the lower-right cell of +:class:`curses.textpad.Textbox`: it no longer takes a cell of its own, +pushing its base character off the window. In insert mode, a combining +character no longer shifts the following text. + +.. + +.. date: 2026-10-17-19-33-13 +.. gh-issue: 159643 +.. nonce: EJfe0IXetV +.. section: Library + +Fix :exc:`EOFError` raised by :mod:`tarfile` in stream mode with bz2, xz or +zstd compression when reading past the end of the compressed data, for +example with *ignore_zeros* or an archive without end-of-archive blocks. + +.. + +.. date: 2026-10-17-15-22-16 +.. gh-issue: 159620 +.. nonce: Ynsu5L +.. section: Library + +Fix ``python -m asyncio ps``, ``pstree`` and async-aware +:mod:`profiling.sampling` stacks stopping at asynchronous generators. Patch +by Timofei Ivankov. + +.. + +.. date: 2026-10-16-18-23-03 +.. gh-issue: 159558 +.. nonce: ctRelk +.. section: Library + +Speed up assignment to the ``value`` attribute of simple :mod:`ctypes` +types, to the ``contents`` attribute of :mod:`ctypes` pointers and to +:mod:`ctypes` structure fields, array items and pointer items on the +free-threaded build by not re-entering the critical section already held on +the object. + +.. + +.. date: 2026-10-16-13-00-00 +.. gh-issue: 159435 +.. nonce: OMNI +.. section: Library + +Fix an offset calculation in :mod:`tarfile` when a PAX header with a +``size`` record is followed by a GNU long-name header. Such archives could +duplicate or skip members while being read. + +.. + +.. date: 2026-10-16-10-25-22 +.. gh-issue: 159458 +.. nonce: q02EnY +.. section: Library + +Fix a ThreadSanitizer data race on the static cache in +``normalize_century()`` of the datetime module under the free-threaded +build. + +.. + +.. date: 2026-10-16-01-56-15 +.. gh-issue: 159503 +.. nonce: WgYiow +.. section: Library + +Fix quadratic copying in :mod:`asyncio` selector socket transports on +partial sends. A :class:`bytearray` passed to +:meth:`~asyncio.WriteTransport.write` now stays locked against resizing +until it has been sent. + +.. + +.. date: 2026-10-15-18-38-22 +.. gh-issue: 159500 +.. nonce: OjAQ8W +.. section: Library + +Speed up :meth:`asyncio.StreamReader.readline` and +:meth:`asyncio.StreamReader.readuntil` when the data is already buffered, +and stop :meth:`~asyncio.StreamReader.readuntil` with a tuple of separators +from scanning the whole buffer on every call. + +.. + +.. date: 2026-10-15-13-44-03 +.. gh-issue: 159465 +.. nonce: Ph-m8w +.. section: Library + +Improve incomplete input detection in :mod:`codeop` without an extra parser +pass, and preserve syntax and compiler errors. + +.. + +.. date: 2026-10-14-14-47-05 +.. gh-issue: 81954 +.. nonce: KvM0gY +.. section: Library + +Fix :class:`zipfile.ZipFile` not being closed at interpreter shutdown when +emitting its :exc:`ResourceWarning` failed with an :exc:`ImportError`. + +.. + +.. date: 2026-10-14-11-41-00 +.. gh-issue: 159430 +.. nonce: ZEW-mx +.. section: Library + +Report the correct exception if looking up ``__name__`` raises an exception +when calling :func:`repr` on a :class:`weakref.ref`. + +.. + +.. date: 2026-10-14-10-26-10 +.. gh-issue: 159387 +.. nonce: xMbQbB +.. section: Library + +Fix :mod:`ssl` reporting TLS 1.0, 1.1, and 1.2 as unavailable when built +with OpenSSL 4.0, despite support for these protocol versions. + +.. + +.. date: 2026-10-14-08-50-00 +.. gh-issue: 159398 +.. nonce: k2Xd7q +.. section: Library + +Fix the sampling profiler crashing when a string read from the target +process contains code points above ``U+10FFFF``. + +.. + +.. date: 2026-10-13-19-10-00 +.. gh-issue: 159322 +.. nonce: hP6k3A +.. section: Library + +Fix :mod:`tarfile` to NUL-terminate user and group names in PAX archive +headers and use extended headers for names longer than 31 characters. + +.. + +.. date: 2026-10-13-12-00-00 +.. gh-issue: 159304 +.. nonce: mFbYq2 +.. section: Library + +Fix :exc:`AttributeError` in :meth:`email.message.EmailMessage.get_body` +when a nested ``multipart`` part is malformed and its payload is a string. + +.. + +.. date: 2026-10-12-21-20-00 +.. gh-issue: 159225 +.. nonce: T8jlor +.. section: Library + +Prevent ``input`` calls in PyREPL from modifying readline history. + +.. + +.. date: 2026-10-12-18-21-06 +.. gh-issue: 159240 +.. nonce: a1b2c3 +.. section: Library + +Deprecate, and schedule for removal in Python 3.20, the ``__author__``, +``__credits__``, ``__copyright__`` and ``__date__`` attributes of +:mod:`bz2`, :mod:`concurrent.futures`, :mod:`inspect`, :mod:`io`, +:mod:`json`, :mod:`logging`, :mod:`optparse`, :mod:`platform`, :mod:`pydoc`, +:mod:`tarfile`, :mod:`tkinter.ttk`, :mod:`tokenize` and :mod:`uuid`. + +.. + +.. date: 2026-10-12-13-05-00 +.. gh-issue: 158501 +.. nonce: n7Qp3k +.. section: Library + +:meth:`datetime.time.fromisoformat` and +:meth:`datetime.datetime.fromisoformat` now reject stray text between the +time and a UTC offset. The C implementation previously accepted strings that +the pure-Python implementation rejected. + +.. + +.. date: 2026-10-12-12-00-00 +.. gh-issue: 72113 +.. nonce: njkm8v +.. section: Library + +Speed up creating a :mod:`ctypes` array from initializers, such as ``(c_int +* 3)(1, 2, 3)``, by roughly 3x. + +.. + +.. date: 2026-10-12-11-36-48 +.. gh-issue: 158856 +.. nonce: jLR-Lv +.. section: Library + +Improve large Enum creation performance. + +.. + +.. date: 2026-10-12-00-00-00 +.. gh-issue: 159044 +.. nonce: Tr8kXv +.. section: Library + +Restore the previous behaviour of +:meth:`http.client.HTTPConnection.set_tunnel` where header values that are +not strings (such as :const:`None`, :class:`int` or other objects) are +converted with :class:`str` instead of raising :exc:`AttributeError` when +the request is sent. + +.. + +.. date: 2026-10-11-13-40-44 +.. gh-issue: 159168 +.. nonce: z3KjB8 +.. section: Library + +Fix :exc:`RecursionError` in ``python -m asyncio pstree`` on a deep chain of +awaiting tasks. Patch by Timofei Ivankov. + +.. + +.. date: 2026-10-11-13-08-07 +.. gh-issue: 88037 +.. nonce: CzKcxh +.. section: Library + +:func:`urllib.parse.urlsplit` and :func:`urllib.parse.urlparse` now raise +:exc:`ValueError` for an invalid port when the URL is parsed, instead of +when the :attr:`!port` attribute is read. Patch by Miguel Brito and Senthil +Kumaran. + +.. + +.. date: 2026-10-11-08-42-00 +.. gh-issue: 159184 +.. nonce: Tq2nSe +.. section: Library + +The output of :func:`marshal.dumps` no longer depends on the order of the +elements of a set which have the same marshalled form, such as NaNs which +are distinct objects. + +.. + +.. date: 2026-10-10-22-29-13 +.. gh-issue: 159049 +.. nonce: eJUiKL +.. section: Library + +Fix reference cycles in :meth:`asyncio.loop.create_connection` and +:meth:`asyncio.loop.start_tls` after a failed TLS handshake. Patch by +Timofei Ivankov. + +.. + +.. date: 2026-10-10-22-14-22 +.. gh-issue: 150751 +.. nonce: JInuGL +.. section: Library + +:mod:`http.client` now ignores optional whitespace around the +``Content-Length`` header value when validating it, as required by +:rfc:`9112`. Previously a value with trailing whitespace (for example +``Content-Length: 5`` followed by a space) was treated as missing and the +response body was read until the connection was closed. + +.. + +.. date: 2026-10-08-21-02-38 +.. gh-issue: 158998 +.. nonce: DZYJI7 +.. section: Library + +Fix :func:`os.statx` ignoring the *follow_symlinks* argument. + +.. + +.. date: 2026-10-08-11-58-52 +.. gh-issue: 158628 +.. nonce: NJxwuw +.. section: Library + +Implement :data:`sys._wasi_info` for exposing WASI-specific system details. + +.. + +.. date: 2026-10-07-17-30-00 +.. gh-issue: 158287 +.. nonce: xY9zQp +.. section: Library + +Fix a bug in :mod:`asyncio` where stopping a child process (such as with +``SIGSTOP``) on macOS caused the event loop to block indefinitely. + +.. + +.. date: 2026-10-07-10-59-27 +.. gh-issue: 155233 +.. nonce: YGuQGK +.. section: Library + +Fix :meth:`asyncio.Barrier.wait` returning duplicate indices when a waiting +task is cancelled before the barrier is released.``` + +.. + +.. date: 2026-10-05-14-58-12 +.. gh-issue: 158839 +.. nonce: wYd6fI +.. section: Library + +The log functions in the :mod:`logging` module and the +:class:`logging.Logger` class now normalize falsy ``exc_info`` arguments to +``None`` before passing them to :meth:`logging.Logger.makeRecord` and +subsequently to :class:`logging.LogRecord`. This aligns +``logging.LogRecord.exc_info`` with the documentation. + +.. + +.. date: 2026-10-05-08-35-10 +.. gh-issue: 130698 +.. nonce: iibO9V +.. section: Library + +Fix PyREPL to convert :data:`sys.ps1` and :data:`sys.ps2` to strings before +use, and fall back to the default prompts if the conversion raises. + +.. + +.. date: 2026-09-29-19-34-30 +.. gh-issue: 158192 +.. nonce: 6wPjiU +.. section: Library + +Expose the future being waited on by a ``FutureIter`` as ``fi_future``. + +.. + +.. date: 2026-09-28-00-00-00 +.. gh-issue: 158370 +.. nonce: ctxmgr +.. section: Library + +:class:`concurrent.interpreters.Interpreter` objects now support the +:term:`context manager` protocol. + +.. + +.. date: 2026-09-27-14-00-00 +.. gh-issue: 158277 +.. nonce: aBcDeF +.. section: Library + +Fix :mod:`sqlite3` CLI completer to include view columns in column-name +completion. + +.. + +.. date: 2026-09-26-05-18-21 +.. gh-issue: 144650 +.. nonce: zqHLOW +.. section: Library + +Use Argument Clinic vectorcall support for the :class:`decimal.Decimal` +type. This offers 1.10x speedup on creation of small instances. + +.. + +.. date: 2026-09-25-17-04-45 +.. gh-issue: 158159 +.. nonce: lbEalG +.. section: Library + +Fix a use-after-free in :meth:`asyncio.Task.get_name` when formatting the +default task name triggers a garbage collection whose finalizers call +:meth:`~asyncio.Task.get_name` on the same task. + +.. + +.. date: 2026-09-25-01-29-55 +.. gh-issue: 158126 +.. nonce: nWmFnq +.. section: Library + +Speed up :func:`pickle.dumps` and creating a :class:`pickle.Pickler` by +avoiding the creation of an :exc:`AttributeError` when looking up the +optional ``reducer_override`` and ``dispatch_table`` attributes. + +.. + +.. date: 2026-09-24-14-59-08 +.. gh-issue: 158095 +.. nonce: lcKOT4 +.. section: Library + +Fix a memory leak in :func:`re.finditer` and regular expression scanners +when their input participates in a reference cycle. + +.. + +.. date: 2026-09-23-23-34-30 +.. gh-issue: 144812 +.. nonce: ujptbVOI +.. section: Library + +Make :class:`argparse.ArgumentGroup` and +:class:`argparse.MutuallyExclusiveGroup` public. The private +``_ArgumentGroup`` and ``_MutuallyExclusiveGroup`` names remain as aliases. + +.. + +.. date: 2026-09-23-19-59-03 +.. gh-issue: 158004 +.. nonce: p68swW +.. section: Library + +Fix :func:`asyncio.as_completed` to continue yielding every awaitable after +a pending iteration step is cancelled. + +.. + +.. date: 2026-09-23-09-21-25 +.. gh-issue: 89977 +.. nonce: fmia18 +.. section: Library + +Avoid releasing the GIL for :meth:`socket.socket.setblocking`, +:meth:`socket.socket.getsockname`, :meth:`socket.socket.getpeername`, and +:func:`socket.gethostname`. + +.. + +.. date: 2026-09-21-21-07-12 +.. gh-issue: 157928 +.. nonce: ZPlNYC +.. section: Library + +Correctly detect and handle OSC sequences in PyREPL. + +.. + +.. date: 2026-09-21-14-41-07 +.. gh-issue: 157899 +.. nonce: s6vBQC +.. section: Library + +:func:`os.readlink` now accepts a file descriptor referring to a symlink on +Linux, Android, macOS and iOS. + +.. + +.. date: 2026-09-20-21-24-05 +.. gh-issue: 157880 +.. nonce: aE7wIu +.. section: Library + +Fix :class:`mailbox.Babyl` to write the visible headers of a +:class:`mailbox.BabylMessage` to the mailbox file. + +.. + +.. date: 2026-09-20-12-00-00 +.. gh-issue: 157856 +.. nonce: wq4xuF +.. section: Library + +Fix :func:`asyncio.as_completed` raising :exc:`RuntimeError` ("no running +event loop") when the iterator is created outside of a running event loop +and driven with :meth:`loop.run_until_complete() +`. + +.. + +.. date: 2026-09-18-00-00-00 +.. gh-issue: 49960 +.. nonce: MDT3jE +.. section: Library + +:mod:`ctypes` callback functions can now return structures and unions by +value. Previously only "simple" types could be returned, and any other +result type raised :exc:`TypeError`. + +.. + +.. date: 2026-09-17-20-36-01 +.. gh-issue: 157290 +.. nonce: NfFmt +.. section: Library + +Add the ``%Nf`` format code (where *N* is from 1 to 6) to +:meth:`datetime.datetime.strftime`, :meth:`datetime.time.strftime`, +:meth:`datetime.date.strftime`, :meth:`datetime.datetime.strptime` and +:meth:`datetime.time.strptime`. + +.. + +.. date: 2026-09-17-10-30-00 +.. gh-issue: 158252 +.. nonce: strptime +.. section: Library + +Speed up :func:`time.strptime`, :meth:`datetime.date.strptime`, +:meth:`datetime.time.strptime`, and :meth:`datetime.datetime.strptime` for +common numeric formats by parsing fields and constructing the result in C. +Other inputs continue to use the Python parser. + +.. + +.. date: 2026-09-05-17-29-50 +.. gh-issue: 160837 +.. nonce: W6kHva +.. section: Library + +:func:`asyncio.print_call_graph` no longer truncates the call stack of a +task suspended inside an asynchronous generator. + +.. + +.. date: 2026-09-01-00-00-00 +.. gh-issue: 156784 +.. nonce: 8gvtt4 +.. section: Library + +:class:`asyncio.ProactorEventLoop` no longer silently ignores +:exc:`ConnectionResetError` on UDP sockets, matching +:class:`asyncio.SelectorEventLoop`. + +.. + +.. date: 2026-08-29-14-28-46 +.. gh-issue: 156568 +.. nonce: mkxLY8 +.. section: Library + +:func:`zipapp.create_archive` now correctly sets the executable bit on the +target archive when copying a zipapp where the target is a path-like object. + +.. + +.. date: 2026-08-29-13-41-56 +.. gh-issue: 154675 +.. nonce: cYvE-Z +.. section: Library + +Improve import time of :mod:`pdb`, :mod:`bdb` and :mod:`code` by lazily +importing several dependencies. + +.. + +.. date: 2026-08-23-16-33-00 +.. gh-issue: 151126 +.. nonce: GUvtG7 +.. section: Library + +Fix a crash caused by failing to set :exc:`MemoryError` on allocation +failure when first using a Chinese, Japanese or Korean codec. + +.. + +.. date: 2026-08-21-06-00-00 +.. gh-issue: 156057 +.. nonce: tarfile-wrap-comp-err +.. section: Library + +Wrap the underlying compression library's exception in +:exc:`tarfile.ReadError` when a member's compressed data is corrupt past its +header, for all of :mod:`tarfile`'s supported compression formats (gzip, +bzip2, lzma, zstd). Previously the raw exception (e.g. :exc:`zlib.error`) +could leak through :meth:`tarfile.TarFile.extractfile`. Patch by Jason Mak. + +.. + +.. date: 2026-08-20-09-56-45 +.. gh-issue: 156078 +.. nonce: -E7gZc +.. section: Library + +:meth:`zipfile.ZipFile.mkdir` now raises :exc:`ValueError` when the archive +is closed or when a writable member handle is open. + +.. + +.. date: 2026-08-15-12-08-33 +.. gh-issue: 155833 +.. nonce: Pi2Yjp +.. section: Library + +Ensure that MD5 and SHA* objects are properly GC-untracked before freeing +their internal HACL* state. Patch by Bénédikt Tran. + +.. + +.. date: 2026-08-15-11-49-30 +.. gh-issue: 155830 +.. nonce: cY5xgK +.. section: Library + +:mod:`!_blake2`: Ensure that BLAKE-2 objects are correctly initialized +before being tracked by the garbage collector. Patch by Bénédikt Tran. + +.. + +.. date: 2026-08-15-11-30-42 +.. gh-issue: 155782 +.. nonce: JUmDP4 +.. section: Library + +:mod:`ssl`: prevent a crash in :meth:`~ssl.SSLSocket.group` for session-less +objects on OpenSSL 3.2 and later. Patch by Bénédikt Tran. + +.. + +.. date: 2026-08-09-14-26-48 +.. gh-issue: 155428 +.. nonce: wi0_fI +.. section: Library + +Ensure that :func:`hmac.digest` and :meth:`hmac.HMAC.digest` output +identical values when using a digest's block size smaller than 16. Patch by +Bénédikt Tran. + +.. + +.. date: 2026-08-09-14-06-27 +.. gh-issue: 155426 +.. nonce: zZVPMd +.. section: Library + +Make sure that the pure Python fallback of :func:`hmac.digest` reject keys +that are neither :class:`bytes` or :class:`bytearray`. Previously, an +:exc:`AttributeError` was raised instead of :exc:`TypeError`. Patch by +Bénédikt Tran. + +.. + +.. date: 2026-08-05-10-13-23 +.. gh-issue: 153711 +.. nonce: 9g9dVz +.. section: Library + +On iOS, add :func:`os.pipe2`, and use the syscalls :manpage:`pipe2 (2)` and +:manpage:`dup3 (2)` internally, when available at both build-time and +run-time. + +.. + +.. date: 2026-08-04-16-38-59 +.. gh-issue: 155152 +.. nonce: qN1EEN +.. section: Library + +Improve performance of some :mod:`zipimport` archives that use Zstandard +compression. + +.. + +.. date: 2026-07-21-00-02-11 +.. gh-issue: 148228 +.. nonce: D3rP8m +.. section: Library + +The :term:`REPL` now displays error messages when editing commands cannot be +performed, such as when tab completion finds no matches or a history search +fails. Patch by Bartosz Sławecki. + +.. + +.. date: 2026-07-16-21-30-00 +.. gh-issue: 74876 +.. nonce: wR3fXk +.. section: Library + +Weak references (:class:`weakref.ref`), :class:`weakref.WeakMethod`, +:class:`weakref.WeakSet`, :class:`weakref.WeakValueDictionary` and +:class:`weakref.WeakKeyDictionary` can now be pickled if the weakly +referenced objects are strongly referenced elsewhere in the same pickle; +otherwise pickling raises :exc:`pickle.PicklingError`. +:class:`weakref.WeakMethod` and :class:`weakref.WeakSet` can now also be +copied. + +.. + +.. date: 2026-07-13-00-10-00 +.. gh-issue: 153679 +.. nonce: RJ2WGB +.. section: Library + +:class:`zipimport.zipimporter` now raises :exc:`zipimport.ZipImportError` +instead of :exc:`UnicodeDecodeError` for a UTF-8-flagged file name that is +not valid UTF-8. Patch by tonghuaroot. + +.. + +.. date: 2026-07-11-16-00-00 +.. gh-issue: 153605 +.. nonce: Rk7Xz3 +.. section: Library + +Fix :class:`gettext.GNUTranslations` raising :exc:`struct.error` instead of +:exc:`OSError` when reading a malformed ``.mo`` file. Patch by tonghuaroot. + +.. + +.. date: 2026-07-04-12-52-22 +.. gh-issue: 149816 +.. nonce: XQOutB +.. section: Library + +:mod:`hashlib`: fix a crash arising from a race condition when looking up an +OpenSSL hash algorithm by name the first time. Patch by Bénédikt Tran. + +.. + +.. date: 2026-06-30-08-39-07 +.. gh-issue: 62222 +.. nonce: Zq7x2P +.. section: Library + +Fix :func:`binascii.a2b_qp` and :mod:`quopri` discarding the character after +a ``=`` that does not start a valid escape: ``b'==41'`` now decodes to +``b'=A'``, as in the :mod:`email` package. Add a keyword-only *strip_ws* +parameter to :func:`binascii.a2b_qp`, :func:`quopri.decode` and +:func:`quopri.decodestring` to strip trailing whitespace from each line, as +required by :rfc:`2045`. + +.. + +.. date: 2026-06-28-14-37-01 +.. gh-issue: 152490 +.. nonce: A8qLbF +.. section: Library + +Fix :exc:`TypeError` when formatting a :exc:`SyntaxError` raised with +:data:`NotImplemented` as its argument, which caused unexpected REPL exit +and IDLE Shell restart. + +.. + +.. date: 2026-06-27-08-31-30 +.. gh-issue: 152373 +.. nonce: 3cd0aa +.. section: Library + +Fix the :mod:`pdb` ``until`` command not stepping over a list comprehension +when a breakpoint is set on it. Patch by tonghuaroot. + +.. + +.. date: 2026-06-18-11-26-00 +.. gh-issue: 151633 +.. nonce: uJzHdc +.. section: Library + +Fix a use-after-free under free-threading when multiple threads update the +same :class:`~collections.Counter` concurrently. + +.. + +.. date: 2026-05-21-16-15-01 +.. gh-issue: 105578 +.. nonce: oq-0pg +.. section: Library + +Deprecate :data:`typing.AnyStr` type variable and raise a warning when it is +imported. + +.. + +.. date: 2026-05-09-12-58-04 +.. gh-issue: 149527 +.. nonce: hX36Yh +.. section: Library + +:mod:`multiprocessing`: fix creation of temporary socket files following +:gh:`137335` (`PR#148578 `__ +and corresponding backports). Patch by Bénédikt Tran. + +.. + +.. date: 2026-05-03-22-52-07 +.. gh-issue: 149309 +.. nonce: gXhk9Y +.. section: Library + +:exc:`bdb.BdbQuit` is now a subclass of :exc:`BaseException`, instead of +:exc:`Exception`, to allow it to be used for cleanly exiting the debugger +without being accidentally caught by user code. + +.. + +.. date: 2026-04-27-20-30-16 +.. gh-issue: 149083 +.. nonce: xcmpSO +.. section: Library + +:data:`functools.Placeholder` is now a :class:`sentinel`. It is no longer an +instance of its own singleton type. + +.. + +.. date: 2026-03-28-14-34-18 +.. gh-issue: 143376 +.. nonce: FvWTwi +.. section: Library + +:mod:`concurrent.interpreters`: fix a crash when internal cross-interpreter +channel ends have incorrect channel IDs. Patch by Bénédikt Tran. + +.. + +.. date: 2026-03-28-10-27-46 +.. gh-issue: 146169 +.. nonce: RBF1xp +.. section: Library + +:mod:`xml.parsers.expat`: raise :exc:`RuntimeError` when an Expat handler +calls :meth:`parser.Parse ` or +:meth:`parser.ParseFile ` on the +parser that called the handler. Patch by Bénédikt Tran. + +.. + +.. date: 2026-03-23-00-04-13 +.. gh-issue: 146065 +.. nonce: FIdn8D +.. section: Library + +Fix a crash in asyncio.Future iterator when send() is called after throw() +or close(). + +.. + +.. date: 2026-01-06-09-13-53 +.. gh-issue: 84644 +.. nonce: V_cYP3 +.. section: Library + +Deprecate using a return annotation to infer the dispatch type for +:func:`functools.singledispatch` and +:class:`functools.singledispatchmethod`. It will raise :exc:`TypeError` in +Python 3.21. Annotate the dispatch parameter or pass the dispatch type +explicitly instead. Patch by Bartosz Sławecki. + +.. + +.. date: 2025-10-20-13-08-04 +.. gh-issue: 84027 +.. nonce: HjBO0B +.. section: Library + +:mod:`shutil` now can unpack wheel packages using the ``whl`` format. + +.. + +.. date: 2025-10-13-15-43-09 +.. gh-issue: 140036 +.. nonce: b_59uN +.. section: Library + +Avoid hanging in floor division of pure Python :class:`decimal.Decimal` +instances when the context precision is very large. Patch by Bénédikt Tran. + +.. + +.. date: 2025-09-18-18-03-32 +.. gh-issue: 139105 +.. nonce: 1QQ_W4 +.. section: Library + +Correct the EOF key shown in the :mod:`sqlite3` CLI ``.help`` text on +Windows. + +.. + +.. date: 2025-09-18-14-13-00 +.. gh-issue: 139122 +.. nonce: m3lp66 +.. section: Library + +:mod:`uuid`: Reimplement :class:`~uuid.UUID` objects, :func:`~uuid.uuid4`, +and :func:`~uuid.uuid7` in C. + +.. + +.. date: 2025-06-12-15-39-48 +.. gh-issue: 89730 +.. nonce: Lu35Fu +.. section: Library + +Add support for Encrypted Client Hello (ECH) to the :mod:`ssl` module with +OpenSSL 4.0 or later. + +.. + +.. date: 2025-03-13-20-21-00 +.. gh-issue: 115810 +.. nonce: qwrb231 +.. section: Library + +Add :attr:`datetime.timezone.name`, the *name* passed to the constructor, or +``None`` if no name was passed. + +.. + +.. date: 2025-02-04-09-01-06 +.. gh-issue: 129605 +.. nonce: wKGNXQ +.. section: Library + +Improve exception handling when computing suggestions in traceback.py. This +fixes a bug that could cause PyREPL to exit unexpectedly. + +.. + +.. date: 2024-12-27-15-41-55 +.. gh-issue: 128289 +.. nonce: -hCzEc +.. section: Library + +Add *eager_tasks* parameter to :func:`asyncio.run` function and +:class:`asyncio.Runner` class constructor. + +.. + +.. date: 2023-01-26-09-36-24 +.. gh-issue: 61448 +.. nonce: SOAkvt +.. section: Library + +Fix :func:`inspect.formatargvalues` raising :exc:`KeyError` for deleted +arguments, including ``*args`` and ``**kwargs``, by displaying +````. Patch by Ezio Melotti and Furkan Onder. + +.. + +.. bpo: 19094 +.. date: 2021-06-11-20-00-16 +.. nonce: rMRoIL +.. section: Library + +The :mod:`urllib.parse` URL parsing functions now raise :exc:`TypeError` +instead of :exc:`AttributeError` when passed an object which is not a +string, bytes, or ``None``. Accepting objects with false values (like ``0`` +and ``[]``) except empty strings, byte-like objects and ``None`` is now +deprecated, extending the deprecation made in Python 3.14 for +:func:`~urllib.parse.parse_qsl` and :func:`~urllib.parse.parse_qs`. + +.. + +.. bpo: 44170 +.. date: 2021-05-24-14-41-29 +.. nonce: _v28mv +.. section: Library + +Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList. + +.. + +.. date: 2025-08-31-02-28-21 +.. gh-issue: 138277 +.. nonce: kkORph +.. section: Documentation + +Rearranged documentation search results to emphasize built-in types and +functions. Patch by Adam Hartz. + +.. + +.. date: 2023-09-04-16-01-20 +.. gh-issue: 107689 +.. nonce: LFLFtG +.. section: Documentation + +Added docstrings to abstract ctypes. + +.. + +.. date: 2026-11-07-01-45-00 +.. gh-issue: 160334 +.. nonce: F2n8qA +.. section: Tests + +Fix the sampling profiler integration test for special frames without file +and line information. + +.. + +.. date: 2026-11-06-23-15-23 +.. gh-issue: 161517 +.. nonce: LkAlDt +.. section: Tests + +Fix test_gdb when Python is built out-of-tree and Python is run from a +virtual environment. Previously, test_gdb was just skipped. Patch by Victor +Stinner. + +.. + +.. date: 2026-11-05-16-20-23 +.. gh-issue: 161430 +.. nonce: M7b8N6 +.. section: Tests + +No longer list tests expected to be skipped in the final report. Patch by +Victor Stinner. + +.. + +.. date: 2026-11-03-13-12-50 +.. gh-issue: 161263 +.. nonce: Voe8VM +.. section: Tests + +Skip tests which use MD5 algorithm for WWW-Authenticate challenges when +libssl is in FIPS mode. + +.. + +.. date: 2026-10-28-20-40-00 +.. gh-issue: 158646 +.. nonce: k2Xw7q +.. section: Tests + +Fix a race in ``test_asyncio.test_sslproto`` where the server side of +``test_shutdown_corrupted_ssl_sends_close_notify`` could fail with a +connection reset on Windows. + +.. + +.. date: 2026-10-19-17-05-00 +.. gh-issue: 107862 +.. nonce: qH3sTa +.. section: Tests + +Add ``hypothesis.stateful`` and the ``timezones()`` and ``ip_addresses()`` +strategies to the Hypothesis stubs used when Hypothesis is not installed. + +.. + +.. date: 2026-10-13-13-34-00 +.. gh-issue: 159293 +.. nonce: Lr8q2N +.. section: Tests + +Restore the recursion limit after the free-threading recursion-limit race +test to avoid failures in subsequent tests run in the same process. + +.. + +.. date: 2026-10-07-12-00-58 +.. gh-issue: 158975 +.. nonce: yR8q2L +.. section: Tests + +Replace fixed sleeps used for synchronization in multiprocessing tests with +bounded waits to avoid failures on slow or overloaded systems. Ensure pool +cleanup runs even when assertions fail in the imap buffer tests. + +.. + +.. date: 2026-10-05-22-36-06 +.. gh-issue: 158895 +.. nonce: 3Dmc4J +.. section: Tests + +When run in GitHub Actions, :mod:`test.regrtest` now annotates each test +failure in the job log and writes a summary of the failures to the job +summary. + +.. + +.. date: 2026-11-06-09-16-30 +.. gh-issue: 161467 +.. nonce: oSuzx_ +.. section: Build + +Enable the use of ``__attribute__((hot))`` when building CPython with Clang +and other compilers that support it. + +.. + +.. date: 2026-11-04-15-00-00 +.. gh-issue: 160924 +.. nonce: WQgVVh +.. section: Build + +Fix cross-compilation failures with ``--enable-shared`` caused by the build +Python loading the target's ``libpython`` through the shared-library search +path (such as musl from glibc). + +.. + +.. date: 2026-11-04-14-37-13 +.. gh-issue: 75929 +.. nonce: nELdwE +.. section: Build + +Fix a ``configure`` probe that could omit functions such as +:func:`os.fchdir` from builds using ``-Wall -Werror``. + +.. + +.. date: 2026-10-26-21-00-00 +.. gh-issue: 160704 +.. nonce: ZRYLKF +.. section: Build + +Fix C11-C17 builds with old C library headers that do not define +``static_assert``, including glibc versions older than 2.16. + +.. + +.. date: 2026-10-25-15-30-00 +.. gh-issue: 160689 +.. nonce: bootsp +.. section: Build + +The build helper programs ``Programs/_freeze_module`` and +``_bootstrap_python`` are now linked only with the modules of +:file:`Modules/Setup.bootstrap`, instead of with every static module. +Builtin modules added via :file:`Modules/Setup.local` or +``MODULE_BUILDTYPE=static`` can now depend on frozen modules or on +``getpath``. + +.. + +.. date: 2026-10-22-18-57-02 +.. gh-issue: 128884 +.. nonce: 3zR5Yp +.. section: Build + +Report a configuration error when ``--enable-bolt`` is used with build flags +that explicitly request the Cortex-A53 erratum 843419 workaround. Document +how to disable the workaround for builds that will not run on affected +processors. + +.. + +.. date: 2026-10-21-18-00-00 +.. gh-issue: 128883 +.. nonce: perf-trampoline +.. section: Build + +Fix crashes with ``-X perf`` and ``-X perf_jit`` in BOLT builds. The ELF +perf trampoline templates now use read-only data sections. + +.. + +.. date: 2026-10-21-14-52-56 +.. gh-issue: 128884 +.. nonce: Xs5jK- +.. section: Build + +Fix ``--enable-bolt`` build failures on AArch64 with recent LLVM: pass +``-frame-opt=hot`` only on x86. Patch by Donghee Na. + +.. + +.. date: 2026-10-20-23-30-00 +.. gh-issue: 160004 +.. nonce: CfgPth +.. section: Build + +Fix the paths in the :pep:`739` ``build-details.json`` file generated by +:file:`Tools/build/generate-build-details.py` with both ``--relative-paths`` +and ``--config-file-path``. Previously, every path except ``base_prefix`` +was wrong and depended on the directory the script was run from. + +.. + +.. date: 2026-10-20-23-00-00 +.. gh-issue: 159998 +.. nonce: FwDyn1 +.. section: Build + +Fix ``libpython.dynamic`` in the :pep:`739` ``build-details.json`` file for +macOS and iOS framework builds. It now points at the framework's library +instead of a nonexistent path under :file:`lib/`. + +.. + +.. date: 2026-10-20-20-00-00 +.. gh-issue: 144942 +.. nonce: MdLibs +.. section: Build + +Fix the macOS shared library and framework builds not linking the libraries +required by built-in modules, such as the HACL* objects needed by +``_blake2`` when it is built in via :file:`Modules/Setup.local`. + +.. + +.. date: 2026-10-20-17-50-00 +.. gh-issue: 143632 +.. nonce: PrCtlM +.. section: Build + +Fix the check that disables ``PR_SET_VMA_ANON_NAME`` on musl libc: it tested +an unset variable, so it never had any effect and the build could fail with +a redefinition of ``struct prctl_mm_map``. + +.. + +.. date: 2026-10-20-12-06-22 +.. gh-issue: 131999 +.. nonce: OWoYGM +.. section: Build + +Platform support for Apple's tvOS, watchOS and visionOS was added. + +.. + +.. date: 2026-10-20-00-56-59 +.. gh-issue: 153195 +.. nonce: 35cRE- +.. section: Build + +The PEG parser generator now supports splitting a grammar into several files +with the ``@include`` meta, and generates a separate C file for the rules of +each included file. The rules for specialized syntax errors moved from +:file:`Grammar/python.gram` to :file:`Grammar/invalid.gram`, which generates +the new :file:`Parser/parser_invalid.c`. + +.. + +.. date: 2026-10-19-17-05-00 +.. gh-issue: 159819 +.. nonce: pyCfgSp +.. section: Build + +Fix the shell version of :program:`python3-config` (used on all POSIX +platforms except macOS) when the installation prefix contains spaces or +other characters special to the shell or :program:`sed`: it printed an empty +``--prefix`` and wrong ``--includes``/``--ldflags`` paths while exiting +successfully. The script now also exits with status 1 if it cannot determine +its installation prefix. + +.. + +.. date: 2026-10-19-13-49-39 +.. gh-issue: 54767 +.. nonce: IdrGm6 +.. section: Build + +The ``--enable-loadable-sqlite-extensions=no`` and +``--disable-loadable-sqlite-extensions`` configure options now disable +loadable SQLite extensions instead of enabling them. + +.. + +.. date: 2026-10-18-12-00-00 +.. gh-issue: 159759 +.. nonce: LibStA +.. section: Build + +Fix the ``libpython.static`` path in :pep:`739` ``build-details.json``, +which pointed to a file that is not installed when building with +:option:`--enable-shared`, and no longer list a static library for framework +builds, which do not install one. + +.. + +.. date: 2026-10-15-19-09-31 +.. gh-issue: 75382 +.. nonce: DSrcra +.. section: Build + +Make :program:`configure` fail early with a clear error when a +``--with-*-sanitizer`` option is used but a program cannot be linked with +the corresponding ``-fsanitize`` flag, for example because the sanitizer +runtime library is not installed. + +.. + +.. date: 2026-10-15-17-19-41 +.. gh-issue: 159452 +.. nonce: oJ1saO +.. section: Build + +``Tools/build/regen-configure.sh`` no longer leaves root-owned files in the +source tree. + +.. + +.. date: 2026-10-15-11-20-48 +.. gh-issue: 159440 +.. nonce: uIz6ms +.. section: Build + +The version string reported by ``sys.version`` no longer carries a spurious +``-dirty`` suffix when CPython is built out-of-tree from a clean git +checkout: the build stamp now checks the source tree instead of the build +directory. + +.. + +.. date: 2026-09-22-09-13-06 +.. gh-issue: 157949 +.. nonce: EZ8avp +.. section: Build + +Remove dead build and preprocessor code for SCO UnixWare, OpenUNIX, SCO +OpenServer and ReliantUNIX. + +.. + +.. date: 2026-09-12-15-33-26 +.. gh-issue: 157354 +.. nonce: 2hw6fe +.. section: Build + +Fix the :samp:`--with-{name}-sanitizer=no` and +:samp:`--without-{name}-sanitizer` :program:`configure` options enabling the +sanitizer instead of disabling it. + +.. + +.. date: 2026-09-01-23-32-22 +.. gh-issue: 156780 +.. nonce: Qw3nRt +.. section: Build + +Emscripten ``libpython.a`` can now be linked into a program statically. + +.. + +.. date: 2026-08-27-23-24-29 +.. gh-issue: 155813 +.. nonce: LxRQu9 +.. section: Build + +On macOS, :option:`LIBFFI_CFLAGS` and :option:`LIBFFI_LIBS` are no longer +ignored when the SDK provides ``libffi``, so :mod:`ctypes` can again be +built against a different ``libffi``. Patch by Advit Arora. + +.. + +.. date: 2026-07-11-01-53-58 +.. gh-issue: 153285 +.. nonce: YmSox4 +.. section: Build + +Fix incorrect executable linking on macOS with ``--with-lto`` caused by +premature expansion of Make's ``$@`` target variable. + +.. + +.. date: 2026-10-22-14-04-17 +.. gh-issue: 160239 +.. nonce: A1b2C3 +.. section: Windows + +Remove dead and broken ``_repr_info()`` methods from +``asyncio.windows_events``; they called ``Future._repr_info()``, which no +longer exists. + +.. + +.. date: 2026-10-20-21-37-34 +.. gh-issue: 159880 +.. nonce: eKBiLr +.. section: Windows + +Fix a key handle leak in :func:`winreg.CreateKey`, +:func:`winreg.CreateKeyEx` and :func:`winreg.OpenKey` when the +``winreg.OpenKey/result`` audit hook fails. + +.. + +.. date: 2026-10-15-18-45-00 +.. gh-issue: 159118 +.. nonce: wmiMTA +.. section: Windows + +Fix a crash (``RPC_NT_INTERNAL_ERROR``) and intermittent window class +failures in :mod:`!_wmi` under high concurrency and load by initializing COM +as multi-threaded (MTA) instead of apartment-threaded (STA). + +.. + +.. date: 2026-09-14-16-49-44 +.. gh-issue: 157418 +.. nonce: r2IggA +.. section: Windows + +Correct the third parameter name of ``IocpProactor.recvfrom_into()`` from +*flags* to *nbytes* to reflect that it specifies the number of bytes to +receive. + +.. + +.. date: 2026-10-19-19-33-35 +.. gh-issue: 159739 +.. nonce: aaf575 +.. section: macOS + +Propagate errors that occur while converting macOS system proxy settings. + +.. + +.. date: 2026-11-04-16-34-48 +.. gh-issue: 161367 +.. nonce: bO5MGy +.. section: WASI + +Add a ``test`` command to :file:`Platforms/WASI` with ``--fast-ci`` and +``--slow-ci`` to select fast CI or slow Buildbot tests and ``--target`` to +test the build, gathered distribution, or release archive. + +.. + +.. date: 2026-10-21-15-26-12 +.. gh-issue: 160135 +.. nonce: UP-DlO +.. section: WASI + +Support wasm32-wasip3. + +.. + +.. date: 2026-11-06-17-12-40 +.. gh-issue: 161477 +.. nonce: smXIKC +.. section: IDLE + +Fix ``repr()`` of ``idlelib.redirector.WidgetRedirector`` and +``idlelib.redirector.OriginalCommand``, which displayed a spurious +one-element tuple. + +.. + +.. date: 2026-10-16-15-12-36 +.. gh-issue: 159517 +.. nonce: uzRtAd +.. section: IDLE + +Fix the completion list and the call tip grabbing the focus on macOS with +Tcl/Tk 9.1. + +.. + +.. date: 2026-10-21-11-53-40 +.. gh-issue: 87613 +.. nonce: tmKBlg +.. section: Tools/Demos + +Extend Argument Clinic :ref:`vectorcall` to support :ref:`heap-types` with +zero argument construction. + +.. + +.. date: 2026-10-20-12-50-00 +.. gh-issue: 114258 +.. nonce: slotNx +.. section: Tools/Demos + +Argument Clinic now supports the ``__next__`` type slot (``tp_iternext``), +including ``@critical_section``. + +.. + +.. date: 2026-10-14-09-48-26 +.. gh-issue: 159385 +.. nonce: 0aWOA9 +.. section: Tools/Demos + +iOS builds can now be configured to generate free-threading or debug output. + +.. + +.. date: 2025-03-25-18-00-00 +.. gh-issue: 131725 +.. nonce: qwfh321 +.. section: Tools/Demos + +``Tools/i18n/msgfmt.py`` now writes a hash table to the generated ``.mo`` +file, as GNU :program:`msgfmt` does. + +.. + +.. date: 2026-11-05-16-56-57 +.. gh-issue: 160109 +.. nonce: Fvs4m2 +.. section: C API + +Remove the unused ``FVS_MASK`` and ``FVS_HAVE_SPEC`` macros. They were used +by the ``FORMAT_VALUE`` opcode, which was removed in Python 3.13. + +.. + +.. date: 2026-11-05-10-51-17 +.. gh-issue: 161407 +.. nonce: rVoqaQ +.. section: C API + +Fix a crash at exit when an audit hook was added with +:c:func:`PySys_AddAuditHook` before :c:func:`Py_Initialize` and the raw +memory allocator changed during startup, for example with +``PYTHONMALLOC=mimalloc`` or in the :ref:`Python Development Mode +`. + +.. + +.. date: 2026-11-03-18-27-42 +.. gh-issue: 160962 +.. nonce: 06816I +.. section: C API + +Fix :c:func:`PyInitConfig_SetStr` when setting an optional string option to +``NULL``. Previously, the function crashed. Patch by Victor Stinner. + +.. + +.. date: 2026-11-03-01-24-01 +.. gh-issue: 161226 +.. nonce: MPrJKp +.. section: C API + +:c:func:`_PyTuple_Resize` now fails with :exc:`SystemError` if the hash +value was already computed. Patch by Victor Stinner. + +.. + +.. date: 2026-11-02-15-17-50 +.. gh-issue: 161185 +.. nonce: 4bpyDC +.. section: C API + +Fix a strict aliasing issue in the :c:func:`!memcpy` implementation of +:c:macro:`Py_SETREF`, :c:macro:`Py_XSETREF` and :c:macro:`Py_CLEAR` macros. +Patch by Nathan Goldbaum and Victor Stinner. + +.. + +.. date: 2026-11-02-13-15-22 +.. gh-issue: 161172 +.. nonce: U6kpuN +.. section: C API + +Add the :ref:`PyUnicodeWriter API ` to the limited C API. +Patch by Victor Stinner. + +.. + +.. date: 2026-10-21-19-55-00 +.. gh-issue: 159739 +.. nonce: ctxt01 +.. section: C API + +Avoid a crash if a context watcher is cleared while other context watchers +are being notified. + +.. + +.. date: 2026-10-19-21-05-12 +.. gh-issue: 159739 +.. nonce: Rk3fTq +.. section: C API + +Fix a crash in :c:func:`_PyTuple_Resize` when resizing a non-empty tuple to +a negative or too large size. It now raises :exc:`SystemError` or +:exc:`MemoryError` respectively. + +.. + +.. date: 2026-10-19-20-38-17 +.. gh-issue: 159739 +.. nonce: eb9cdc +.. section: C API + +:c:func:`PyInitConfig_SetStr` now releases a previously configured value and +preserves it if conversion of the replacement fails. + +.. + +.. date: 2026-10-19-19-33-35 +.. gh-issue: 159739 +.. nonce: dee9a0 +.. section: C API + +Fix a reference leak when :c:func:`_PyTuple_Resize` fails. + +.. + +.. date: 2026-09-12-18-37-22 +.. gh-issue: 157384 +.. nonce: uFbdle +.. section: C API + +:c:func:`PyImport_CreateModuleFromInitfunc` now accepts non-ASCII module +names for multi-phase init modules, instead of raising +:exc:`UnicodeEncodeError`. + +.. + +.. date: 2026-08-20-12-00-00 +.. gh-issue: 156133 +.. nonce: GetLoc +.. section: C API + +Add :c:func:`PyUnstable_InterpreterFrame_GetLocal` to read a local variable +of an internal interpreter frame by its localsplus index, and +:c:func:`PyUnstable_Code_GetLocalPlusNames` to get the matching variable +names. + +.. + +.. date: 2026-07-09-13-52-17 +.. gh-issue: 153410 +.. nonce: neaDgY +.. section: C API + +Add :c:func:`PyDict_AsFrozenDictAndClear` C-API function to create +:class:`frozendict` from existing :class:`dict` object and clear the input +dict instance. diff --git a/Misc/NEWS.d/next/Build/2026-07-11-01-53-58.gh-issue-153285.YmSox4.rst b/Misc/NEWS.d/next/Build/2026-07-11-01-53-58.gh-issue-153285.YmSox4.rst deleted file mode 100644 index d672c8b9910d76d..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-07-11-01-53-58.gh-issue-153285.YmSox4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix incorrect executable linking on macOS with ``--with-lto`` caused by -premature expansion of Make's ``$@`` target variable. diff --git a/Misc/NEWS.d/next/Build/2026-08-27-23-24-29.gh-issue-155813.LxRQu9.rst b/Misc/NEWS.d/next/Build/2026-08-27-23-24-29.gh-issue-155813.LxRQu9.rst deleted file mode 100644 index c614826ba4c19ad..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-08-27-23-24-29.gh-issue-155813.LxRQu9.rst +++ /dev/null @@ -1,3 +0,0 @@ -On macOS, :option:`LIBFFI_CFLAGS` and :option:`LIBFFI_LIBS` are no longer -ignored when the SDK provides ``libffi``, so :mod:`ctypes` can again be built -against a different ``libffi``. Patch by Advit Arora. diff --git a/Misc/NEWS.d/next/Build/2026-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst b/Misc/NEWS.d/next/Build/2026-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst deleted file mode 100644 index fe664df69097405..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst +++ /dev/null @@ -1 +0,0 @@ -Emscripten ``libpython.a`` can now be linked into a program statically. diff --git a/Misc/NEWS.d/next/Build/2026-09-12-15-33-26.gh-issue-157354.2hw6fe.rst b/Misc/NEWS.d/next/Build/2026-09-12-15-33-26.gh-issue-157354.2hw6fe.rst deleted file mode 100644 index 5bc35aaf9870c2a..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-09-12-15-33-26.gh-issue-157354.2hw6fe.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the :samp:`--with-{name}-sanitizer=no` and :samp:`--without-{name}-sanitizer` -:program:`configure` options enabling the sanitizer instead of disabling it. diff --git a/Misc/NEWS.d/next/Build/2026-09-22-09-13-06.gh-issue-157949.EZ8avp.rst b/Misc/NEWS.d/next/Build/2026-09-22-09-13-06.gh-issue-157949.EZ8avp.rst deleted file mode 100644 index 0b1034807b7dd17..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-09-22-09-13-06.gh-issue-157949.EZ8avp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove dead build and preprocessor code for SCO UnixWare, OpenUNIX, SCO -OpenServer and ReliantUNIX. diff --git a/Misc/NEWS.d/next/Build/2026-10-15-11-20-48.gh-issue-159440.uIz6ms.rst b/Misc/NEWS.d/next/Build/2026-10-15-11-20-48.gh-issue-159440.uIz6ms.rst deleted file mode 100644 index 6c6cc43fd2e60d0..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-15-11-20-48.gh-issue-159440.uIz6ms.rst +++ /dev/null @@ -1,3 +0,0 @@ -The version string reported by ``sys.version`` no longer carries a spurious -``-dirty`` suffix when CPython is built out-of-tree from a clean git checkout: -the build stamp now checks the source tree instead of the build directory. diff --git a/Misc/NEWS.d/next/Build/2026-10-15-17-19-41.gh-issue-159452.oJ1saO.rst b/Misc/NEWS.d/next/Build/2026-10-15-17-19-41.gh-issue-159452.oJ1saO.rst deleted file mode 100644 index 531239d3e33f4a4..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-15-17-19-41.gh-issue-159452.oJ1saO.rst +++ /dev/null @@ -1,2 +0,0 @@ -``Tools/build/regen-configure.sh`` no longer leaves -root-owned files in the source tree. diff --git a/Misc/NEWS.d/next/Build/2026-10-15-19-09-31.gh-issue-75382.DSrcra.rst b/Misc/NEWS.d/next/Build/2026-10-15-19-09-31.gh-issue-75382.DSrcra.rst deleted file mode 100644 index eb295fd292a6116..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-15-19-09-31.gh-issue-75382.DSrcra.rst +++ /dev/null @@ -1,4 +0,0 @@ -Make :program:`configure` fail early with a clear error when a -``--with-*-sanitizer`` option is used but a program cannot be linked with -the corresponding ``-fsanitize`` flag, for example because the sanitizer -runtime library is not installed. diff --git a/Misc/NEWS.d/next/Build/2026-10-18-12-00-00.gh-issue-159759.LibStA.rst b/Misc/NEWS.d/next/Build/2026-10-18-12-00-00.gh-issue-159759.LibStA.rst deleted file mode 100644 index 1fb6a51fded6dcb..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-18-12-00-00.gh-issue-159759.LibStA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix the ``libpython.static`` path in :pep:`739` ``build-details.json``, which -pointed to a file that is not installed when building with -:option:`--enable-shared`, and no longer list a static library for framework -builds, which do not install one. diff --git a/Misc/NEWS.d/next/Build/2026-10-19-13-49-39.gh-issue-54767.IdrGm6.rst b/Misc/NEWS.d/next/Build/2026-10-19-13-49-39.gh-issue-54767.IdrGm6.rst deleted file mode 100644 index 6154892709ee12d..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-19-13-49-39.gh-issue-54767.IdrGm6.rst +++ /dev/null @@ -1,3 +0,0 @@ -The ``--enable-loadable-sqlite-extensions=no`` and -``--disable-loadable-sqlite-extensions`` configure options now disable -loadable SQLite extensions instead of enabling them. diff --git a/Misc/NEWS.d/next/Build/2026-10-19-17-05-00.gh-issue-159819.pyCfgSp.rst b/Misc/NEWS.d/next/Build/2026-10-19-17-05-00.gh-issue-159819.pyCfgSp.rst deleted file mode 100644 index a7fcdaa05471eb4..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-19-17-05-00.gh-issue-159819.pyCfgSp.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix the shell version of :program:`python3-config` (used on all POSIX -platforms except macOS) when the installation prefix contains spaces or -other characters special to the shell or :program:`sed`: it printed an -empty ``--prefix`` and wrong ``--includes``/``--ldflags`` paths while -exiting successfully. The script now also exits with status 1 if it cannot -determine its installation prefix. diff --git a/Misc/NEWS.d/next/Build/2026-10-20-00-56-59.gh-issue-153195.35cRE-.rst b/Misc/NEWS.d/next/Build/2026-10-20-00-56-59.gh-issue-153195.35cRE-.rst deleted file mode 100644 index 75cc37cbe62d6a6..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-20-00-56-59.gh-issue-153195.35cRE-.rst +++ /dev/null @@ -1,5 +0,0 @@ -The PEG parser generator now supports splitting a grammar into several files -with the ``@include`` meta, and generates a separate C file for the rules of -each included file. The rules for specialized syntax errors moved from -:file:`Grammar/python.gram` to :file:`Grammar/invalid.gram`, which generates -the new :file:`Parser/parser_invalid.c`. diff --git a/Misc/NEWS.d/next/Build/2026-10-20-12-06-22.gh-issue-131999.OWoYGM.rst b/Misc/NEWS.d/next/Build/2026-10-20-12-06-22.gh-issue-131999.OWoYGM.rst deleted file mode 100644 index 8f282fa2d14c891..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-20-12-06-22.gh-issue-131999.OWoYGM.rst +++ /dev/null @@ -1 +0,0 @@ -Platform support for Apple's tvOS, watchOS and visionOS was added. diff --git a/Misc/NEWS.d/next/Build/2026-10-20-17-50-00.gh-issue-143632.PrCtlM.rst b/Misc/NEWS.d/next/Build/2026-10-20-17-50-00.gh-issue-143632.PrCtlM.rst deleted file mode 100644 index 5a2c7b8208cd363..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-20-17-50-00.gh-issue-143632.PrCtlM.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix the check that disables ``PR_SET_VMA_ANON_NAME`` on musl libc: it tested -an unset variable, so it never had any effect and the build could fail with -a redefinition of ``struct prctl_mm_map``. diff --git a/Misc/NEWS.d/next/Build/2026-10-20-20-00-00.gh-issue-144942.MdLibs.rst b/Misc/NEWS.d/next/Build/2026-10-20-20-00-00.gh-issue-144942.MdLibs.rst deleted file mode 100644 index 94b25d67af2fe46..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-20-20-00-00.gh-issue-144942.MdLibs.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix the macOS shared library and framework builds not linking the libraries -required by built-in modules, such as the HACL* objects needed by ``_blake2`` -when it is built in via :file:`Modules/Setup.local`. diff --git a/Misc/NEWS.d/next/Build/2026-10-20-23-00-00.gh-issue-159998.FwDyn1.rst b/Misc/NEWS.d/next/Build/2026-10-20-23-00-00.gh-issue-159998.FwDyn1.rst deleted file mode 100644 index 33a5c6414936e77..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-20-23-00-00.gh-issue-159998.FwDyn1.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix ``libpython.dynamic`` in the :pep:`739` ``build-details.json`` file for -macOS and iOS framework builds. It now points at the framework's library -instead of a nonexistent path under :file:`lib/`. diff --git a/Misc/NEWS.d/next/Build/2026-10-20-23-30-00.gh-issue-160004.CfgPth.rst b/Misc/NEWS.d/next/Build/2026-10-20-23-30-00.gh-issue-160004.CfgPth.rst deleted file mode 100644 index 247f4677fc27c64..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-20-23-30-00.gh-issue-160004.CfgPth.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix the paths in the :pep:`739` ``build-details.json`` file generated by -:file:`Tools/build/generate-build-details.py` with both ``--relative-paths`` -and ``--config-file-path``. Previously, every path except ``base_prefix`` was -wrong and depended on the directory the script was run from. diff --git a/Misc/NEWS.d/next/Build/2026-10-21-14-52-56.gh-issue-128884.Xs5jK-.rst b/Misc/NEWS.d/next/Build/2026-10-21-14-52-56.gh-issue-128884.Xs5jK-.rst deleted file mode 100644 index 53233d06c5053a4..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-21-14-52-56.gh-issue-128884.Xs5jK-.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``--enable-bolt`` build failures on AArch64 with recent LLVM: pass -``-frame-opt=hot`` only on x86. Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/Build/2026-10-21-18-00-00.gh-issue-128883.perf-trampoline.rst b/Misc/NEWS.d/next/Build/2026-10-21-18-00-00.gh-issue-128883.perf-trampoline.rst deleted file mode 100644 index e250415e72af144..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-21-18-00-00.gh-issue-128883.perf-trampoline.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crashes with ``-X perf`` and ``-X perf_jit`` in BOLT builds. -The ELF perf trampoline templates now use read-only data sections. diff --git a/Misc/NEWS.d/next/Build/2026-10-22-18-57-02.gh-issue-128884.3zR5Yp.rst b/Misc/NEWS.d/next/Build/2026-10-22-18-57-02.gh-issue-128884.3zR5Yp.rst deleted file mode 100644 index 8d0227cf6841445..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-22-18-57-02.gh-issue-128884.3zR5Yp.rst +++ /dev/null @@ -1,3 +0,0 @@ -Report a configuration error when ``--enable-bolt`` is used with build flags -that explicitly request the Cortex-A53 erratum 843419 workaround. Document how to -disable the workaround for builds that will not run on affected processors. diff --git a/Misc/NEWS.d/next/Build/2026-10-25-15-30-00.gh-issue-160689.bootsp.rst b/Misc/NEWS.d/next/Build/2026-10-25-15-30-00.gh-issue-160689.bootsp.rst deleted file mode 100644 index e83c9708142b396..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-25-15-30-00.gh-issue-160689.bootsp.rst +++ /dev/null @@ -1,5 +0,0 @@ -The build helper programs ``Programs/_freeze_module`` and -``_bootstrap_python`` are now linked only with the modules of -:file:`Modules/Setup.bootstrap`, instead of with every static module. Builtin -modules added via :file:`Modules/Setup.local` or ``MODULE_BUILDTYPE=static`` -can now depend on frozen modules or on ``getpath``. diff --git a/Misc/NEWS.d/next/Build/2026-10-26-21-00-00.gh-issue-160704.ZRYLKF.rst b/Misc/NEWS.d/next/Build/2026-10-26-21-00-00.gh-issue-160704.ZRYLKF.rst deleted file mode 100644 index 330415c155cd89a..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-10-26-21-00-00.gh-issue-160704.ZRYLKF.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix C11-C17 builds with old C library headers that do not define -``static_assert``, including glibc versions older than 2.16. diff --git a/Misc/NEWS.d/next/Build/2026-11-04-14-37-13.gh-issue-75929.nELdwE.rst b/Misc/NEWS.d/next/Build/2026-11-04-14-37-13.gh-issue-75929.nELdwE.rst deleted file mode 100644 index 8ec7f8a0147399f..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-11-04-14-37-13.gh-issue-75929.nELdwE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a ``configure`` probe that could omit functions such -as :func:`os.fchdir` from builds using ``-Wall -Werror``. diff --git a/Misc/NEWS.d/next/Build/2026-11-04-15-00-00.gh-issue-160924.WQgVVh.rst b/Misc/NEWS.d/next/Build/2026-11-04-15-00-00.gh-issue-160924.WQgVVh.rst deleted file mode 100644 index 2817321dc7e4d1b..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-11-04-15-00-00.gh-issue-160924.WQgVVh.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix cross-compilation failures with ``--enable-shared`` caused by the -build Python loading the target's ``libpython`` through the shared-library -search path (such as musl from glibc). diff --git a/Misc/NEWS.d/next/Build/2026-11-06-09-16-30.gh-issue-161467.oSuzx_.rst b/Misc/NEWS.d/next/Build/2026-11-06-09-16-30.gh-issue-161467.oSuzx_.rst deleted file mode 100644 index ae34c60e989ead1..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-11-06-09-16-30.gh-issue-161467.oSuzx_.rst +++ /dev/null @@ -1 +0,0 @@ -Enable the use of ``__attribute__((hot))`` when building CPython with Clang and other compilers that support it. diff --git a/Misc/NEWS.d/next/C_API/2026-07-09-13-52-17.gh-issue-153410.neaDgY.rst b/Misc/NEWS.d/next/C_API/2026-07-09-13-52-17.gh-issue-153410.neaDgY.rst deleted file mode 100644 index 259981100b5bf15..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-07-09-13-52-17.gh-issue-153410.neaDgY.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :c:func:`PyDict_AsFrozenDictAndClear` C-API function to create -:class:`frozendict` from existing :class:`dict` object and clear the input -dict instance. diff --git a/Misc/NEWS.d/next/C_API/2026-08-20-12-00-00.gh-issue-156133.GetLoc.rst b/Misc/NEWS.d/next/C_API/2026-08-20-12-00-00.gh-issue-156133.GetLoc.rst deleted file mode 100644 index be780cba1214d30..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-08-20-12-00-00.gh-issue-156133.GetLoc.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :c:func:`PyUnstable_InterpreterFrame_GetLocal` to read a local variable of -an internal interpreter frame by its localsplus index, and -:c:func:`PyUnstable_Code_GetLocalPlusNames` to get the matching variable names. diff --git a/Misc/NEWS.d/next/C_API/2026-09-12-18-37-22.gh-issue-157384.uFbdle.rst b/Misc/NEWS.d/next/C_API/2026-09-12-18-37-22.gh-issue-157384.uFbdle.rst deleted file mode 100644 index 0ba43518705c50a..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-09-12-18-37-22.gh-issue-157384.uFbdle.rst +++ /dev/null @@ -1,3 +0,0 @@ -:c:func:`PyImport_CreateModuleFromInitfunc` now accepts non-ASCII module -names for multi-phase init modules, instead of raising -:exc:`UnicodeEncodeError`. diff --git a/Misc/NEWS.d/next/C_API/2026-10-19-19-33-35.gh-issue-159739.dee9a0.rst b/Misc/NEWS.d/next/C_API/2026-10-19-19-33-35.gh-issue-159739.dee9a0.rst deleted file mode 100644 index 891cf551730af2b..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-10-19-19-33-35.gh-issue-159739.dee9a0.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a reference leak when :c:func:`_PyTuple_Resize` fails. diff --git a/Misc/NEWS.d/next/C_API/2026-10-19-20-38-17.gh-issue-159739.eb9cdc.rst b/Misc/NEWS.d/next/C_API/2026-10-19-20-38-17.gh-issue-159739.eb9cdc.rst deleted file mode 100644 index e3ca598d37fdcef..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-10-19-20-38-17.gh-issue-159739.eb9cdc.rst +++ /dev/null @@ -1,2 +0,0 @@ -:c:func:`PyInitConfig_SetStr` now releases a previously configured value and -preserves it if conversion of the replacement fails. diff --git a/Misc/NEWS.d/next/C_API/2026-10-19-21-05-12.gh-issue-159739.Rk3fTq.rst b/Misc/NEWS.d/next/C_API/2026-10-19-21-05-12.gh-issue-159739.Rk3fTq.rst deleted file mode 100644 index 2d510c63ac07a87..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-10-19-21-05-12.gh-issue-159739.Rk3fTq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in :c:func:`_PyTuple_Resize` when resizing a non-empty tuple to a -negative or too large size. It now raises :exc:`SystemError` or -:exc:`MemoryError` respectively. diff --git a/Misc/NEWS.d/next/C_API/2026-10-21-19-55-00.gh-issue-159739.ctxt01.rst b/Misc/NEWS.d/next/C_API/2026-10-21-19-55-00.gh-issue-159739.ctxt01.rst deleted file mode 100644 index 10c268e1043c9ed..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-10-21-19-55-00.gh-issue-159739.ctxt01.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid a crash if a context watcher is cleared while other context watchers -are being notified. diff --git a/Misc/NEWS.d/next/C_API/2026-11-02-13-15-22.gh-issue-161172.U6kpuN.rst b/Misc/NEWS.d/next/C_API/2026-11-02-13-15-22.gh-issue-161172.U6kpuN.rst deleted file mode 100644 index 638290ef247f37a..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-11-02-13-15-22.gh-issue-161172.U6kpuN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add the :ref:`PyUnicodeWriter API ` to the limited C API. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2026-11-02-15-17-50.gh-issue-161185.4bpyDC.rst b/Misc/NEWS.d/next/C_API/2026-11-02-15-17-50.gh-issue-161185.4bpyDC.rst deleted file mode 100644 index 92887e84ade4cd2..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-11-02-15-17-50.gh-issue-161185.4bpyDC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a strict aliasing issue in the :c:func:`!memcpy` implementation of -:c:macro:`Py_SETREF`, :c:macro:`Py_XSETREF` and :c:macro:`Py_CLEAR` macros. -Patch by Nathan Goldbaum and Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2026-11-03-01-24-01.gh-issue-161226.MPrJKp.rst b/Misc/NEWS.d/next/C_API/2026-11-03-01-24-01.gh-issue-161226.MPrJKp.rst deleted file mode 100644 index 601db29c66cd9b8..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-11-03-01-24-01.gh-issue-161226.MPrJKp.rst +++ /dev/null @@ -1,2 +0,0 @@ -:c:func:`_PyTuple_Resize` now fails with :exc:`SystemError` if the hash value -was already computed. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2026-11-03-18-27-42.gh-issue-160962.06816I.rst b/Misc/NEWS.d/next/C_API/2026-11-03-18-27-42.gh-issue-160962.06816I.rst deleted file mode 100644 index 3b043a8d495f1c6..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-11-03-18-27-42.gh-issue-160962.06816I.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :c:func:`PyInitConfig_SetStr` when setting an optional string option to -``NULL``. Previously, the function crashed. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2026-11-05-10-51-17.gh-issue-161407.rVoqaQ.rst b/Misc/NEWS.d/next/C_API/2026-11-05-10-51-17.gh-issue-161407.rVoqaQ.rst deleted file mode 100644 index 421329930f961a7..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-11-05-10-51-17.gh-issue-161407.rVoqaQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash at exit when an audit hook was added with :c:func:`PySys_AddAuditHook` -before :c:func:`Py_Initialize` and the raw memory allocator changed during startup, for -example with ``PYTHONMALLOC=mimalloc`` or in the :ref:`Python Development Mode `. diff --git a/Misc/NEWS.d/next/C_API/2026-11-05-16-56-57.gh-issue-160109.Fvs4m2.rst b/Misc/NEWS.d/next/C_API/2026-11-05-16-56-57.gh-issue-160109.Fvs4m2.rst deleted file mode 100644 index 3f41492e70c8d92..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-11-05-16-56-57.gh-issue-160109.Fvs4m2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove the unused ``FVS_MASK`` and ``FVS_HAVE_SPEC`` macros. They were used by -the ``FORMAT_VALUE`` opcode, which was removed in Python 3.13. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-03-07-22-38-07.gh-issue-116477.AqwRAV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-03-07-22-38-07.gh-issue-116477.AqwRAV.rst deleted file mode 100644 index 420912ace188fa0..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-03-07-22-38-07.gh-issue-116477.AqwRAV.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up the :class:`range` object constructor in the case of a single argument. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-11-12-06-23.gh-issue-122901.B2Huom.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-11-12-06-23.gh-issue-122901.B2Huom.rst deleted file mode 100644 index 734bbc63f381ab4..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-11-12-06-23.gh-issue-122901.B2Huom.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up creating :class:`str` objects from non-ASCII character data. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-22-23-57.gh-issue-125873.BG97KU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-22-23-57.gh-issue-125873.BG97KU.rst deleted file mode 100644 index fc4818781ee1322..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-22-23-57.gh-issue-125873.BG97KU.rst +++ /dev/null @@ -1 +0,0 @@ -Improved error message when an environment variable is misconfigured. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-19-16-10-26.gh-issue-137966.nEgFAt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-19-16-10-26.gh-issue-137966.nEgFAt.rst deleted file mode 100644 index 6a2ce5e3318fdbb..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-19-16-10-26.gh-issue-137966.nEgFAt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add support for additional special names to the built-in :func:`help` function. -Patch by Adam Hartz. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-14-11-20-52.gh-issue-145019.ls2EXo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-14-11-20-52.gh-issue-145019.ls2EXo.rst deleted file mode 100644 index c249c07b212f771..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-14-11-20-52.gh-issue-145019.ls2EXo.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve :exc:`SyntaxError` when :keyword:`match` alternative patterns bind -different names. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-12-13-48-26.gh-issue-69605.T9HWOL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-12-13-48-26.gh-issue-69605.T9HWOL.rst deleted file mode 100644 index 5011debb72e07ad..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-12-13-48-26.gh-issue-69605.T9HWOL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Insert ``import`` after ``from x.y.z `` in the :term:`REPL` -auto-completion. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-15-21-55-45.gh-issue-69605.NwoP6l.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-15-21-55-45.gh-issue-69605.NwoP6l.rst deleted file mode 100644 index 77491c6e5f62070..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-15-21-55-45.gh-issue-69605.NwoP6l.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :term:`REPL` completions inserted incorrectly when there was spaces in -import statements. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-23-08-36-47.gh-issue-148891.AMMHow.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-23-08-36-47.gh-issue-148891.AMMHow.rst deleted file mode 100644 index bae5c1366ca6a2c..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-23-08-36-47.gh-issue-148891.AMMHow.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a crash in :func:`contextvars.Context.run` where a GC invocation could free the current HAMT during traverse. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-21-14-21-06.gh-issue-150075.dLI21Z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-21-14-21-06.gh-issue-150075.dLI21Z.rst deleted file mode 100644 index 25213e3283f169f..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-21-14-21-06.gh-issue-150075.dLI21Z.rst +++ /dev/null @@ -1 +0,0 @@ -tar.addfile() now sets member offsets in TarInfo objects diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-07-22-03-20.gh-issue-151060.tdexEE.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-07-22-03-20.gh-issue-151060.tdexEE.rst deleted file mode 100644 index 1efde8a536c0939..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-07-22-03-20.gh-issue-151060.tdexEE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash or memory corruption in :func:`sum` when summing a -:class:`float` subclass instance into a complex running total. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-20-18-00-00.gh-issue-151818.LsApDf.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-20-18-00-00.gh-issue-151818.LsApDf.rst deleted file mode 100644 index f1e56c81f2ff029..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-20-18-00-00.gh-issue-151818.LsApDf.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a double-free crash in ``list.append()`` (the specialized -``CALL_LIST_APPEND`` instruction) when growing the list fails with -:exc:`MemoryError`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-00-02-29.gh-issue-149816.JKN-5B.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-00-02-29.gh-issue-149816.JKN-5B.rst deleted file mode 100644 index 255507548703b12..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-00-02-29.gh-issue-149816.JKN-5B.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a race in free-threaded builds where storing an instance attribute could -use a stale borrowed reference to an instance ``__dict__`` after another thread -replaced it. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-23-01-07.gh-issue-152631.LKgCM_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-23-01-07.gh-issue-152631.LKgCM_.rst deleted file mode 100644 index 51208108f10647b..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-23-01-07.gh-issue-152631.LKgCM_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`ast.parse` ability to parse lazy imports with *feature_version* -older than 3.15, now it properly raises :exc:`SyntaxError`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-18-00-00.gh-issue-153014.gCdBg1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-18-00-00.gh-issue-153014.gCdBg1.rst deleted file mode 100644 index dec0bfc572118d6..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-18-00-00.gh-issue-153014.gCdBg1.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a data race on the garbage collector debug flag in free-threading builds: -:func:`gc.set_debug` and :func:`gc.get_debug` now access it atomically. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-00-13.gh-issue-153568.atomfp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-00-13.gh-issue-153568.atomfp.rst deleted file mode 100644 index 32ee3f72d6ab059..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-00-13.gh-issue-153568.atomfp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up parsing of simple expressions by recognizing single-token atoms -without descending the full precedence rule chain. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-21-10-42-15.gh-issue-154318.Kq3Wm7.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-21-10-42-15.gh-issue-154318.Kq3Wm7.rst deleted file mode 100644 index b530d8cf06357bd..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-21-10-42-15.gh-issue-154318.Kq3Wm7.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash when hashing a deeply nested :class:`tuple` or :class:`frozendict`. -A :exc:`RecursionError` is now raised instead of overflowing the C stack. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-14-37-50.gh-issue-127716.KrPth8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-14-37-50.gh-issue-127716.KrPth8.rst deleted file mode 100644 index ca2e9bf7a892f21..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-21-14-37-50.gh-issue-127716.KrPth8.rst +++ /dev/null @@ -1,10 +0,0 @@ -On the free-threaded build, :class:`memoryview` lifetime and bookkeeping -operations are now thread safe: -an operation that reads the buffer holds the managed buffer for its duration, -so a concurrent :meth:`~memoryview.release` can no longer free the buffer under -it; slicing a memoryview from several threads no longer loses buffer -registrations and raises ``ValueError: operation forbidden on released -memoryview object``; and :func:`hash` of a memoryview is no longer a data race. -As a result :meth:`~memoryview.release` may now return while an operation -started by another thread is still running, see -:ref:`thread-safety-memoryview`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-27-00-00-00.gh-issue-156399.aB3xYz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-27-00-00-00.gh-issue-156399.aB3xYz.rst deleted file mode 100644 index 7680a16bfe6fd7f..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-27-00-00-00.gh-issue-156399.aB3xYz.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a regression where attribute access on a :class:`types.ModuleType` -subclass could bypass a data descriptor defined on the subclass. The -``LOAD_ATTR_MODULE`` specialization guarded on ``tp_getattro``, which -subclasses inherit, and then read the module dictionary directly. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-27-17-26-21.gh-issue-106045.wzTH7f.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-27-17-26-21.gh-issue-106045.wzTH7f.rst deleted file mode 100644 index a092b72d2ecdaa9..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-27-17-26-21.gh-issue-106045.wzTH7f.rst +++ /dev/null @@ -1 +0,0 @@ -Fix standard library discovery in virtual environments created from a Python executable symlink located outside its base installation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-30-16-30-00.gh-issue-135385.Wn7Rd4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-30-16-30-00.gh-issue-135385.Wn7Rd4.rst deleted file mode 100644 index 43b72921fe9e45c..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-30-16-30-00.gh-issue-135385.Wn7Rd4.rst +++ /dev/null @@ -1,3 +0,0 @@ -Reduce the memory consumption of instances of classes which define -``__slots__`` and also have a ``__dict__``. Space for the attributes stored -in slots is no longer reserved in the inline values of every instance. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-07-09-52-21.gh-issue-156443.tT40sm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-07-09-52-21.gh-issue-156443.tT40sm.rst deleted file mode 100644 index 54c345aad025ce8..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-07-09-52-21.gh-issue-156443.tT40sm.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up :func:`float` conversion and true division of large :class:`int` values. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-08-17-40-00.gh-issue-157194.17Ffhk.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-08-17-40-00.gh-issue-157194.17Ffhk.rst deleted file mode 100644 index c9817f876cd79fb..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-08-17-40-00.gh-issue-157194.17Ffhk.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a data race in the free-threaded build, reported by ThreadSanitizer, -between the specializer reading an instruction opcode and another thread -writing it. Only reachable when thread-local bytecode is disabled with -``-X tlbc=0``, which makes threads share one copy of the bytecode. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-10-00-00.gh-issue-157361.aiterpy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-10-00-00.gh-issue-157361.aiterpy.rst deleted file mode 100644 index b0a3c584da911fb..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-10-00-00.gh-issue-157361.aiterpy.rst +++ /dev/null @@ -1,5 +0,0 @@ -Implement :func:`aiter` in Python instead of C, in the frozen ``_pybuiltins`` -module. The async iterator returned by ``aiter(callable, ...)`` now has a -plain coroutine ``__anext__()``, so introspection tools such as -:func:`asyncio.print_call_graph` can see through it into the awaited -callable result. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-21-12-44-38.gh-issue-132414.uwxmd5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-21-12-44-38.gh-issue-132414.uwxmd5.rst deleted file mode 100644 index 5f795e987650af0..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-21-12-44-38.gh-issue-132414.uwxmd5.rst +++ /dev/null @@ -1,4 +0,0 @@ -Quote the invalid option in the error message for an unknown command-line -option: ``Invalid option '-z'`` instead of ``Unknown option: -z``. A stray -space, as passed by a shebang line like ``#!/usr/bin/python3 -q -i``, is now -reported as ``Invalid option '- '``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-25-11-05-00.gh-issue-157746.q7Lk3d.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-25-11-05-00.gh-issue-157746.q7Lk3d.rst deleted file mode 100644 index eb9e95defa3d6c2..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-25-11-05-00.gh-issue-157746.q7Lk3d.rst +++ /dev/null @@ -1,3 +0,0 @@ -Reduce the overhead of :c:func:`PyErr_CheckSignals` when no signal is -pending: the cheap flag checks are now done before the main-thread check and -the remote debugger configuration lookup. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-28-10-12-25.gh-issue-158304.3pdcnJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-28-10-12-25.gh-issue-158304.3pdcnJ.rst deleted file mode 100644 index 394367d0dba8eb5..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-28-10-12-25.gh-issue-158304.3pdcnJ.rst +++ /dev/null @@ -1,4 +0,0 @@ -Speed up iteration over asynchronous generators: values yielded to -``async for`` no longer create and discard a :exc:`StopIteration` exception -at every nesting level. The ``async_generators`` pyperformance benchmark is -about 30% faster. Patch by Shamil Abdulaev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-29-14-35-00.gh-issue-158121.Kj4nQ8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-29-14-35-00.gh-issue-158121.Kj4nQ8.rst deleted file mode 100644 index 3256ce99a84952b..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-29-14-35-00.gh-issue-158121.Kj4nQ8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix missing :mod:`sys.monitoring` ``INSTRUCTION`` events when a ``LINE`` -callback returns :data:`sys.monitoring.DISABLE`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-21-29-54.gh-issue-158529.kZO7mX.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-21-29-54.gh-issue-158529.kZO7mX.rst deleted file mode 100644 index 712f26f6493877c..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-21-29-54.gh-issue-158529.kZO7mX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed crash on Android x86_64, API levels 27-28, caused by mimalloc calling -system calls directly. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-00-00-00.gh-issue-158525.Xw3K9a.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-00-00-00.gh-issue-158525.Xw3K9a.rst deleted file mode 100644 index 18ab4418020d1f0..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-01-00-00-00.gh-issue-158525.Xw3K9a.rst +++ /dev/null @@ -1 +0,0 @@ -Fix slow ``dict.copy()`` for dicts with many deleted entries in the free-threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-13-00-00.gh-issue-158197.Qk7sVd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-13-00-00.gh-issue-158197.Qk7sVd.rst deleted file mode 100644 index 10fdabcb1f2fd84..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-03-13-00-00.gh-issue-158197.Qk7sVd.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix crashes and debug-build assertion failures in the :term:`free-threaded -build` when iterating over a split dictionary while another thread deletes -entries or clears it. Publish insertion-order updates atomically so readers -cannot observe an increased size before the corresponding order entry is -initialized. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-20-04-55.gh-issue-158816.9ulhWb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-20-04-55.gh-issue-158816.9ulhWb.rst deleted file mode 100644 index fd9997afb3e1d34..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-04-20-04-55.gh-issue-158816.9ulhWb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Speed up deleting items from split dictionaries (for example the -materialized ``__dict__`` of an instance with inline values) by skipping a -redundant hash table probe. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-05-13-19-49.gh-issue-87613.JLTGBy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-05-13-19-49.gh-issue-87613.JLTGBy.rst deleted file mode 100644 index 830e8ef1dc23b28..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-05-13-19-49.gh-issue-87613.JLTGBy.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :class:`bytes` and :class:`bytearray` construction by adding -:ref:`vectorcall` support. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-07-00-00-00.gh-issue-158973.warnings-fork.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-07-00-00-00.gh-issue-158973.warnings-fork.rst deleted file mode 100644 index 9a98cf6e0333845..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-07-00-00-00.gh-issue-158973.warnings-fork.rst +++ /dev/null @@ -1,3 +0,0 @@ -Reset the warnings lock in the child after :func:`os.fork`, before clearing -other threads' states, to avoid deadlocks when their finalizers issue warnings. -Preserve the lock's recursion depth if the forking thread owns it. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst deleted file mode 100644 index 4dd3338e41a4df5..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a use-after-free in :mod:`asyncio` task bookkeeping when a finalizer -creates a task during thread shutdown. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-10-20-21-13.gh-issue-159111.XVf2k-.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-10-20-21-13.gh-issue-159111.XVf2k-.rst deleted file mode 100644 index 3433eaa6ea65b97..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-10-20-21-13.gh-issue-159111.XVf2k-.rst +++ /dev/null @@ -1 +0,0 @@ -Update mimalloc version from 2.1.2 to 2.5.2. Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-06-36-25.gh-issue-159188.Fq7wLm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-06-36-25.gh-issue-159188.Fq7wLm.rst deleted file mode 100644 index 2637fb8e5326e3e..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-11-06-36-25.gh-issue-159188.Fq7wLm.rst +++ /dev/null @@ -1,5 +0,0 @@ -In the :term:`free-threaded build`, :func:`compile` no longer immortalizes -the constants of the code objects when it is given an AST object, as was -already the case for a source string. The output of :mod:`marshal` for -such code no longer depends on the code objects which were created before -in the process. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-11-59-37.gh-issue-68241.jVpLIJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-11-59-37.gh-issue-68241.jVpLIJ.rst deleted file mode 100644 index 3dd3e44d7210785..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-11-59-37.gh-issue-68241.jVpLIJ.rst +++ /dev/null @@ -1 +0,0 @@ -Add EXIT_FAILURE and EXIT_SUCCESS in the sys module. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-16-00-34.gh-issue-159244.ThiobE.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-16-00-34.gh-issue-159244.ThiobE.rst deleted file mode 100644 index 954cccf38c43676..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-16-00-34.gh-issue-159244.ThiobE.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in :class:`dict` and :meth:`dict.update` in the -:term:`free-threaded build` when another thread concurrently mutates a list -used as a key/value pair. Patch by Christian Aurich Zanettini Martins. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-20-00-00.gh-issue-159136.fnAttr.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-20-00-00.gh-issue-159136.fnAttr.rst deleted file mode 100644 index 21fe270bbc1474a..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-12-20-00-00.gh-issue-159136.fnAttr.rst +++ /dev/null @@ -1,8 +0,0 @@ -Setting ``__name__``, ``__qualname__``, ``__doc__`` or ``__module__`` on a -function no longer stops all threads in the free-threaded build. The setters -now use the per-object lock and defer freeing the old value with QSBR, the -same approach used for generator ``__name__``. This fixes a large slowdown -in multi-threaded code using :func:`functools.wraps` since 3.14.7. Readers -of these attributes now use atomic loads, and argument error reporting reads -``__qualname__`` after keyword comparisons, which may execute Python code and -replace it. Previously this could use a freed object. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-04-03-48.gh-issue-159288.vhqBQG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-04-03-48.gh-issue-159288.vhqBQG.rst deleted file mode 100644 index 2d969e8cbac8c4c..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-04-03-48.gh-issue-159288.vhqBQG.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash when walking a very long chain of lazy import placeholders, as -created by an aliased dotted import such as ``lazy import a.b.c as d``. -:exc:`RecursionError` is now raised instead. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-06-23-56.gh-issue-159294.9LTsNA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-06-23-56.gh-issue-159294.9LTsNA.rst deleted file mode 100644 index 9504f1f753cbec6..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-06-23-56.gh-issue-159294.9LTsNA.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in a debug build when creating an :exc:`ExceptionGroup` or -:exc:`BaseExceptionGroup` fails with :exc:`MemoryError`. Patch by Christian -Aurich Zanettini Martins. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-16-38-00.gh-issue-159320._Yhg1s.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-16-38-00.gh-issue-159320._Yhg1s.rst deleted file mode 100644 index 25e8745ccc02bb0..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-13-16-38-00.gh-issue-159320._Yhg1s.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add dynamic detection of stack sizes on Android, allowing safer deep -recursion. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-07-36-44.gh-issue-159399.cjuE0N.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-07-36-44.gh-issue-159399.cjuE0N.rst deleted file mode 100644 index f7020fee0cbc86a..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-07-36-44.gh-issue-159399.cjuE0N.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in :func:`codecs.encode` and :func:`codecs.decode` in the -:term:`free-threaded build` when another thread concurrently unregisters a -codec search function. Patch by Christian Aurich Zanettini Martins. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-11-40-06.gh-issue-159430.OWpyPm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-11-40-06.gh-issue-159430.OWpyPm.rst deleted file mode 100644 index fb4a6d6da4568a6..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-11-40-06.gh-issue-159430.OWpyPm.rst +++ /dev/null @@ -1,2 +0,0 @@ -Report the correct exception if looking up :meth:`~object.__bytes__` raises -an exception while performing %-formatting of :class:`bytes`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-18-45-35.gh-issue-159431.mVTMsA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-18-45-35.gh-issue-159431.mVTMsA.rst deleted file mode 100644 index 7ed642e4c707918..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-18-45-35.gh-issue-159431.mVTMsA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix an assertion failure, or an immortal object becoming mortal, on the -:term:`free threaded ` build when an object passed to the -internal delayed decref mechanism is immortal or becomes immortal before the -delayed decref is processed by the garbage collector. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-19-16-53.gh-issue-159434.T4AsrQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-19-16-53.gh-issue-159434.T4AsrQ.rst deleted file mode 100644 index f7d7fbffa3371e3..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-19-16-53.gh-issue-159434.T4AsrQ.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix reading a function's :attr:`~function.__annotations__` using borrowed -references to :attr:`~function.__annotate__` and the internal annotations -tuple while running code that can replace them. If ``__annotations__`` is -set while ``__annotate__`` runs, the newly set value is now kept. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-23-05-00.gh-issue-132042.k7Qs2w.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-23-05-00.gh-issue-132042.k7Qs2w.rst deleted file mode 100644 index 887df4fca00a789..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-23-05-00.gh-issue-132042.k7Qs2w.rst +++ /dev/null @@ -1,3 +0,0 @@ -Speed up class creation by resolving the special method slots of a new class -with one pass over each class dict along the MRO instead of one lookup per -slot name and per class. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-10-15-27.gh-issue-159449.xsdzFf.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-10-15-27.gh-issue-159449.xsdzFf.rst deleted file mode 100644 index f592e76fdbff707..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-10-15-27.gh-issue-159449.xsdzFf.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a free-threaded asyncio race leading to segfaults. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-10-00.gh-issue-87613.Wr4fvc.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-10-00.gh-issue-87613.Wr4fvc.rst deleted file mode 100644 index 8a57b1443365e7f..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-10-00.gh-issue-87613.Wr4fvc.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up creating :class:`weakref.ref` objects by adding a vectorcall implementation of the constructor generated by Argument Clinic. Patch by Pieter Eendebak. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-36-09.gh-issue-130279.N9N-sw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-36-09.gh-issue-130279.N9N-sw.rst deleted file mode 100644 index 996eb7020f53449..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-36-09.gh-issue-130279.N9N-sw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix bug where compiler did not set the right exception handler for some jump -instructions. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-40-00.gh-issue-87613.Sl1ceV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-40-00.gh-issue-87613.Sl1ceV.rst deleted file mode 100644 index bd96edc0b9f1770..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-12-40-00.gh-issue-87613.Sl1ceV.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up calling :class:`slice` by adding a vectorcall implementation of the constructor generated by Argument Clinic. Patch by Pieter Eendebak. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-16-18-25-48.gh-issue-159564.WMRRFA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-16-18-25-48.gh-issue-159564.WMRRFA.rst deleted file mode 100644 index 8e59f52fa4090ee..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-16-18-25-48.gh-issue-159564.WMRRFA.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve the performance and thread scaling of dereferencing weak references -in the :term:`free-threaded build`. Dereferencing a weak reference no longer -takes a lock. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-17-02-47-26.gh-issue-159603.OFagJz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-17-02-47-26.gh-issue-159603.OFagJz.rst deleted file mode 100644 index bd46b85509dbc2b..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-17-02-47-26.gh-issue-159603.OFagJz.rst +++ /dev/null @@ -1,2 +0,0 @@ -On FreeBSD, call ``pthread_attr_get_np()`` to get the stack size (and guard -size) of the current thread. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-18-00-30-00.gh-issue-159652.Qk3vLm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-18-00-30-00.gh-issue-159652.Qk3vLm.rst deleted file mode 100644 index 2b213c983b87187..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-18-00-30-00.gh-issue-159652.Qk3vLm.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix a hang when an import fails with :exc:`RecursionError` because the C -stack is nearly exhausted: releasing the import locks could itself fail and -leave the global import lock or a module lock held forever, so that later -imports in other threads blocked indefinitely. The import system now raises -:exc:`RecursionError` before taking an import lock when there is not enough -stack left to release it. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-16-38-39.gh-issue-159813.wHfSOc.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-16-38-39.gh-issue-159813.wHfSOc.rst deleted file mode 100644 index 7d5f0c8f2f53540..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-16-38-39.gh-issue-159813.wHfSOc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a fast path for :const:`None` in :c:func:`PyUnicodeWriter_WriteRepr`. -Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.09590b.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.09590b.rst deleted file mode 100644 index 3231be417aa80ec..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.09590b.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid a crash when a module filename cannot be encoded as UTF-8. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.173d49.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.173d49.rst deleted file mode 100644 index a552d06ff7bb898..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.173d49.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid a crash if formatting an integer with the ``c`` presentation type runs -out of memory. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.1fc3af.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.1fc3af.rst deleted file mode 100644 index a02631f2da2f9b2..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.1fc3af.rst +++ /dev/null @@ -1,2 +0,0 @@ -Preserve the original :exc:`SyntaxError` if allocating optional error metadata -fails. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.a8775b.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.a8775b.rst deleted file mode 100644 index 73c2e9bd5d36b95..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.a8775b.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid a crash when an interpreter configuration string cannot be encoded as -UTF-8. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.c25c27.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.c25c27.rst deleted file mode 100644 index b48d1e76954193a..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.c25c27.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leaks during interpreter startup when a path configuration file is -empty or too large, or when an audit hook rejects a command. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.f81734.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.f81734.rst deleted file mode 100644 index 0342bdef3e86994..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-19-33-35.gh-issue-159739.f81734.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash in :func:`warnings.warn` when ``skip_file_prefixes`` is an empty tuple subclass. -Propagate allocation errors when creating the fallback filename. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.55fd08.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.55fd08.rst deleted file mode 100644 index e574bab861b740c..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.55fd08.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a reference leak in :func:`gc.get_objects` and :func:`gc.get_referrers` -when an internal allocation fails in the free-threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.78e530.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.78e530.rst deleted file mode 100644 index 999efdac87f9745..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.78e530.rst +++ /dev/null @@ -1,2 +0,0 @@ -Raise :exc:`OverflowError` when the absolute value of a dynamic printf-style -formatting width is too large. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.b62fc9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.b62fc9.rst deleted file mode 100644 index 64d84ae260bb3b9..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.b62fc9.rst +++ /dev/null @@ -1,2 +0,0 @@ -Propagate errors during generic class construction and avoid a crash after -failed generic alias substitution. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.fc0ed2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.fc0ed2.rst deleted file mode 100644 index 6a549a6ded5e6a5..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-20-38-17.gh-issue-159739.fc0ed2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid a crash if allocation fails while creating a -:class:`string.templatelib.Template` iterator. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-21-30-00.gh-issue-159860.qsbr128.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-21-30-00.gh-issue-159860.qsbr128.rst deleted file mode 100644 index fc0d6fca4f79962..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-19-21-30-00.gh-issue-159860.qsbr128.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix false sharing between the per-thread QSBR states in the -:term:`free-threaded build` on AArch64 CPUs with 128-byte cache lines, such -as Apple Silicon. CPU-bound threads that share no objects now scale like -processes there. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.848b18.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.848b18.rst deleted file mode 100644 index 07f842471b4d0f2..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.848b18.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow the garbage collector to break reference cycles involving property -accessors or names. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.a3cc5d.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.a3cc5d.rst deleted file mode 100644 index 69c53e84e216b33..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.a3cc5d.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a crash when constructing an error for invalid generic alias arguments. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.bf569b.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.bf569b.rst deleted file mode 100644 index 797d2de2240ef3c..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.bf569b.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a reference leak in binary, ternary, and comparison operations on weak -reference proxies when another operand is a dead proxy. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.d76e6d.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.d76e6d.rst deleted file mode 100644 index a027883426be196..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.d76e6d.rst +++ /dev/null @@ -1,2 +0,0 @@ -Prevent free-threaded builds from interning and immortalizing complex -subclasses in code object constants. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.da0f04.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.da0f04.rst deleted file mode 100644 index 386eb5535b70252..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-00-00-00.gh-issue-159739.da0f04.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix reference leaks when free-threaded garbage collection aborts early because -of an allocation error. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-10-00-00.gh-issue-158195.UnpinTL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-10-00-00.gh-issue-158195.UnpinTL.rst deleted file mode 100644 index f189c7c563ca58c..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-10-00-00.gh-issue-158195.UnpinTL.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fix a deadlock in the free-threaded build between a thread that stops the -world while running Python code under the type lock (for example a custom -metaclass ``mro()`` or the ``__eq__`` of a :class:`str` subclass used as a -class ``__dict__`` key) and a thread that assigns to a special method or to -``__bases__`` of a class. The type lock is no longer held while waiting -for a stop-the-world; the prepared slot, MRO and bases updates are -validated with the world stopped and recomputed if they went stale. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-20-38-38.gh-issue-159101.vFwrZY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-20-38-38.gh-issue-159101.vFwrZY.rst deleted file mode 100644 index 33beb8d571d7eca..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-20-38-38.gh-issue-159101.vFwrZY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix use-after-free when :meth:`~object.__del__` changes the base class. Patch by -Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-22-00-00.gh-issue-159963.pLkHld.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-22-00-00.gh-issue-159963.pLkHld.rst deleted file mode 100644 index 5c3bd8f63f1d4fb..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-20-22-00-00.gh-issue-159963.pLkHld.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix a regression in Python 3.15.0 where threads importing submodules of -packages for the first time could raise ``_DeadlockError`` or a spurious -"partially initialized module" :exc:`ImportError`, or block one another, with -no circular import involved. :func:`importlib.import_module` now returns the -partially initialized module in a cross-thread circular import, as the -:keyword:`import` statement does, instead of raising ``_DeadlockError``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-14-00-00.gh-issue-159493.sfLf1b.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-14-00-00.gh-issue-159493.sfLf1b.rst deleted file mode 100644 index 35289d995b3d357..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-14-00-00.gh-issue-159493.sfLf1b.rst +++ /dev/null @@ -1,3 +0,0 @@ -Superinstructions are now formed after the compiler decides which -:opcode:`LOAD_FAST` instructions can borrow their reference. -This fixes reference count contention slowdown on the free-threaded build for single-line loops and comprehensions. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-18-00-00.gh-issue-160368.sem-clockwait.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-18-00-00.gh-issue-160368.sem-clockwait.rst deleted file mode 100644 index 11d7467b38413af..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-18-00-00.gh-issue-160368.sem-clockwait.rst +++ /dev/null @@ -1,3 +0,0 @@ -On Linux, allow timed semaphore waits to use the monotonic clock when -``sem_clockwait`` is available at runtime, even if Python was built with -older libc headers. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-19-55-00.gh-issue-159739.codc01.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-19-55-00.gh-issue-159739.codc01.rst deleted file mode 100644 index 985d4264befae2e..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-19-55-00.gh-issue-159739.codc01.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash in free-threaded builds when :func:`codecs.unregister` is -called concurrently. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-19-55-00.gh-issue-159739.tali01.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-19-55-00.gh-issue-159739.tali01.rst deleted file mode 100644 index a4f672fed87cfe7..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-19-55-00.gh-issue-159739.tali01.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a reference leak when the value of a type alias is evaluated -recursively or concurrently. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.Aw61LR.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.Aw61LR.rst deleted file mode 100644 index 5de64814b1ca55d..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.Aw61LR.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a race in ``list.__sizeof__()`` when a list is concurrently resized on a free-threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.CbTE4N.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.CbTE4N.rst deleted file mode 100644 index 20072791808dd1f..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.CbTE4N.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a possible crash when assigning metadata to :class:`classmethod` or :class:`staticmethod` objects. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.Zw43uX.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.Zw43uX.rst deleted file mode 100644 index 628a146b059a436..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-21-26-10.gh-issue-159739.Zw43uX.rst +++ /dev/null @@ -1 +0,0 @@ -Raise :exc:`MemoryError` instead of returning an incorrect result if :func:`round` runs out of memory while rounding a float. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-23-18-38.gh-issue-160132.Nj4rQa.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-23-18-38.gh-issue-160132.Nj4rQa.rst deleted file mode 100644 index 156f387f7a69983..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-21-23-18-38.gh-issue-160132.Nj4rQa.rst +++ /dev/null @@ -1,3 +0,0 @@ -Reduce memory usage during compilation by reordering the fields of the -internal basic block structure to reduce padding. -Patch by Shamil Abdulaev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-00-30-02.gh-issue-160179.dQ2e7m.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-00-30-02.gh-issue-160179.dQ2e7m.rst deleted file mode 100644 index 5377e59a60ea48d..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-00-30-02.gh-issue-160179.dQ2e7m.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :meth:`!bytearray.__init__` leaving uninitialized memory in the object -when it fails on a :class:`list` or :class:`tuple` containing an integer not -in ``range(256)``. The object is now empty after the failed call. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-00-30-21.gh-issue-160162.lShf32.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-00-30-21.gh-issue-160162.lShf32.rst deleted file mode 100644 index 5aa2021bddd44ee..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-00-30-21.gh-issue-160162.lShf32.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash on 32-bit platforms when left shifting an :class:`int` by a -count close to ``sys.maxsize * sys.int_info.bits_per_digit``. -:exc:`OverflowError` is now raised, as for other too large counts. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-12-30-11.gh-issue-106045.fkMW34.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-12-30-11.gh-issue-106045.fkMW34.rst deleted file mode 100644 index 98abf1debe08d6d..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-12-30-11.gh-issue-106045.fkMW34.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix :mod:`venv` path discovery when the base executable resolves to a -location outside the installation pointed to by the ``home`` key in -``pyvenv.cfg``. The standard library is now found from ``home``, and -extension modules from the resolved executable's installation, instead of -falling back to the compiled-in prefix. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-14-45-00.gh-issue-160245.mEmO24.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-14-45-00.gh-issue-160245.mEmO24.rst deleted file mode 100644 index f62d937f556caa8..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-14-45-00.gh-issue-160245.mEmO24.rst +++ /dev/null @@ -1,2 +0,0 @@ -Reduce the size of parser memo entries on 64-bit platforms by reordering -fields to remove padding. Patch by Shamil Abdulaev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst deleted file mode 100644 index 7533a25e0b57a50..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a crash in :mod:`tokenize` when a mismatched closing bracket follows the opening brace of an f-string or t-string replacement field, as in ``f'{]a'``. It now raises the same "unmatched" error as the parser. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-37-00.gh-issue-160050.aB3x9Q.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-37-00.gh-issue-160050.aB3x9Q.rst deleted file mode 100644 index 9089e4bc4dbb14b..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-37-00.gh-issue-160050.aB3x9Q.rst +++ /dev/null @@ -1 +0,0 @@ -Use asymptotically faster :func:`divmod` implementation for integer modulo of large integers. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-20-15-24.gh-issue-160491.yIQ_Vf.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-20-15-24.gh-issue-160491.yIQ_Vf.rst deleted file mode 100644 index 2227224ba0fa2c1..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-20-15-24.gh-issue-160491.yIQ_Vf.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix the location reported for invalid escape sequences in f-strings and -t-strings. The :exc:`SyntaxError` now points at the literal part that -contains the escape, instead of at the closing quote. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst deleted file mode 100644 index 98628f6a74208f7..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst +++ /dev/null @@ -1,3 +0,0 @@ -Speed up :func:`ast.parse` by ~15.8% and reduce the memory used by ~51.6% by -storing node attributes as inline values instead of in a per-node -dictionary. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-12-00-00.gh-issue-160682.Q7mK2x.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-12-00-00.gh-issue-160682.Q7mK2x.rst deleted file mode 100644 index 5c783665e789659..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-12-00-00.gh-issue-160682.Q7mK2x.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug in the :term:`free-threaded build` where garbage collection could -prematurely collect the function object of a running method. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-28-13-47-43.gh-issue-160773.7VFxkA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-28-13-47-43.gh-issue-160773.7VFxkA.rst deleted file mode 100644 index 09d282e1dbeee7d..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-28-13-47-43.gh-issue-160773.7VFxkA.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash in :func:`str` of an :exc:`OSError` when formatting one of its -attributes modifies the others. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-02-05-50-35.gh-issue-160872.x8NvzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-02-05-50-35.gh-issue-160872.x8NvzN.rst deleted file mode 100644 index 2911abe2f166f1c..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-02-05-50-35.gh-issue-160872.x8NvzN.rst +++ /dev/null @@ -1 +0,0 @@ -Optimize division and remainder operations on large :class:`int` values. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-05-52-49.gh-issue-161229.8pOPOI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-05-52-49.gh-issue-161229.8pOPOI.rst deleted file mode 100644 index 3a9eae1e4af91ba..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-05-52-49.gh-issue-161229.8pOPOI.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`!__lazy_import__` now treats ``None`` for *globals* and *locals* the -same as omitting them, matching its signature. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-13-44-26.gh-issue-159184.X_sOdY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-13-44-26.gh-issue-159184.X_sOdY.rst deleted file mode 100644 index 651ac1d0b9bae9f..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-13-44-26.gh-issue-159184.X_sOdY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a leak of NaN constants in code objects created by :func:`marshal.loads` -on the free-threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-20-28-15.gh-issue-161288.LikC_U.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-20-28-15.gh-issue-161288.LikC_U.rst deleted file mode 100644 index 4cf8201d02ce7c8..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-03-20-28-15.gh-issue-161288.LikC_U.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in the perf trampoline (:option:`-X perf <-X>`) on builds with -clang's ``-fsanitize=function``, which ``-fsanitize=undefined`` enables -since clang 17. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-10-19-48.gh-issue-161419.wc4DV0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-10-19-48.gh-issue-161419.wc4DV0.rst deleted file mode 100644 index 0d6605a9eeb5503..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-10-19-48.gh-issue-161419.wc4DV0.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in the :term:`free-threaded build` where a weak reference could -revive an object whose finalizer had already run during deallocation, causing -it to be deallocated twice. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-21-00-57.gh-issue-161366.ouxAvv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-21-00-57.gh-issue-161366.ouxAvv.rst deleted file mode 100644 index 94098a0cf5195cb..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-05-21-00-57.gh-issue-161366.ouxAvv.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a deadlock in the garbage collector of the :term:`free-threaded build` -when :data:`gc.DEBUG_COLLECTABLE` is set and another thread is writing to -:data:`sys.stderr`. The debug output is now written after the other threads -are resumed. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst deleted file mode 100644 index a259af1dcea0f95..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-02-30-00.gh-issue-129724.Mn8sLq.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix the :mod:`marshal` output of compiled code, and so ``.pyc`` files, -differing for the same source depending on what ran before in the process. -This affected private names in classes with non-ASCII names, and -:func:`compile` of an AST with names not created by :func:`ast.parse`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-15-15-00.gh-issue-161433.A9b3Kq.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-15-15-00.gh-issue-161433.A9b3Kq.rst deleted file mode 100644 index 251837ac6d632a4..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-15-15-00.gh-issue-161433.A9b3Kq.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add context notes to :exc:`TypeError` raised while hashing a -:class:`tuple`, identifying the index of the item whose hash failed. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-21-21-04.gh-issue-161242.d4vNgC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-21-21-04.gh-issue-161242.d4vNgC.rst deleted file mode 100644 index 3dbd521612feedb..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-06-21-21-04.gh-issue-161242.d4vNgC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Speed up :exc:`AttributeError`, :exc:`ImportError` and -:exc:`NameError` by enabling :ref:`vectorcall`. Unexpected -keyword arguments are now reported by name. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-07-09-30-00.gh-issue-161420.mIdFre.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-07-09-30-00.gh-issue-161420.mIdFre.rst deleted file mode 100644 index 296a2267f397be0..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-07-09-30-00.gh-issue-161420.mIdFre.rst +++ /dev/null @@ -1,2 +0,0 @@ -Restore ``mimalloc``'s previous behavior of processing delayed frees from other -threads on every generic malloc call, reverting the change introduced in v2.2.2. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst deleted file mode 100644 index c4467c7b64cddf1..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-14-30-00.gh-issue-160267.sTfrEe.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a crash when :mod:`profiling.sampling` profiles a process that creates and -destroys threads or subinterpreters quickly. Memory of deleted thread and -interpreter states is now reused for new states instead of being freed until -the runtime is finalized. diff --git a/Misc/NEWS.d/next/Documentation/2023-09-04-16-01-20.gh-issue-107689.LFLFtG.rst b/Misc/NEWS.d/next/Documentation/2023-09-04-16-01-20.gh-issue-107689.LFLFtG.rst deleted file mode 100644 index 5c60e29bc0b44a3..000000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-09-04-16-01-20.gh-issue-107689.LFLFtG.rst +++ /dev/null @@ -1 +0,0 @@ -Added docstrings to abstract ctypes. diff --git a/Misc/NEWS.d/next/Documentation/2025-08-31-02-28-21.gh-issue-138277.kkORph.rst b/Misc/NEWS.d/next/Documentation/2025-08-31-02-28-21.gh-issue-138277.kkORph.rst deleted file mode 100644 index 198d1de3121c5b2..000000000000000 --- a/Misc/NEWS.d/next/Documentation/2025-08-31-02-28-21.gh-issue-138277.kkORph.rst +++ /dev/null @@ -1,2 +0,0 @@ -Rearranged documentation search results to emphasize built-in types and -functions. Patch by Adam Hartz. diff --git a/Misc/NEWS.d/next/IDLE/2026-10-16-15-12-36.gh-issue-159517.uzRtAd.rst b/Misc/NEWS.d/next/IDLE/2026-10-16-15-12-36.gh-issue-159517.uzRtAd.rst deleted file mode 100644 index f5e506c3e1f96e5..000000000000000 --- a/Misc/NEWS.d/next/IDLE/2026-10-16-15-12-36.gh-issue-159517.uzRtAd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the completion list and the call tip grabbing the focus on macOS with -Tcl/Tk 9.1. diff --git a/Misc/NEWS.d/next/IDLE/2026-11-06-17-12-40.gh-issue-161477.smXIKC.rst b/Misc/NEWS.d/next/IDLE/2026-11-06-17-12-40.gh-issue-161477.smXIKC.rst deleted file mode 100644 index ca955e608bc8936..000000000000000 --- a/Misc/NEWS.d/next/IDLE/2026-11-06-17-12-40.gh-issue-161477.smXIKC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix ``repr()`` of ``idlelib.redirector.WidgetRedirector`` and -``idlelib.redirector.OriginalCommand``, which displayed a spurious -one-element tuple. diff --git a/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst b/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst deleted file mode 100644 index db8366f6ef0f099..000000000000000 --- a/Misc/NEWS.d/next/Library/2021-05-24-14-41-29.bpo-44170._v28mv.rst +++ /dev/null @@ -1 +0,0 @@ -Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList. diff --git a/Misc/NEWS.d/next/Library/2021-06-11-20-00-16.bpo-19094.rMRoIL.rst b/Misc/NEWS.d/next/Library/2021-06-11-20-00-16.bpo-19094.rMRoIL.rst deleted file mode 100644 index 53cae4448ed9d54..000000000000000 --- a/Misc/NEWS.d/next/Library/2021-06-11-20-00-16.bpo-19094.rMRoIL.rst +++ /dev/null @@ -1,6 +0,0 @@ -The :mod:`urllib.parse` URL parsing functions now raise :exc:`TypeError` -instead of :exc:`AttributeError` when passed an object which is not a string, -bytes, or ``None``. Accepting objects with false values (like ``0`` and -``[]``) except empty strings, byte-like objects and ``None`` is now -deprecated, extending the deprecation made in Python 3.14 for -:func:`~urllib.parse.parse_qsl` and :func:`~urllib.parse.parse_qs`. diff --git a/Misc/NEWS.d/next/Library/2023-01-26-09-36-24.gh-issue-61448.SOAkvt.rst b/Misc/NEWS.d/next/Library/2023-01-26-09-36-24.gh-issue-61448.SOAkvt.rst deleted file mode 100644 index 6ea99ba3310b7cd..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-26-09-36-24.gh-issue-61448.SOAkvt.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :func:`inspect.formatargvalues` raising :exc:`KeyError` for deleted -arguments, including ``*args`` and ``**kwargs``, by displaying ````. -Patch by Ezio Melotti and Furkan Onder. diff --git a/Misc/NEWS.d/next/Library/2024-12-27-15-41-55.gh-issue-128289.-hCzEc.rst b/Misc/NEWS.d/next/Library/2024-12-27-15-41-55.gh-issue-128289.-hCzEc.rst deleted file mode 100644 index 015512afad6ed38..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-12-27-15-41-55.gh-issue-128289.-hCzEc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add *eager_tasks* parameter to :func:`asyncio.run` function and -:class:`asyncio.Runner` class constructor. diff --git a/Misc/NEWS.d/next/Library/2025-02-04-09-01-06.gh-issue-129605.wKGNXQ.rst b/Misc/NEWS.d/next/Library/2025-02-04-09-01-06.gh-issue-129605.wKGNXQ.rst deleted file mode 100644 index 9dcf05913870b92..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-02-04-09-01-06.gh-issue-129605.wKGNXQ.rst +++ /dev/null @@ -1 +0,0 @@ -Improve exception handling when computing suggestions in traceback.py. This fixes a bug that could cause PyREPL to exit unexpectedly. diff --git a/Misc/NEWS.d/next/Library/2025-03-13-20-21-00.gh-issue-115810.qwrb231.rst b/Misc/NEWS.d/next/Library/2025-03-13-20-21-00.gh-issue-115810.qwrb231.rst deleted file mode 100644 index 253f4a5de95738a..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-03-13-20-21-00.gh-issue-115810.qwrb231.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :attr:`datetime.timezone.name`, the *name* passed to the constructor, -or ``None`` if no name was passed. diff --git a/Misc/NEWS.d/next/Library/2025-06-12-15-39-48.gh-issue-89730.Lu35Fu.rst b/Misc/NEWS.d/next/Library/2025-06-12-15-39-48.gh-issue-89730.Lu35Fu.rst deleted file mode 100644 index bf4f8cfc476d2f1..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-12-15-39-48.gh-issue-89730.Lu35Fu.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add support for Encrypted Client Hello (ECH) to the :mod:`ssl` module with -OpenSSL 4.0 or later. diff --git a/Misc/NEWS.d/next/Library/2025-09-18-14-13-00.gh-issue-139122.m3lp66.rst b/Misc/NEWS.d/next/Library/2025-09-18-14-13-00.gh-issue-139122.m3lp66.rst deleted file mode 100644 index cb656aaf793c630..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-18-14-13-00.gh-issue-139122.m3lp66.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`uuid`: Reimplement :class:`~uuid.UUID` objects, :func:`~uuid.uuid4`, -and :func:`~uuid.uuid7` in C. diff --git a/Misc/NEWS.d/next/Library/2025-09-18-18-03-32.gh-issue-139105.1QQ_W4.rst b/Misc/NEWS.d/next/Library/2025-09-18-18-03-32.gh-issue-139105.1QQ_W4.rst deleted file mode 100644 index 0494a66b094eee9..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-18-18-03-32.gh-issue-139105.1QQ_W4.rst +++ /dev/null @@ -1 +0,0 @@ -Correct the EOF key shown in the :mod:`sqlite3` CLI ``.help`` text on Windows. diff --git a/Misc/NEWS.d/next/Library/2025-10-13-15-43-09.gh-issue-140036.b_59uN.rst b/Misc/NEWS.d/next/Library/2025-10-13-15-43-09.gh-issue-140036.b_59uN.rst deleted file mode 100644 index 4bd92b4d7352251..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-13-15-43-09.gh-issue-140036.b_59uN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid hanging in floor division of pure Python :class:`decimal.Decimal` -instances when the context precision is very large. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-10-20-13-08-04.gh-issue-84027.HjBO0B.rst b/Misc/NEWS.d/next/Library/2025-10-20-13-08-04.gh-issue-84027.HjBO0B.rst deleted file mode 100644 index 060eeabd903bbb8..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-20-13-08-04.gh-issue-84027.HjBO0B.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`shutil` now can unpack wheel packages using the ``whl`` format. diff --git a/Misc/NEWS.d/next/Library/2026-01-06-09-13-53.gh-issue-84644.V_cYP3.rst b/Misc/NEWS.d/next/Library/2026-01-06-09-13-53.gh-issue-84644.V_cYP3.rst deleted file mode 100644 index a1f4a598c0e0701..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-01-06-09-13-53.gh-issue-84644.V_cYP3.rst +++ /dev/null @@ -1,5 +0,0 @@ -Deprecate using a return annotation to infer the dispatch type for -:func:`functools.singledispatch` and -:class:`functools.singledispatchmethod`. It will raise :exc:`TypeError` in -Python 3.21. Annotate the dispatch parameter or pass the dispatch type -explicitly instead. Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Library/2026-03-23-00-04-13.gh-issue-146065.FIdn8D.rst b/Misc/NEWS.d/next/Library/2026-03-23-00-04-13.gh-issue-146065.FIdn8D.rst deleted file mode 100644 index db91ce76198c01e..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-03-23-00-04-13.gh-issue-146065.FIdn8D.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash in asyncio.Future iterator when send() is -called after throw() or close(). diff --git a/Misc/NEWS.d/next/Library/2026-03-28-10-27-46.gh-issue-146169.RBF1xp.rst b/Misc/NEWS.d/next/Library/2026-03-28-10-27-46.gh-issue-146169.RBF1xp.rst deleted file mode 100644 index d9460540a5a7d38..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-03-28-10-27-46.gh-issue-146169.RBF1xp.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`xml.parsers.expat`: raise :exc:`RuntimeError` when an Expat handler -calls :meth:`parser.Parse ` or -:meth:`parser.ParseFile ` on the parser -that called the handler. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-03-28-14-34-18.gh-issue-143376.FvWTwi.rst b/Misc/NEWS.d/next/Library/2026-03-28-14-34-18.gh-issue-143376.FvWTwi.rst deleted file mode 100644 index 981e0ede58fde87..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-03-28-14-34-18.gh-issue-143376.FvWTwi.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`concurrent.interpreters`: fix a crash when internal cross-interpreter -channel ends have incorrect channel IDs. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-04-27-20-30-16.gh-issue-149083.xcmpSO.rst b/Misc/NEWS.d/next/Library/2026-04-27-20-30-16.gh-issue-149083.xcmpSO.rst deleted file mode 100644 index 38b8c904a68e25b..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-27-20-30-16.gh-issue-149083.xcmpSO.rst +++ /dev/null @@ -1,2 +0,0 @@ -:data:`functools.Placeholder` is now a :class:`sentinel`. It is no longer an -instance of its own singleton type. diff --git a/Misc/NEWS.d/next/Library/2026-05-03-22-52-07.gh-issue-149309.gXhk9Y.rst b/Misc/NEWS.d/next/Library/2026-05-03-22-52-07.gh-issue-149309.gXhk9Y.rst deleted file mode 100644 index 69df9f1f6daff3f..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-03-22-52-07.gh-issue-149309.gXhk9Y.rst +++ /dev/null @@ -1 +0,0 @@ -:exc:`bdb.BdbQuit` is now a subclass of :exc:`BaseException`, instead of :exc:`Exception`, to allow it to be used for cleanly exiting the debugger without being accidentally caught by user code. diff --git a/Misc/NEWS.d/next/Library/2026-05-09-12-58-04.gh-issue-149527.hX36Yh.rst b/Misc/NEWS.d/next/Library/2026-05-09-12-58-04.gh-issue-149527.hX36Yh.rst deleted file mode 100644 index d75b285554c0fa8..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-09-12-58-04.gh-issue-149527.hX36Yh.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`multiprocessing`: fix creation of temporary socket files following -:gh:`137335` (`PR#148578 `__ -and corresponding backports). Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-05-21-16-15-01.gh-issue-105578.oq-0pg.rst b/Misc/NEWS.d/next/Library/2026-05-21-16-15-01.gh-issue-105578.oq-0pg.rst deleted file mode 100644 index 98b6f5abaeb94b5..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-21-16-15-01.gh-issue-105578.oq-0pg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Deprecate :data:`typing.AnyStr` type variable and raise a warning when it -is imported. diff --git a/Misc/NEWS.d/next/Library/2026-06-18-11-26-00.gh-issue-151633.uJzHdc.rst b/Misc/NEWS.d/next/Library/2026-06-18-11-26-00.gh-issue-151633.uJzHdc.rst deleted file mode 100644 index 73d533c0d109bbe..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-06-18-11-26-00.gh-issue-151633.uJzHdc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a use-after-free under free-threading when multiple threads -update the same :class:`~collections.Counter` concurrently. diff --git a/Misc/NEWS.d/next/Library/2026-06-27-08-31-30.gh-issue-152373.3cd0aa.rst b/Misc/NEWS.d/next/Library/2026-06-27-08-31-30.gh-issue-152373.3cd0aa.rst deleted file mode 100644 index bf6a70c1374a0db..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-06-27-08-31-30.gh-issue-152373.3cd0aa.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the :mod:`pdb` ``until`` command not stepping over a list comprehension -when a breakpoint is set on it. Patch by tonghuaroot. diff --git a/Misc/NEWS.d/next/Library/2026-06-28-14-37-01.gh-issue-152490.A8qLbF.rst b/Misc/NEWS.d/next/Library/2026-06-28-14-37-01.gh-issue-152490.A8qLbF.rst deleted file mode 100644 index 704295749909974..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-06-28-14-37-01.gh-issue-152490.A8qLbF.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :exc:`TypeError` when formatting a :exc:`SyntaxError` raised with :data:`NotImplemented` as its argument, which caused unexpected REPL exit and IDLE Shell restart. diff --git a/Misc/NEWS.d/next/Library/2026-06-30-08-39-07.gh-issue-62222.Zq7x2P.rst b/Misc/NEWS.d/next/Library/2026-06-30-08-39-07.gh-issue-62222.Zq7x2P.rst deleted file mode 100644 index d70b183b7cfa502..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-06-30-08-39-07.gh-issue-62222.Zq7x2P.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix :func:`binascii.a2b_qp` and :mod:`quopri` discarding the character after -a ``=`` that does not start a valid escape: ``b'==41'`` now decodes to -``b'=A'``, as in the :mod:`email` package. Add a keyword-only *strip_ws* -parameter to :func:`binascii.a2b_qp`, :func:`quopri.decode` and -:func:`quopri.decodestring` to strip trailing whitespace from each line, as -required by :rfc:`2045`. diff --git a/Misc/NEWS.d/next/Library/2026-07-04-12-52-22.gh-issue-149816.XQOutB.rst b/Misc/NEWS.d/next/Library/2026-07-04-12-52-22.gh-issue-149816.XQOutB.rst deleted file mode 100644 index eb273865894aae0..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-07-04-12-52-22.gh-issue-149816.XQOutB.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`hashlib`: fix a crash arising from a race condition when looking -up an OpenSSL hash algorithm by name the first time. Patch by Bénédikt Tran. - diff --git a/Misc/NEWS.d/next/Library/2026-07-11-16-00-00.gh-issue-153605.Rk7Xz3.rst b/Misc/NEWS.d/next/Library/2026-07-11-16-00-00.gh-issue-153605.Rk7Xz3.rst deleted file mode 100644 index a08fde3045c0778..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-07-11-16-00-00.gh-issue-153605.Rk7Xz3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :class:`gettext.GNUTranslations` raising :exc:`struct.error` instead of -:exc:`OSError` when reading a malformed ``.mo`` file. Patch by tonghuaroot. diff --git a/Misc/NEWS.d/next/Library/2026-07-13-00-10-00.gh-issue-153679.RJ2WGB.rst b/Misc/NEWS.d/next/Library/2026-07-13-00-10-00.gh-issue-153679.RJ2WGB.rst deleted file mode 100644 index d612903f2250211..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-07-13-00-10-00.gh-issue-153679.RJ2WGB.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`zipimport.zipimporter` now raises :exc:`zipimport.ZipImportError` -instead of :exc:`UnicodeDecodeError` for a UTF-8-flagged file name that is not -valid UTF-8. Patch by tonghuaroot. diff --git a/Misc/NEWS.d/next/Library/2026-07-16-21-30-00.gh-issue-74876.wR3fXk.rst b/Misc/NEWS.d/next/Library/2026-07-16-21-30-00.gh-issue-74876.wR3fXk.rst deleted file mode 100644 index aef5cea67047fc5..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-07-16-21-30-00.gh-issue-74876.wR3fXk.rst +++ /dev/null @@ -1,8 +0,0 @@ -Weak references (:class:`weakref.ref`), :class:`weakref.WeakMethod`, -:class:`weakref.WeakSet`, :class:`weakref.WeakValueDictionary` and -:class:`weakref.WeakKeyDictionary` can now be pickled -if the weakly referenced objects are strongly referenced -elsewhere in the same pickle; -otherwise pickling raises :exc:`pickle.PicklingError`. -:class:`weakref.WeakMethod` and :class:`weakref.WeakSet` -can now also be copied. diff --git a/Misc/NEWS.d/next/Library/2026-07-21-00-02-11.gh-issue-148228.D3rP8m.rst b/Misc/NEWS.d/next/Library/2026-07-21-00-02-11.gh-issue-148228.D3rP8m.rst deleted file mode 100644 index d00d072fb2da08b..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-07-21-00-02-11.gh-issue-148228.D3rP8m.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :term:`REPL` now displays error messages when editing commands cannot be -performed, such as when tab completion finds no matches or a history search -fails. Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Library/2026-08-04-16-38-59.gh-issue-155152.qN1EEN.rst b/Misc/NEWS.d/next/Library/2026-08-04-16-38-59.gh-issue-155152.qN1EEN.rst deleted file mode 100644 index c7cff1866cc51f3..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-04-16-38-59.gh-issue-155152.qN1EEN.rst +++ /dev/null @@ -1 +0,0 @@ -Improve performance of some :mod:`zipimport` archives that use Zstandard compression. diff --git a/Misc/NEWS.d/next/Library/2026-08-05-10-13-23.gh-issue-153711.9g9dVz.rst b/Misc/NEWS.d/next/Library/2026-08-05-10-13-23.gh-issue-153711.9g9dVz.rst deleted file mode 100644 index ca825ce6917db9a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-05-10-13-23.gh-issue-153711.9g9dVz.rst +++ /dev/null @@ -1,3 +0,0 @@ -On iOS, add :func:`os.pipe2`, and use the syscalls :manpage:`pipe2 (2)` and -:manpage:`dup3 (2)` internally, when available at both build-time and -run-time. diff --git a/Misc/NEWS.d/next/Library/2026-08-09-14-06-27.gh-issue-155426.zZVPMd.rst b/Misc/NEWS.d/next/Library/2026-08-09-14-06-27.gh-issue-155426.zZVPMd.rst deleted file mode 100644 index f1839defbe56742..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-09-14-06-27.gh-issue-155426.zZVPMd.rst +++ /dev/null @@ -1,4 +0,0 @@ -Make sure that the pure Python fallback of :func:`hmac.digest` reject -keys that are neither :class:`bytes` or :class:`bytearray`. Previously, -an :exc:`AttributeError` was raised instead of :exc:`TypeError`. -Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-08-09-14-26-48.gh-issue-155428.wi0_fI.rst b/Misc/NEWS.d/next/Library/2026-08-09-14-26-48.gh-issue-155428.wi0_fI.rst deleted file mode 100644 index f0c6201cbe592ee..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-09-14-26-48.gh-issue-155428.wi0_fI.rst +++ /dev/null @@ -1,3 +0,0 @@ -Ensure that :func:`hmac.digest` and :meth:`hmac.HMAC.digest` output -identical values when using a digest's block size smaller than 16. Patch by -Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-08-15-11-30-42.gh-issue-155782.JUmDP4.rst b/Misc/NEWS.d/next/Library/2026-08-15-11-30-42.gh-issue-155782.JUmDP4.rst deleted file mode 100644 index cafa887044fa443..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-15-11-30-42.gh-issue-155782.JUmDP4.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`ssl`: prevent a crash in :meth:`~ssl.SSLSocket.group` -for session-less objects on OpenSSL 3.2 and later. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-08-15-11-49-30.gh-issue-155830.cY5xgK.rst b/Misc/NEWS.d/next/Library/2026-08-15-11-49-30.gh-issue-155830.cY5xgK.rst deleted file mode 100644 index 9d6c2a82903d027..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-15-11-49-30.gh-issue-155830.cY5xgK.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`!_blake2`: Ensure that BLAKE-2 objects are correctly initialized -before being tracked by the garbage collector. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-08-15-12-08-33.gh-issue-155833.Pi2Yjp.rst b/Misc/NEWS.d/next/Library/2026-08-15-12-08-33.gh-issue-155833.Pi2Yjp.rst deleted file mode 100644 index 96bf5de44bd8361..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-15-12-08-33.gh-issue-155833.Pi2Yjp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ensure that MD5 and SHA* objects are properly GC-untracked before freeing -their internal HACL* state. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-08-20-09-56-45.gh-issue-156078.-E7gZc.rst b/Misc/NEWS.d/next/Library/2026-08-20-09-56-45.gh-issue-156078.-E7gZc.rst deleted file mode 100644 index d2a3a219fa8958d..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-20-09-56-45.gh-issue-156078.-E7gZc.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`zipfile.ZipFile.mkdir` now raises :exc:`ValueError` when the archive -is closed or when a writable member handle is open. diff --git a/Misc/NEWS.d/next/Library/2026-08-21-06-00-00.gh-issue-156057.tarfile-wrap-comp-err.rst b/Misc/NEWS.d/next/Library/2026-08-21-06-00-00.gh-issue-156057.tarfile-wrap-comp-err.rst deleted file mode 100644 index 1c8d8839dbf7bd7..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-21-06-00-00.gh-issue-156057.tarfile-wrap-comp-err.rst +++ /dev/null @@ -1,5 +0,0 @@ -Wrap the underlying compression library's exception in :exc:`tarfile.ReadError` -when a member's compressed data is corrupt past its header, for all of -:mod:`tarfile`'s supported compression formats (gzip, bzip2, lzma, zstd). -Previously the raw exception (e.g. :exc:`zlib.error`) could leak through -:meth:`tarfile.TarFile.extractfile`. Patch by Jason Mak. diff --git a/Misc/NEWS.d/next/Library/2026-08-23-16-33-00.gh-issue-151126.GUvtG7.rst b/Misc/NEWS.d/next/Library/2026-08-23-16-33-00.gh-issue-151126.GUvtG7.rst deleted file mode 100644 index 01a491384695980..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-23-16-33-00.gh-issue-151126.GUvtG7.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash caused by failing to set :exc:`MemoryError` on allocation -failure when first using a Chinese, Japanese or Korean codec. diff --git a/Misc/NEWS.d/next/Library/2026-08-29-13-41-56.gh-issue-154675.cYvE-Z.rst b/Misc/NEWS.d/next/Library/2026-08-29-13-41-56.gh-issue-154675.cYvE-Z.rst deleted file mode 100644 index 02d4996702b4584..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-29-13-41-56.gh-issue-154675.cYvE-Z.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve import time of :mod:`pdb`, :mod:`bdb` and :mod:`code` by lazily -importing several dependencies. diff --git a/Misc/NEWS.d/next/Library/2026-08-29-14-28-46.gh-issue-156568.mkxLY8.rst b/Misc/NEWS.d/next/Library/2026-08-29-14-28-46.gh-issue-156568.mkxLY8.rst deleted file mode 100644 index 3efd5a0481874c3..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-08-29-14-28-46.gh-issue-156568.mkxLY8.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`zipapp.create_archive` now correctly sets the executable bit on the -target archive when copying a zipapp where the target is a path-like object. diff --git a/Misc/NEWS.d/next/Library/2026-09-01-00-00-00.gh-issue-156784.8gvtt4.rst b/Misc/NEWS.d/next/Library/2026-09-01-00-00-00.gh-issue-156784.8gvtt4.rst deleted file mode 100644 index 699448059114ece..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-01-00-00-00.gh-issue-156784.8gvtt4.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`asyncio.ProactorEventLoop` no longer silently ignores -:exc:`ConnectionResetError` on UDP sockets, matching -:class:`asyncio.SelectorEventLoop`. diff --git a/Misc/NEWS.d/next/Library/2026-09-05-17-29-50.gh-issue-160837.W6kHva.rst b/Misc/NEWS.d/next/Library/2026-09-05-17-29-50.gh-issue-160837.W6kHva.rst deleted file mode 100644 index 701e42548d5d269..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-05-17-29-50.gh-issue-160837.W6kHva.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`asyncio.print_call_graph` no longer truncates the call stack of a -task suspended inside an asynchronous generator. diff --git a/Misc/NEWS.d/next/Library/2026-09-17-10-30-00.gh-issue-158252.strptime.rst b/Misc/NEWS.d/next/Library/2026-09-17-10-30-00.gh-issue-158252.strptime.rst deleted file mode 100644 index 9d3eeb245023a9d..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-17-10-30-00.gh-issue-158252.strptime.rst +++ /dev/null @@ -1,4 +0,0 @@ -Speed up :func:`time.strptime`, :meth:`datetime.date.strptime`, -:meth:`datetime.time.strptime`, and :meth:`datetime.datetime.strptime` for -common numeric formats by parsing fields and constructing the result in C. -Other inputs continue to use the Python parser. diff --git a/Misc/NEWS.d/next/Library/2026-09-17-20-36-01.gh-issue-157290.NfFmt.rst b/Misc/NEWS.d/next/Library/2026-09-17-20-36-01.gh-issue-157290.NfFmt.rst deleted file mode 100644 index 7a001e8784b2be2..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-17-20-36-01.gh-issue-157290.NfFmt.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add the ``%Nf`` format code (where *N* is from 1 to 6) to -:meth:`datetime.datetime.strftime`, :meth:`datetime.time.strftime`, -:meth:`datetime.date.strftime`, :meth:`datetime.datetime.strptime` and -:meth:`datetime.time.strptime`. diff --git a/Misc/NEWS.d/next/Library/2026-09-18-00-00-00.gh-issue-49960.MDT3jE.rst b/Misc/NEWS.d/next/Library/2026-09-18-00-00-00.gh-issue-49960.MDT3jE.rst deleted file mode 100644 index 9793c1143c7d0d8..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-18-00-00-00.gh-issue-49960.MDT3jE.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`ctypes` callback functions can now return structures and unions by -value. Previously only "simple" types could be returned, and any other result -type raised :exc:`TypeError`. diff --git a/Misc/NEWS.d/next/Library/2026-09-20-12-00-00.gh-issue-157856.wq4xuF.rst b/Misc/NEWS.d/next/Library/2026-09-20-12-00-00.gh-issue-157856.wq4xuF.rst deleted file mode 100644 index 9f221fa657d4298..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-20-12-00-00.gh-issue-157856.wq4xuF.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :func:`asyncio.as_completed` raising :exc:`RuntimeError` ("no running -event loop") when the iterator is created outside of a running event loop and -driven with :meth:`loop.run_until_complete() `. diff --git a/Misc/NEWS.d/next/Library/2026-09-20-21-24-05.gh-issue-157880.aE7wIu.rst b/Misc/NEWS.d/next/Library/2026-09-20-21-24-05.gh-issue-157880.aE7wIu.rst deleted file mode 100644 index 7fbb1080047989b..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-20-21-24-05.gh-issue-157880.aE7wIu.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :class:`mailbox.Babyl` to write the visible headers of a -:class:`mailbox.BabylMessage` to the mailbox file. diff --git a/Misc/NEWS.d/next/Library/2026-09-21-14-41-07.gh-issue-157899.s6vBQC.rst b/Misc/NEWS.d/next/Library/2026-09-21-14-41-07.gh-issue-157899.s6vBQC.rst deleted file mode 100644 index d70cef97fb20762..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-21-14-41-07.gh-issue-157899.s6vBQC.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`os.readlink` now accepts a file descriptor referring to a -symlink on Linux, Android, macOS and iOS. diff --git a/Misc/NEWS.d/next/Library/2026-09-21-21-07-12.gh-issue-157928.ZPlNYC.rst b/Misc/NEWS.d/next/Library/2026-09-21-21-07-12.gh-issue-157928.ZPlNYC.rst deleted file mode 100644 index dfa7a122a9e22e8..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-21-21-07-12.gh-issue-157928.ZPlNYC.rst +++ /dev/null @@ -1 +0,0 @@ -Correctly detect and handle OSC sequences in PyREPL. diff --git a/Misc/NEWS.d/next/Library/2026-09-23-09-21-25.gh-issue-89977.fmia18.rst b/Misc/NEWS.d/next/Library/2026-09-23-09-21-25.gh-issue-89977.fmia18.rst deleted file mode 100644 index fcce4b6923dd3e6..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-23-09-21-25.gh-issue-89977.fmia18.rst +++ /dev/null @@ -1,3 +0,0 @@ -Avoid releasing the GIL for :meth:`socket.socket.setblocking`, -:meth:`socket.socket.getsockname`, :meth:`socket.socket.getpeername`, and -:func:`socket.gethostname`. diff --git a/Misc/NEWS.d/next/Library/2026-09-23-19-59-03.gh-issue-158004.p68swW.rst b/Misc/NEWS.d/next/Library/2026-09-23-19-59-03.gh-issue-158004.p68swW.rst deleted file mode 100644 index 0db4a3d4ca4c4aa..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-23-19-59-03.gh-issue-158004.p68swW.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`asyncio.as_completed` to continue yielding every awaitable after -a pending iteration step is cancelled. diff --git a/Misc/NEWS.d/next/Library/2026-09-23-23-34-30.gh-issue-144812.ujptbVOI.rst b/Misc/NEWS.d/next/Library/2026-09-23-23-34-30.gh-issue-144812.ujptbVOI.rst deleted file mode 100644 index d24f028fdbeaae7..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-23-23-34-30.gh-issue-144812.ujptbVOI.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make :class:`argparse.ArgumentGroup` and -:class:`argparse.MutuallyExclusiveGroup` public. The private -``_ArgumentGroup`` and ``_MutuallyExclusiveGroup`` names remain as aliases. diff --git a/Misc/NEWS.d/next/Library/2026-09-24-14-59-08.gh-issue-158095.lcKOT4.rst b/Misc/NEWS.d/next/Library/2026-09-24-14-59-08.gh-issue-158095.lcKOT4.rst deleted file mode 100644 index 4fe92b8c8097d85..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-24-14-59-08.gh-issue-158095.lcKOT4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a memory leak in :func:`re.finditer` and regular expression scanners when -their input participates in a reference cycle. diff --git a/Misc/NEWS.d/next/Library/2026-09-25-01-29-55.gh-issue-158126.nWmFnq.rst b/Misc/NEWS.d/next/Library/2026-09-25-01-29-55.gh-issue-158126.nWmFnq.rst deleted file mode 100644 index 82bf49faef56d4b..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-25-01-29-55.gh-issue-158126.nWmFnq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Speed up :func:`pickle.dumps` and creating a :class:`pickle.Pickler` by -avoiding the creation of an :exc:`AttributeError` when looking up the -optional ``reducer_override`` and ``dispatch_table`` attributes. diff --git a/Misc/NEWS.d/next/Library/2026-09-25-17-04-45.gh-issue-158159.lbEalG.rst b/Misc/NEWS.d/next/Library/2026-09-25-17-04-45.gh-issue-158159.lbEalG.rst deleted file mode 100644 index a0d365111d92ff6..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-25-17-04-45.gh-issue-158159.lbEalG.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a use-after-free in :meth:`asyncio.Task.get_name` when formatting the -default task name triggers a garbage collection whose finalizers call -:meth:`~asyncio.Task.get_name` on the same task. diff --git a/Misc/NEWS.d/next/Library/2026-09-26-05-18-21.gh-issue-144650.zqHLOW.rst b/Misc/NEWS.d/next/Library/2026-09-26-05-18-21.gh-issue-144650.zqHLOW.rst deleted file mode 100644 index ef4323571e59984..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-26-05-18-21.gh-issue-144650.zqHLOW.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use Argument Clinic vectorcall support for the :class:`decimal.Decimal` type. This offers -1.10x speedup on creation of small instances. diff --git a/Misc/NEWS.d/next/Library/2026-09-27-14-00-00.gh-issue-158277.aBcDeF.rst b/Misc/NEWS.d/next/Library/2026-09-27-14-00-00.gh-issue-158277.aBcDeF.rst deleted file mode 100644 index 7f0434aeea67081..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-27-14-00-00.gh-issue-158277.aBcDeF.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :mod:`sqlite3` CLI completer to include view columns in -column-name completion. - diff --git a/Misc/NEWS.d/next/Library/2026-09-28-00-00-00.gh-issue-158370.ctxmgr.rst b/Misc/NEWS.d/next/Library/2026-09-28-00-00-00.gh-issue-158370.ctxmgr.rst deleted file mode 100644 index f4b9e2469daa3ed..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-28-00-00-00.gh-issue-158370.ctxmgr.rst +++ /dev/null @@ -1,2 +0,0 @@ -:class:`concurrent.interpreters.Interpreter` objects now support the -:term:`context manager` protocol. diff --git a/Misc/NEWS.d/next/Library/2026-09-29-19-34-30.gh-issue-158192.6wPjiU.rst b/Misc/NEWS.d/next/Library/2026-09-29-19-34-30.gh-issue-158192.6wPjiU.rst deleted file mode 100644 index b0634084c6af8ad..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-29-19-34-30.gh-issue-158192.6wPjiU.rst +++ /dev/null @@ -1 +0,0 @@ -Expose the future being waited on by a ``FutureIter`` as ``fi_future``. diff --git a/Misc/NEWS.d/next/Library/2026-10-05-08-35-10.gh-issue-130698.iibO9V.rst b/Misc/NEWS.d/next/Library/2026-10-05-08-35-10.gh-issue-130698.iibO9V.rst deleted file mode 100644 index 9aef7647e64e412..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-05-08-35-10.gh-issue-130698.iibO9V.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix PyREPL to convert :data:`sys.ps1` and :data:`sys.ps2` to strings before -use, and fall back to the default prompts if the conversion raises. diff --git a/Misc/NEWS.d/next/Library/2026-10-05-14-58-12.gh-issue-158839.wYd6fI.rst b/Misc/NEWS.d/next/Library/2026-10-05-14-58-12.gh-issue-158839.wYd6fI.rst deleted file mode 100644 index 9bdf648f42783f2..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-05-14-58-12.gh-issue-158839.wYd6fI.rst +++ /dev/null @@ -1,5 +0,0 @@ -The log functions in the :mod:`logging` module and the -:class:`logging.Logger` class now normalize falsy ``exc_info`` arguments to -``None`` before passing them to :meth:`logging.Logger.makeRecord` and -subsequently to :class:`logging.LogRecord`. This aligns -``logging.LogRecord.exc_info`` with the documentation. diff --git a/Misc/NEWS.d/next/Library/2026-10-07-10-59-27.gh-issue-155233.YGuQGK.rst b/Misc/NEWS.d/next/Library/2026-10-07-10-59-27.gh-issue-155233.YGuQGK.rst deleted file mode 100644 index ebcd4a4bc29a5c3..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-07-10-59-27.gh-issue-155233.YGuQGK.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :meth:`asyncio.Barrier.wait` returning duplicate indices when a waiting task is cancelled before the barrier is released.``` diff --git a/Misc/NEWS.d/next/Library/2026-10-07-17-30-00.gh-issue-158287.xY9zQp.rst b/Misc/NEWS.d/next/Library/2026-10-07-17-30-00.gh-issue-158287.xY9zQp.rst deleted file mode 100644 index d7432faa0b7f92b..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-07-17-30-00.gh-issue-158287.xY9zQp.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in :mod:`asyncio` where stopping a child process (such as with ``SIGSTOP``) on macOS caused the event loop to block indefinitely. diff --git a/Misc/NEWS.d/next/Library/2026-10-08-11-58-52.gh-issue-158628.NJxwuw.rst b/Misc/NEWS.d/next/Library/2026-10-08-11-58-52.gh-issue-158628.NJxwuw.rst deleted file mode 100644 index eb1e0f1d3271f8a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-08-11-58-52.gh-issue-158628.NJxwuw.rst +++ /dev/null @@ -1 +0,0 @@ -Implement :data:`sys._wasi_info` for exposing WASI-specific system details. diff --git a/Misc/NEWS.d/next/Library/2026-10-08-21-02-38.gh-issue-158998.DZYJI7.rst b/Misc/NEWS.d/next/Library/2026-10-08-21-02-38.gh-issue-158998.DZYJI7.rst deleted file mode 100644 index 65ce9201fb01352..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-08-21-02-38.gh-issue-158998.DZYJI7.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :func:`os.statx` ignoring the *follow_symlinks* argument. diff --git a/Misc/NEWS.d/next/Library/2026-10-10-22-14-22.gh-issue-150751.JInuGL.rst b/Misc/NEWS.d/next/Library/2026-10-10-22-14-22.gh-issue-150751.JInuGL.rst deleted file mode 100644 index 40c8d606aa67734..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-10-22-14-22.gh-issue-150751.JInuGL.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`http.client` now ignores optional whitespace around the -``Content-Length`` header value when validating it, as required by -:rfc:`9112`. Previously a value with trailing whitespace (for example -``Content-Length: 5`` followed by a space) was treated as missing and the -response body was read until the connection was closed. diff --git a/Misc/NEWS.d/next/Library/2026-10-10-22-29-13.gh-issue-159049.eJUiKL.rst b/Misc/NEWS.d/next/Library/2026-10-10-22-29-13.gh-issue-159049.eJUiKL.rst deleted file mode 100644 index 96f1cc3c0520d57..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-10-22-29-13.gh-issue-159049.eJUiKL.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix reference cycles in :meth:`asyncio.loop.create_connection` and -:meth:`asyncio.loop.start_tls` after a failed TLS handshake. -Patch by Timofei Ivankov. diff --git a/Misc/NEWS.d/next/Library/2026-10-11-08-42-00.gh-issue-159184.Tq2nSe.rst b/Misc/NEWS.d/next/Library/2026-10-11-08-42-00.gh-issue-159184.Tq2nSe.rst deleted file mode 100644 index 240738671ffaaf4..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-11-08-42-00.gh-issue-159184.Tq2nSe.rst +++ /dev/null @@ -1,3 +0,0 @@ -The output of :func:`marshal.dumps` no longer depends on the order of the -elements of a set which have the same marshalled form, such as NaNs which -are distinct objects. diff --git a/Misc/NEWS.d/next/Library/2026-10-11-13-08-07.gh-issue-88037.CzKcxh.rst b/Misc/NEWS.d/next/Library/2026-10-11-13-08-07.gh-issue-88037.CzKcxh.rst deleted file mode 100644 index 697b44ba2985a0d..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-11-13-08-07.gh-issue-88037.CzKcxh.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`urllib.parse.urlsplit` and :func:`urllib.parse.urlparse` now raise -:exc:`ValueError` for an invalid port when the URL is parsed, instead of -when the :attr:`!port` attribute is read. Patch by Miguel Brito and Senthil Kumaran. diff --git a/Misc/NEWS.d/next/Library/2026-10-11-13-40-44.gh-issue-159168.z3KjB8.rst b/Misc/NEWS.d/next/Library/2026-10-11-13-40-44.gh-issue-159168.z3KjB8.rst deleted file mode 100644 index 54b8738e06c270a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-11-13-40-44.gh-issue-159168.z3KjB8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :exc:`RecursionError` in ``python -m asyncio pstree`` on a deep chain of -awaiting tasks. Patch by Timofei Ivankov. diff --git a/Misc/NEWS.d/next/Library/2026-10-12-00-00-00.gh-issue-159044.Tr8kXv.rst b/Misc/NEWS.d/next/Library/2026-10-12-00-00-00.gh-issue-159044.Tr8kXv.rst deleted file mode 100644 index ef51dc8eca1066c..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-12-00-00-00.gh-issue-159044.Tr8kXv.rst +++ /dev/null @@ -1,4 +0,0 @@ -Restore the previous behaviour of :meth:`http.client.HTTPConnection.set_tunnel` -where header values that are not strings (such as :const:`None`, :class:`int` -or other objects) are converted with :class:`str` instead of raising -:exc:`AttributeError` when the request is sent. diff --git a/Misc/NEWS.d/next/Library/2026-10-12-11-36-48.gh-issue-158856.jLR-Lv.rst b/Misc/NEWS.d/next/Library/2026-10-12-11-36-48.gh-issue-158856.jLR-Lv.rst deleted file mode 100644 index a95c7b0cb44af30..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-12-11-36-48.gh-issue-158856.jLR-Lv.rst +++ /dev/null @@ -1 +0,0 @@ -Improve large Enum creation performance. diff --git a/Misc/NEWS.d/next/Library/2026-10-12-12-00-00.gh-issue-72113.njkm8v.rst b/Misc/NEWS.d/next/Library/2026-10-12-12-00-00.gh-issue-72113.njkm8v.rst deleted file mode 100644 index 9057f3246b3e2b7..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-12-12-00-00.gh-issue-72113.njkm8v.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up creating a :mod:`ctypes` array from initializers, such as -``(c_int * 3)(1, 2, 3)``, by roughly 3x. diff --git a/Misc/NEWS.d/next/Library/2026-10-12-13-05-00.gh-issue-158501.n7Qp3k.rst b/Misc/NEWS.d/next/Library/2026-10-12-13-05-00.gh-issue-158501.n7Qp3k.rst deleted file mode 100644 index e47c5da38c8e860..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-12-13-05-00.gh-issue-158501.n7Qp3k.rst +++ /dev/null @@ -1,4 +0,0 @@ -:meth:`datetime.time.fromisoformat` and -:meth:`datetime.datetime.fromisoformat` now reject stray text between the -time and a UTC offset. The C implementation previously accepted strings -that the pure-Python implementation rejected. diff --git a/Misc/NEWS.d/next/Library/2026-10-12-18-21-06.gh-issue-159240.a1b2c3.rst b/Misc/NEWS.d/next/Library/2026-10-12-18-21-06.gh-issue-159240.a1b2c3.rst deleted file mode 100644 index 85439b77aaa8da4..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-12-18-21-06.gh-issue-159240.a1b2c3.rst +++ /dev/null @@ -1,6 +0,0 @@ -Deprecate, and schedule for removal in Python 3.20, the ``__author__``, -``__credits__``, ``__copyright__`` and ``__date__`` attributes of :mod:`bz2`, -:mod:`concurrent.futures`, -:mod:`inspect`, :mod:`io`, :mod:`json`, :mod:`logging`, :mod:`optparse`, -:mod:`platform`, :mod:`pydoc`, :mod:`tarfile`, :mod:`tkinter.ttk`, -:mod:`tokenize` and :mod:`uuid`. diff --git a/Misc/NEWS.d/next/Library/2026-10-12-21-20-00.gh-issue-159225.T8jlor.rst b/Misc/NEWS.d/next/Library/2026-10-12-21-20-00.gh-issue-159225.T8jlor.rst deleted file mode 100644 index a9ee0e6a7eb97d2..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-12-21-20-00.gh-issue-159225.T8jlor.rst +++ /dev/null @@ -1 +0,0 @@ -Prevent ``input`` calls in PyREPL from modifying readline history. diff --git a/Misc/NEWS.d/next/Library/2026-10-13-12-00-00.gh-issue-159304.mFbYq2.rst b/Misc/NEWS.d/next/Library/2026-10-13-12-00-00.gh-issue-159304.mFbYq2.rst deleted file mode 100644 index 8f5b61efe039593..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-13-12-00-00.gh-issue-159304.mFbYq2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :exc:`AttributeError` in :meth:`email.message.EmailMessage.get_body` when a -nested ``multipart`` part is malformed and its payload is a string. diff --git a/Misc/NEWS.d/next/Library/2026-10-13-19-10-00.gh-issue-159322.hP6k3A.rst b/Misc/NEWS.d/next/Library/2026-10-13-19-10-00.gh-issue-159322.hP6k3A.rst deleted file mode 100644 index f9c3d1a98acc357..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-13-19-10-00.gh-issue-159322.hP6k3A.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :mod:`tarfile` to NUL-terminate user and group names in PAX archive -headers and use extended headers for names longer than 31 characters. diff --git a/Misc/NEWS.d/next/Library/2026-10-14-08-50-00.gh-issue-159398.k2Xd7q.rst b/Misc/NEWS.d/next/Library/2026-10-14-08-50-00.gh-issue-159398.k2Xd7q.rst deleted file mode 100644 index 65e10f4ba792840..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-14-08-50-00.gh-issue-159398.k2Xd7q.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the sampling profiler crashing when a string read from the target process -contains code points above ``U+10FFFF``. diff --git a/Misc/NEWS.d/next/Library/2026-10-14-10-26-10.gh-issue-159387.xMbQbB.rst b/Misc/NEWS.d/next/Library/2026-10-14-10-26-10.gh-issue-159387.xMbQbB.rst deleted file mode 100644 index fddef6e097ba5cf..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-14-10-26-10.gh-issue-159387.xMbQbB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :mod:`ssl` reporting TLS 1.0, 1.1, and 1.2 as unavailable when built -with OpenSSL 4.0, despite support for these protocol versions. diff --git a/Misc/NEWS.d/next/Library/2026-10-14-11-41-00.gh-issue-159430.ZEW-mx.rst b/Misc/NEWS.d/next/Library/2026-10-14-11-41-00.gh-issue-159430.ZEW-mx.rst deleted file mode 100644 index 5b9041891431c0f..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-14-11-41-00.gh-issue-159430.ZEW-mx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Report the correct exception if looking up ``__name__`` raises an exception -when calling :func:`repr` on a :class:`weakref.ref`. diff --git a/Misc/NEWS.d/next/Library/2026-10-14-14-47-05.gh-issue-81954.KvM0gY.rst b/Misc/NEWS.d/next/Library/2026-10-14-14-47-05.gh-issue-81954.KvM0gY.rst deleted file mode 100644 index 5ced3b94d223b25..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-14-14-47-05.gh-issue-81954.KvM0gY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :class:`zipfile.ZipFile` not being closed at interpreter shutdown when -emitting its :exc:`ResourceWarning` failed with an :exc:`ImportError`. diff --git a/Misc/NEWS.d/next/Library/2026-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst b/Misc/NEWS.d/next/Library/2026-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst deleted file mode 100644 index 21199b550f9a54f..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-15-13-44-03.gh-issue-159465.Ph-m8w.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve incomplete input detection in :mod:`codeop` without an extra parser -pass, and preserve syntax and compiler errors. diff --git a/Misc/NEWS.d/next/Library/2026-10-15-18-38-22.gh-issue-159500.OjAQ8W.rst b/Misc/NEWS.d/next/Library/2026-10-15-18-38-22.gh-issue-159500.OjAQ8W.rst deleted file mode 100644 index 2cfaec1e5999a6f..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-15-18-38-22.gh-issue-159500.OjAQ8W.rst +++ /dev/null @@ -1,4 +0,0 @@ -Speed up :meth:`asyncio.StreamReader.readline` and -:meth:`asyncio.StreamReader.readuntil` when the data is already buffered, -and stop :meth:`~asyncio.StreamReader.readuntil` with a tuple of separators from -scanning the whole buffer on every call. diff --git a/Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst b/Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst deleted file mode 100644 index 158968d0c2a1cea..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-16-01-56-15.gh-issue-159503.WgYiow.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix quadratic copying in :mod:`asyncio` selector socket transports on partial -sends. A :class:`bytearray` passed to :meth:`~asyncio.WriteTransport.write` -now stays locked against resizing until it has been sent. diff --git a/Misc/NEWS.d/next/Library/2026-10-16-10-25-22.gh-issue-159458.q02EnY.rst b/Misc/NEWS.d/next/Library/2026-10-16-10-25-22.gh-issue-159458.q02EnY.rst deleted file mode 100644 index 6565f4265b5877c..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-16-10-25-22.gh-issue-159458.q02EnY.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a ThreadSanitizer data race on the static cache in ``normalize_century()`` of the datetime module under the free-threaded build. diff --git a/Misc/NEWS.d/next/Library/2026-10-16-13-00-00.gh-issue-159435.OMNI.rst b/Misc/NEWS.d/next/Library/2026-10-16-13-00-00.gh-issue-159435.OMNI.rst deleted file mode 100644 index 3f4c71c2c2e0a6a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-16-13-00-00.gh-issue-159435.OMNI.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an offset calculation in :mod:`tarfile` when a PAX header with a ``size`` -record is followed by a GNU long-name header. Such archives could duplicate or -skip members while being read. diff --git a/Misc/NEWS.d/next/Library/2026-10-16-18-23-03.gh-issue-159558.ctRelk.rst b/Misc/NEWS.d/next/Library/2026-10-16-18-23-03.gh-issue-159558.ctRelk.rst deleted file mode 100644 index a8b8570365f3f48..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-16-18-23-03.gh-issue-159558.ctRelk.rst +++ /dev/null @@ -1,4 +0,0 @@ -Speed up assignment to the ``value`` attribute of simple :mod:`ctypes` types, to -the ``contents`` attribute of :mod:`ctypes` pointers and to :mod:`ctypes` -structure fields, array items and pointer items on the free-threaded build by -not re-entering the critical section already held on the object. diff --git a/Misc/NEWS.d/next/Library/2026-10-17-15-22-16.gh-issue-159620.Ynsu5L.rst b/Misc/NEWS.d/next/Library/2026-10-17-15-22-16.gh-issue-159620.Ynsu5L.rst deleted file mode 100644 index 2cbc3461cfaa7d8..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-17-15-22-16.gh-issue-159620.Ynsu5L.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix ``python -m asyncio ps``, ``pstree`` and async-aware -:mod:`profiling.sampling` stacks stopping at asynchronous generators. -Patch by Timofei Ivankov. diff --git a/Misc/NEWS.d/next/Library/2026-10-17-19-33-13.gh-issue-159643.EJfe0IXetV.rst b/Misc/NEWS.d/next/Library/2026-10-17-19-33-13.gh-issue-159643.EJfe0IXetV.rst deleted file mode 100644 index a76e54de38cc4e0..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-17-19-33-13.gh-issue-159643.EJfe0IXetV.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :exc:`EOFError` raised by :mod:`tarfile` in stream mode with bz2, xz or -zstd compression when reading past the end of the compressed data, for -example with *ignore_zeros* or an archive without end-of-archive blocks. diff --git a/Misc/NEWS.d/next/Library/2026-10-17-19-41-12.gh-issue-157210.o9hycSeJ.rst b/Misc/NEWS.d/next/Library/2026-10-17-19-41-12.gh-issue-157210.o9hycSeJ.rst deleted file mode 100644 index 0c399257b497fff..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-17-19-41-12.gh-issue-157210.o9hycSeJ.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix entering a combining character in the lower-right cell of -:class:`curses.textpad.Textbox`: -it no longer takes a cell of its own, -pushing its base character off the window. -In insert mode, a combining character no longer shifts the following text. diff --git a/Misc/NEWS.d/next/Library/2026-10-18-08-30-10.gh-issue-155976.feSum4_j.rst b/Misc/NEWS.d/next/Library/2026-10-18-08-30-10.gh-issue-155976.feSum4_j.rst deleted file mode 100644 index 64aa76725de723e..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-18-08-30-10.gh-issue-155976.feSum4_j.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`curses.set_term` and :meth:`curses.screen.use` now update -:data:`curses.LINES`, :data:`curses.COLS`, :data:`curses.COLORS` -and :data:`curses.COLOR_PAIRS` for the current screen. diff --git a/Misc/NEWS.d/next/Library/2026-10-18-12-20-00.gh-issue-159465.partial-escapes.rst b/Misc/NEWS.d/next/Library/2026-10-18-12-20-00.gh-issue-159465.partial-escapes.rst deleted file mode 100644 index ecddc0f540761d1..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-18-12-20-00.gh-issue-159465.partial-escapes.rst +++ /dev/null @@ -1,2 +0,0 @@ -Preserve :exc:`SyntaxError` for malformed escapes in unfinished f-strings and -t-strings passed to :func:`codeop.compile_command`. diff --git a/Misc/NEWS.d/next/Library/2026-10-18-16-09-24.gh-issue-159684.VHVYjVuE.rst b/Misc/NEWS.d/next/Library/2026-10-18-16-09-24.gh-issue-159684.VHVYjVuE.rst deleted file mode 100644 index 43f648af4784d9c..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-18-16-09-24.gh-issue-159684.VHVYjVuE.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update :mod:`configparser` and its tests to modern linter standards: -printf-style and :meth:`str.format` call sites are now f-strings, and a -handful of constructs have simpler spellings. No behavior change. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-00-00-00.gh-issue-149015.bdb.rst b/Misc/NEWS.d/next/Library/2026-10-19-00-00-00.gh-issue-149015.bdb.rst deleted file mode 100644 index a7bcbc4b1aa3224..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-00-00-00.gh-issue-149015.bdb.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`bdb.Bdb.clear_all_file_breaks` leaving breakpoints behind when -multiple breakpoints are set on the same line. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-12-00-00.gh-issue-159714.j59fix.rst b/Misc/NEWS.d/next/Library/2026-10-19-12-00-00.gh-issue-159714.j59fix.rst deleted file mode 100644 index 7b61ca79cbd2b57..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-12-00-00.gh-issue-159714.j59fix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :class:`zoneinfo.ZoneInfo` interpreting the Julian day rule ``J59`` as -February 29 in leap years instead of February 28. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-12-00-00.gh-issue-78327.Qk3hzA.rst b/Misc/NEWS.d/next/Library/2026-10-19-12-00-00.gh-issue-78327.Qk3hzA.rst deleted file mode 100644 index b0d4107bcadf931..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-12-00-00.gh-issue-78327.Qk3hzA.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`ctypes` data types now set :meth:`~object.__hash__` to ``None``, so -:class:`collections.abc.Hashable` checks report them correctly. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-16-00-00.gh-issue-139122.bYtEs1.rst b/Misc/NEWS.d/next/Library/2026-10-19-16-00-00.gh-issue-139122.bYtEs1.rst deleted file mode 100644 index 256b36941aa910d..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-16-00-00.gh-issue-139122.bYtEs1.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`uuid.UUID` now consistently raises :exc:`TypeError` when the *bytes* -or *bytes_le* argument is not a :class:`bytes` object. Previously, some invalid -inputs raised :exc:`AssertionError` or :exc:`ValueError`. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-16-26-57.gh-issue-71990.K3FlJ9.rst b/Misc/NEWS.d/next/Library/2026-10-19-16-26-57.gh-issue-71990.K3FlJ9.rst deleted file mode 100644 index 87276db65054a03..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-16-26-57.gh-issue-71990.K3FlJ9.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`ctypes`: objects with an :attr:`!_as_parameter_` attribute are now -implicitly converted when passed for a :func:`~ctypes.POINTER` argument. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-16-49-09.gh-issue-159706.ushGrx.rst b/Misc/NEWS.d/next/Library/2026-10-19-16-49-09.gh-issue-159706.ushGrx.rst deleted file mode 100644 index ee90605ed0af2b3..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-16-49-09.gh-issue-159706.ushGrx.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix the pure-Python :class:`zoneinfo.ZoneInfo` setting ``fold=0`` in -:meth:`~datetime.tzinfo.fromutc` for times in the fold created by the only -transition of a zone with a single transition. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-19-33-35.gh-issue-159739.a1c3c7.rst b/Misc/NEWS.d/next/Library/2026-10-19-19-33-35.gh-issue-159739.a1c3c7.rst deleted file mode 100644 index 2d723061b9369d8..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-19-33-35.gh-issue-159739.a1c3c7.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix resource leaks in :mod:`ssl` when certificate conversion or key log setup -fails. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-19-33-35.gh-issue-159739.b5fbca.rst b/Misc/NEWS.d/next/Library/2026-10-19-19-33-35.gh-issue-159739.b5fbca.rst deleted file mode 100644 index d4a662b10d9f4f4..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-19-33-35.gh-issue-159739.b5fbca.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a memory leak when Unicode normalization cannot grow its temporary buffer. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.08caa7.rst b/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.08caa7.rst deleted file mode 100644 index 3574193b51f343e..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.08caa7.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix incorrect :mod:`ctypes` scalar values after :func:`ctypes.resize` enlarges -the object. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.243aa8.rst b/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.243aa8.rst deleted file mode 100644 index a9452184e7838f4..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.243aa8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid reading past the end of an encoding name when the ``surrogatepass`` -error handler receives a truncated UTF-16 or UTF-32 name. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.5a6e94.rst b/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.5a6e94.rst deleted file mode 100644 index e7eac98810cf48c..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.5a6e94.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid a crash when memory allocation fails while creating a -:class:`collections.OrderedDict` items iterator. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.a81270.rst b/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.a81270.rst deleted file mode 100644 index 413fc0d7f18f680..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.a81270.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :class:`io.BytesIO` reads into overlapping views of the same buffer. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.bc65da.rst b/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.bc65da.rst deleted file mode 100644 index fd831b6907e7778..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.bc65da.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow cyclic garbage collection to reclaim group records returned by -:mod:`grp`. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.ff088c.rst b/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.ff088c.rst deleted file mode 100644 index af8b06fecbfc3fd..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-20-38-17.gh-issue-159739.ff088c.rst +++ /dev/null @@ -1 +0,0 @@ -Fix copying :class:`collections.deque` subclasses with a large maximum length. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-21-20-28.gh-issue-159786.fThehQ.rst b/Misc/NEWS.d/next/Library/2026-10-19-21-20-28.gh-issue-159786.fThehQ.rst deleted file mode 100644 index 3f901326c87f68e..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-21-20-28.gh-issue-159786.fThehQ.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix the error handling of the HACL* routine used by :func:`hmac.new` and -:meth:`hmac.HMAC.update`. If the maximum input length of the underlying hash -function is exceeded, an :exc:`OverflowError` is now raised on all builds, -instead of crashing on DEBUG builds and being silently ignored otherwise. -Code correctness. Realistically this can only happen at implausible sizes. -Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-23-25-29.gh-issue-159811.P6yy0z.rst b/Misc/NEWS.d/next/Library/2026-10-19-23-25-29.gh-issue-159811.P6yy0z.rst deleted file mode 100644 index bcd76f209e52647..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-23-25-29.gh-issue-159811.P6yy0z.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in :mod:`hmac` when the OpenSSL routine used by -:meth:`hmac.HMAC.update` or by :func:`hmac.new` fails on data larger -than 2048 bytes. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-10-19-23-25-29.gh-issue-159811.cMB4DO.rst b/Misc/NEWS.d/next/Library/2026-10-19-23-25-29.gh-issue-159811.cMB4DO.rst deleted file mode 100644 index f63cd928a6a33cf..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-19-23-25-29.gh-issue-159811.cMB4DO.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in :mod:`hashlib` when the OpenSSL routine used by -:meth:`hash.update() ` or by hash object constructors -fails on data larger than 2048 bytes. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.0c4ae9.rst b/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.0c4ae9.rst deleted file mode 100644 index 3392832f714adce..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.0c4ae9.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix reference cycles involving the types of grapheme iterators and segments -created by :func:`unicodedata.iter_graphemes`. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.1e4599.rst b/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.1e4599.rst deleted file mode 100644 index 3122ca29c7c49b3..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.1e4599.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix off-by-one output buffer checks in the fallback implementation of -``getnameinfo()``. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.35942a.rst b/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.35942a.rst deleted file mode 100644 index 548ac837589f078..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.35942a.rst +++ /dev/null @@ -1 +0,0 @@ -Improve performance when parsing email address groups with many members. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.3660a6.rst b/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.3660a6.rst deleted file mode 100644 index 56d11c767319cc3..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.3660a6.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix multibyte encoders silently consuming a non-BMP character when looking up -a two-character mapping. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.4c9783.rst b/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.4c9783.rst deleted file mode 100644 index b65ac4cede106bf..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.4c9783.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`io.IOBase.readline` returning more bytes than requested when a -stream supports :meth:`!peek` and returns short reads. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.5d4269.rst b/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.5d4269.rst deleted file mode 100644 index 94ee3cf4790112d..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.5d4269.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a double free if remote unwinder initialization runs out of memory. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.9ef89c.rst b/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.9ef89c.rst deleted file mode 100644 index 1afbadc1cf4e918..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.9ef89c.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix allocation error handling in the fallback implementation of -``getaddrinfo()``. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.eb8987.rst b/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.eb8987.rst deleted file mode 100644 index 11c88411bb9125a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.eb8987.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid undefined behavior when :func:`struct.pack_into` copies a bytearray into -overlapping output. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-00-12-05.gh-issue-159690.vZgLP6.rst b/Misc/NEWS.d/next/Library/2026-10-20-00-12-05.gh-issue-159690.vZgLP6.rst deleted file mode 100644 index b94d79501b9abed..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-00-12-05.gh-issue-159690.vZgLP6.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :class:`logging.handlers.TimedRotatingFileHandler` permanently stopping -rollover after a rollover failed or its backup file already existed, for -example at a DST change. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-12-30-00.gh-issue-159864.4FZ6MR.rst b/Misc/NEWS.d/next/Library/2026-10-20-12-30-00.gh-issue-159864.4FZ6MR.rst deleted file mode 100644 index e5b9b02a27cb978..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-12-30-00.gh-issue-159864.4FZ6MR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash in :func:`syslog.openlog` under the free-threaded build when -:data:`sys.argv` is modified concurrently. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-16-00-00.gh-issue-159868.qLhYkP.rst b/Misc/NEWS.d/next/Library/2026-10-20-16-00-00.gh-issue-159868.qLhYkP.rst deleted file mode 100644 index fa664aa31d42dad..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-16-00-00.gh-issue-159868.qLhYkP.rst +++ /dev/null @@ -1,3 +0,0 @@ -Allow :meth:`!collections.UserString.startswith` and -:meth:`!collections.UserString.endswith` to accept a -:class:`~collections.UserString` instance as the prefix or suffix argument. diff --git a/Misc/NEWS.d/next/Library/2026-10-20-17-29-14.gh-issue-138641.FN6qM0.rst b/Misc/NEWS.d/next/Library/2026-10-20-17-29-14.gh-issue-138641.FN6qM0.rst deleted file mode 100644 index 6d9ac1742828394..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-20-17-29-14.gh-issue-138641.FN6qM0.rst +++ /dev/null @@ -1 +0,0 @@ -Ignore :func:`pdb.set_trace` when called directly at the :mod:`pdb` prompt. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-15-10-00.gh-issue-160125.pdbpkg.rst b/Misc/NEWS.d/next/Library/2026-10-21-15-10-00.gh-issue-160125.pdbpkg.rst deleted file mode 100644 index 822d0dd3d79e913..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-15-10-00.gh-issue-160125.pdbpkg.rst +++ /dev/null @@ -1,3 +0,0 @@ -Support setting breakpoints with the :mod:`pdb` ``break`` and ``tbreak`` -commands using ``package.module.function`` names when the module has not been -imported in the current namespace. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-15-42-29.gh-issue-159875.p6j3yW.rst b/Misc/NEWS.d/next/Library/2026-10-21-15-42-29.gh-issue-159875.p6j3yW.rst deleted file mode 100644 index 824fa9567d79111..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-15-42-29.gh-issue-159875.p6j3yW.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fix :exc:`AttributeError` from the methods of an :mod:`asyncio` SSL transport, -such as :meth:`~asyncio.WriteTransport.write` and -:meth:`~asyncio.BaseTransport.set_protocol`, when the transport is closed after -its connection was lost. This also affected the fallback of -:meth:`loop.sendfile() ` over SSL when the peer went -away mid-transfer. The SSL context is still released once the connection is -lost. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-19-38-33.gh-issue-89721.uMpxRw.rst b/Misc/NEWS.d/next/Library/2026-10-21-19-38-33.gh-issue-89721.uMpxRw.rst deleted file mode 100644 index 519cf44beb54c91..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-19-38-33.gh-issue-89721.uMpxRw.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add ``copy_dir_stat`` parameter to :func:`shutil.copytree`: set it to -false to skip copying permissions and times of directories with -:func:`shutil.copystat`, e.g. when copying a read-only tree. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-19-40-00.gh-issue-100609.THXgRC.rst b/Misc/NEWS.d/next/Library/2026-10-21-19-40-00.gh-issue-100609.THXgRC.rst deleted file mode 100644 index 4a793a0fdcd91e8..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-19-40-00.gh-issue-100609.THXgRC.rst +++ /dev/null @@ -1,5 +0,0 @@ -``str()`` of a :class:`zipfile.Path` whose underlying zipfile has no -filename, such as one constructed from an :class:`io.BytesIO`, now renders -that filename as ``:zipfile:`` instead of raising :exc:`TypeError`. The root -of a zipfile is also no longer rendered with a trailing slash. Synchronized -with :pypi:`zipp` 3.23.1. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-19-55-00.gh-issue-159739.sque01.rst b/Misc/NEWS.d/next/Library/2026-10-21-19-55-00.gh-issue-159739.sque01.rst deleted file mode 100644 index 120caf1e27c6cba..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-19-55-00.gh-issue-159739.sque01.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid a crash when a :class:`queue.SimpleQueue` subclass finalizer adds an -item after construction fails. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-19-55-00.gh-issue-159739.tloc01.rst b/Misc/NEWS.d/next/Library/2026-10-21-19-55-00.gh-issue-159739.tloc01.rst deleted file mode 100644 index 9641d11deb84d65..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-19-55-00.gh-issue-159739.tloc01.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid a crash when a :class:`threading.local` subclass finalizer accesses -attributes after construction fails. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-20-05-00.gh-issue-160104.BUAL5C.rst b/Misc/NEWS.d/next/Library/2026-10-21-20-05-00.gh-issue-160104.BUAL5C.rst deleted file mode 100644 index bab414b247ff259..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-20-05-00.gh-issue-160104.BUAL5C.rst +++ /dev/null @@ -1,6 +0,0 @@ -Synchronized :class:`zipfile.Path` with :pypi:`zipp` 4.1.2. -:meth:`zipfile.Path.iterdir` now raises :exc:`NotADirectoryError` instead of -:exc:`ValueError` when the path does not reference a directory. Character -classes in :meth:`!zipfile.Path.glob` and :meth:`!zipfile.Path.rglob` patterns -now follow shell-style semantics, treating a leading ``!`` as negation and -``^`` as a literal, and no longer match path separators. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.JIqZRY.rst b/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.JIqZRY.rst deleted file mode 100644 index fa000802ef713bf..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.JIqZRY.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a possible crash when a list of :mod:`sqlite3` parameters is modified concurrently on a free-threaded build. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.b4ueJ7.rst b/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.b4ueJ7.rst deleted file mode 100644 index 6230801bbc09278..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.b4ueJ7.rst +++ /dev/null @@ -1 +0,0 @@ -Detect recursive ``_as_parameter_`` chains for :mod:`ctypes` calls without argument types. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.ghGctt.rst b/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.ghGctt.rst deleted file mode 100644 index dd9dd5dce88f430..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.ghGctt.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :class:`contextvars.ContextVar` returning a stale value after a reentrant finalizer changes it during :meth:`~contextvars.ContextVar.reset`. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.k8uuGe.rst b/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.k8uuGe.rst deleted file mode 100644 index 24dd0b7c7776e34..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.k8uuGe.rst +++ /dev/null @@ -1,3 +0,0 @@ -Raise :exc:`TypeError` if a string subclass returns a non-string during newline -translation in :class:`io.TextIOWrapper`. Recheck the stream state if the -replacement callback closes, detaches or reinitializes the stream. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.rVWF2a.rst b/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.rVWF2a.rst deleted file mode 100644 index f2e13ee926eaa11..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.rVWF2a.rst +++ /dev/null @@ -1 +0,0 @@ -Fix the ``truncate`` method on :class:`io.BytesIO` when converting its size argument closes the stream or exports its buffer. diff --git a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.yBc6Ei.rst b/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.yBc6Ei.rst deleted file mode 100644 index e299599e9763f0e..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-21-21-26-10.gh-issue-159739.yBc6Ei.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a possible crash in :class:`abc.ABCMeta` instance checks when a subclass-checking callback changes the instance’s class. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-00-29-50.gh-issue-160176.tkNst1.rst b/Misc/NEWS.d/next/Library/2026-10-22-00-29-50.gh-issue-160176.tkNst1.rst deleted file mode 100644 index 5f11a8237ba81a0..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-00-29-50.gh-issue-160176.tkNst1.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a crash in :mod:`tkinter` when a deeply nested or recursive list or -tuple is passed to Tcl, or a deeply nested Tcl list is converted to Python. -These now raise :exc:`ValueError` (beyond 1000 levels) and -:exc:`RecursionError` respectively. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst b/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst deleted file mode 100644 index a9d7aef03214a34..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-00-29-55.gh-issue-160185.BPz40E.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix :meth:`codecs.StreamWriter.reset` of the stateful CJK codecs (``hz`` and -the ``iso2022_*`` family) not writing the sequence that returns the output to -ASCII. ``seek(0)`` of :class:`codecs.StreamWriter`, -:class:`codecs.StreamReaderWriter` and :class:`codecs.StreamRecoder` now -resets the writer before moving the stream position. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-00-30-04.gh-issue-160187.9dcj2B.rst b/Misc/NEWS.d/next/Library/2026-10-22-00-30-04.gh-issue-160187.9dcj2B.rst deleted file mode 100644 index 74ee6c1fa7bbb92..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-00-30-04.gh-issue-160187.9dcj2B.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix the ISO-2022 decoders (``iso2022_jp``, ``iso2022_kr`` and related) -accepting escape sequences that designate a character set to the wrong code -element (G0, G1 or G2). Such sequences now raise :exc:`UnicodeDecodeError` -instead of producing wrong text or :exc:`RuntimeError`. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst b/Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst deleted file mode 100644 index 5644b99cce17209..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :meth:`io.BytesIO.getvalue` sometimes returning a :class:`bytes` object -with the hash of earlier contents after the buffer was written to, which made -:class:`dict` and :class:`set` lookups with it fail. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-00-33-42.gh-issue-160171.11l4om.rst b/Misc/NEWS.d/next/Library/2026-10-22-00-33-42.gh-issue-160171.11l4om.rst deleted file mode 100644 index 73c51e66ea26cf6..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-00-33-42.gh-issue-160171.11l4om.rst +++ /dev/null @@ -1,4 +0,0 @@ -On the server side, :meth:`ssl.SSLSocket.get_unverified_chain` now returns -the client's certificate when the session was resumed from a session ticket, -instead of an empty list, and returns an empty list instead of raising -:exc:`AttributeError` when the client sent no certificate. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst b/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst deleted file mode 100644 index 7052cdb6bd9682b..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.a8bf30.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix clearing a :class:`pickle.Unpickler` memo so that the unpickler can be -reused for independent records. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.b4c371.rst b/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.b4c371.rst deleted file mode 100644 index a3682d0cea74dc9..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.b4c371.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash when a :mod:`ctypes` structure subclass cannot be created due -to an allocation failure. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.ce26e1.rst b/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.ce26e1.rst deleted file mode 100644 index 94f97951af2712d..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-03-55-33.gh-issue-159739.ce26e1.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :mod:`time` functions returning a negative year or failing to accept -their own output for large timestamps on platforms that support them. Avoid -reporting an incorrect year in errors raised by :mod:`datetime` for those -timestamps. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-19-56.gh-issue-161090.pxZ1IA.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-19-56.gh-issue-161090.pxZ1IA.rst deleted file mode 100644 index 40b506b39cc4332..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-19-56.gh-issue-161090.pxZ1IA.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix the ``punycode`` codec accepting a delimiter (``-``) that is not preceded -by any basic code point. Such input, for example ``b'-80a1acny'``, now raises -:exc:`UnicodeDecodeError`, as required by :rfc:`3492`. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-20-02.gh-issue-161092.OtC5l6.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-20-02.gh-issue-161092.OtC5l6.rst deleted file mode 100644 index abdc68c7f7a5dbd..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-20-02.gh-issue-161092.OtC5l6.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix the ``uu`` codec decoder taking any line that starts with ``begin``, such -as ``beginning of the report``, as the header of the encoded data. Like the -:mod:`email` package, it now requires ``begin`` followed by an octal mode and -skips other lines as leading text. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-20-02.gh-issue-161102.gqDOIM7X.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-20-02.gh-issue-161102.gqDOIM7X.rst deleted file mode 100644 index e3089a0560bcbf7..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-20-02.gh-issue-161102.gqDOIM7X.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix the incremental decoder of the :mod:`idna ` codec -decoding some dots twice when its input starts with empty labels, for example -``b".example.com"``. Reading a run of dots through :class:`io.TextIOWrapper` -no longer produces quadratically more text than the input holds. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-20-06.gh-issue-161100.pLgnlA.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-20-06.gh-issue-161100.pLgnlA.rst deleted file mode 100644 index b23de8a22ca9683..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-20-06.gh-issue-161100.pLgnlA.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :attr:`!ipaddress.IPv6Interface.is_unspecified` returning ``False`` for -the IPv4-mapped address ``::ffff:0.0.0.0``. It now gives the same result as -:attr:`ipaddress.IPv6Address.is_unspecified`. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-20-11.gh-issue-161110.miKrSQ.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-20-11.gh-issue-161110.miKrSQ.rst deleted file mode 100644 index e71214d60f1cd1e..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-20-11.gh-issue-161110.miKrSQ.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :meth:`~xml.dom.Document.getElementById` in :mod:`xml.dom.minidom` so -that it returns the first element in document order when several elements -have the same ID. Previously, the result could differ between calls on an -unmodified document. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-20-18.gh-issue-161108.xOVsRA.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-20-18.gh-issue-161108.xOVsRA.rst deleted file mode 100644 index d58214c823a5525..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-20-18.gh-issue-161108.xOVsRA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix quadratic time complexity of :meth:`!clear` and :meth:`!popitem` on -:class:`configparser.ConfigParser` objects and their section proxies. -Clearing a section no longer fails part way with -:exc:`~configparser.InterpolationError` on a value it removes. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-20-27.gh-issue-161098.voMVnd.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-20-27.gh-issue-161098.voMVnd.rst deleted file mode 100644 index 73849a0136aafcc..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-20-27.gh-issue-161098.voMVnd.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :class:`multiprocessing.shared_memory.ShareableList` leaving its newly -created shared memory block behind when a value cannot be stored, such as an -integer outside the signed 64-bit range. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-20-44.gh-issue-161106.t0ux5q.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-20-44.gh-issue-161106.t0ux5q.rst deleted file mode 100644 index 78ab254e8fd353a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-20-44.gh-issue-161106.t0ux5q.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :func:`plistlib.dump` and :func:`plistlib.dumps` with -:data:`~plistlib.FMT_BINARY` holding on to memory proportional to the size of -large arrays and dictionaries after the call returns. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-20-52.gh-issue-161096.vkhfyy.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-20-52.gh-issue-161096.vkhfyy.rst deleted file mode 100644 index bae5465f42c6a18..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-20-52.gh-issue-161096.vkhfyy.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :meth:`zoneinfo.ZoneInfo.from_file` accepting TZif data with a negative or -oversized abbreviation block, which could use hundreds of times the file size -in memory; it now raises :exc:`ValueError`. Abbreviation indexes above 127 now -also give the correct time zone abbreviation. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-21-00.gh-issue-161094.MDjCNm.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-21-00.gh-issue-161094.MDjCNm.rst deleted file mode 100644 index 716da00504cc594..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-21-00.gh-issue-161094.MDjCNm.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :func:`zipfile.is_zipfile` raising :exc:`ValueError` instead of returning -``False`` for a file-like object, such as :class:`io.BytesIO`, whose end of -central directory record places the central directory before the start of the -file. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-23-40.gh-issue-161088._-HEWz.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-23-40.gh-issue-161088._-HEWz.rst deleted file mode 100644 index cd8820baffa08f7..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-23-40.gh-issue-161088._-HEWz.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix the :mod:`gzip` command-line interface writing through a symbolic link -at the output file name. Like :program:`gzip`, it now replaces an existing -output file with a new one that has the permission bits of the input file. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-24-21.gh-issue-161112.BFrZRw.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-24-21.gh-issue-161112.BFrZRw.rst deleted file mode 100644 index 73dc245dcb76dd0..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-24-21.gh-issue-161112.BFrZRw.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :func:`shutil.make_archive` adding the zip archive being created to itself -when it is written inside the directory being archived or a symbolic link in -that directory refers to it. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-24-34.gh-issue-161086.FGPjqA.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-24-34.gh-issue-161086.FGPjqA.rst deleted file mode 100644 index d94fadf8f8a6528..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-05-24-34.gh-issue-161086.FGPjqA.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the :mod:`gzip` command-line interface ignoring :option:`gzip --fast` -and :option:`gzip --best` when compressing files given as arguments. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-08-04-26.gh-issue-160297.hL4kQz.rst b/Misc/NEWS.d/next/Library/2026-10-22-08-04-26.gh-issue-160297.hL4kQz.rst deleted file mode 100644 index 1419910948fe15c..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-08-04-26.gh-issue-160297.hL4kQz.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :meth:`tarfile.TarFile.extractall` and :meth:`tarfile.TarFile.extract` -changing the mode, modification time and owner of a file outside the -destination directory when a hard link member is extracted as a copy of a -symbolic link member, for example with the ``'tar'`` filter. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-08-06-39.gh-issue-160513.q7RkZd.rst b/Misc/NEWS.d/next/Library/2026-10-22-08-06-39.gh-issue-160513.q7RkZd.rst deleted file mode 100644 index 1b020723366cd61..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-08-06-39.gh-issue-160513.q7RkZd.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix :meth:`asyncio.loop.start_tls` and :meth:`asyncio.StreamWriter.start_tls` -not calling the protocol's :meth:`~asyncio.BaseProtocol.connection_lost` -when the connection is lost, the handshake times out or the call is -cancelled during the TLS handshake. :meth:`asyncio.StreamWriter.wait_closed` -no longer waits forever after such a failure. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-08-09-29.gh-issue-160406.q7Rk2d.rst b/Misc/NEWS.d/next/Library/2026-10-22-08-09-29.gh-issue-160406.q7Rk2d.rst deleted file mode 100644 index 42f1774b829eee2..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-08-09-29.gh-issue-160406.q7Rk2d.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix in-place filtering in :mod:`fileinput` deleting the file when it cannot -be opened or the output file cannot be created: the backup file is now kept. -The output file is now created exclusively, so :exc:`FileExistsError` is -raised if a file or symbolic link appears under the name of the input file. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst b/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst deleted file mode 100644 index d54069128e2c0eb..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash or garbage data when an :class:`io.StringIO` object is used again -after one of its methods failed with :exc:`MemoryError`. The object now keeps -its contents. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-08-17-51.gh-issue-160316.sRe9pQ.rst b/Misc/NEWS.d/next/Library/2026-10-22-08-17-51.gh-issue-160316.sRe9pQ.rst deleted file mode 100644 index 0e52137bde01ed5..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-08-17-51.gh-issue-160316.sRe9pQ.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix :meth:`re.Pattern.match`, :meth:`re.Pattern.prefixmatch`, -:meth:`re.Pattern.fullmatch` and the ``match()`` and ``prefixmatch()`` methods -of scanner objects returning an empty match, or reading past the end of the -string, when *pos* is greater than *endpos*. They now return ``None``, as -documented. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-08-18-31.gh-issue-160522.cJkSt8.rst b/Misc/NEWS.d/next/Library/2026-10-22-08-18-31.gh-issue-160522.cJkSt8.rst deleted file mode 100644 index ae93269f8714760..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-08-18-31.gh-issue-160522.cJkSt8.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix the incremental encoders and stream writers of the stateful CJK codecs -(``iso2022_*`` and ``hz``) keeping a changed shift state after an -:meth:`!encode` or :meth:`!write` call failed. The next call could omit the -escape sequence, writing multibyte characters in ASCII mode. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-08-34-22.gh-issue-160310.q7Zm2K.rst b/Misc/NEWS.d/next/Library/2026-10-22-08-34-22.gh-issue-160310.q7Zm2K.rst deleted file mode 100644 index 82e62d73bb32ec9..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-08-34-22.gh-issue-160310.q7Zm2K.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix quadratic time complexity of :func:`gzip.decompress`, -:func:`bz2.decompress`, :func:`lzma.decompress` and -:func:`compression.zstd.decompress` on data consisting of many concatenated -members. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-16-18-42.gh-issue-159739.TioR3a.rst b/Misc/NEWS.d/next/Library/2026-10-22-16-18-42.gh-issue-159739.TioR3a.rst deleted file mode 100644 index 69f90a8d30e1ddc..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-16-18-42.gh-issue-159739.TioR3a.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix crashes in :class:`io.TextIOWrapper` and -:class:`io.IncrementalNewlineDecoder` when a callback changes the active -codec or decoded buffer. :class:`io.StringIO` now raises :exc:`ValueError` -if converting the size passed to ``truncate()`` closes the stream. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst b/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst deleted file mode 100644 index 78a3ecc46247c46..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.CjkR5a.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix the ``hz`` codec sometimes encoding ``~`` as a single ``~`` in text that -mixes ASCII and Chinese characters, and the incremental ``iso2022_*`` codecs -mishandling an escape sequence split across calls or keeping the previous -character set after :meth:`!reset`. Also fix possible crashes in the CJK -incremental and stream codecs when their ``errors`` attribute or their class's -``codec`` attribute is replaced while they are in use. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.XmlH3a.rst b/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.XmlH3a.rst deleted file mode 100644 index bda00a7f01f2762..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-16-23-01.gh-issue-159739.XmlH3a.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix crashes in :mod:`xml.parsers.expat` when a character data callback -disables buffering or clears another handler during a flush, or when interning -an attribute name changes :attr:`~xml.parsers.expat.xmlparser.ordered_attributes`. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-16-30-00.gh-issue-159739.mFWntK.rst b/Misc/NEWS.d/next/Library/2026-10-22-16-30-00.gh-issue-159739.mFWntK.rst deleted file mode 100644 index 73e7f5e0c1c15f1..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-16-30-00.gh-issue-159739.mFWntK.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix crashes in :mod:`profiling.tracing` when a custom timer reinitializes the -profiler or its result clears or disables the profiler during conversion or -cleanup. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-16-30-00.gh-issue-159739.tRq417.rst b/Misc/NEWS.d/next/Library/2026-10-22-16-30-00.gh-issue-159739.tRq417.rst deleted file mode 100644 index 1e174fbfc25959a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-16-30-00.gh-issue-159739.tRq417.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix crashes in :mod:`sqlite3` when a row or text factory is replaced during a -query, when cursor metadata is read during execution in a free-threaded build, -or when allocating cursor metadata fails. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst b/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst deleted file mode 100644 index 3694e255546f68a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160314.q7RkZd.rst +++ /dev/null @@ -1,4 +0,0 @@ -Raise an error in :class:`lzma.LZMAFile`, :func:`lzma.decompress`, -:class:`bz2.BZ2File` and :func:`bz2.decompress` for a damaged ``.xz`` or bzip2 -stream that follows another stream, instead of silently ignoring it and -everything after it as trailing data. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160326.nAn7Tq.rst b/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160326.nAn7Tq.rst deleted file mode 100644 index 4349a7380c377d9..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160326.nAn7Tq.rst +++ /dev/null @@ -1,5 +0,0 @@ -Raise :exc:`ValueError` for a NaN *timeout* in -:meth:`threading.Condition.wait` and the blocking calls built on it, such as -:meth:`threading.Event.wait`, :meth:`threading.Semaphore.acquire` and -:meth:`queue.Queue.get`, instead of returning at once or looping without ever -timing out. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160520.sSdTmo.rst b/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160520.sSdTmo.rst deleted file mode 100644 index 241087806c62093..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-18-30-00.gh-issue-160520.sSdTmo.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :mod:`asyncio` SSL transports not applying *ssl_shutdown_timeout* when -the peer has sent EOF but does not read the data still queued for it. -Closing such a connection could wait indefinitely; it is now aborted with -:exc:`TimeoutError` once the timeout expires. diff --git a/Misc/NEWS.d/next/Library/2026-10-22-22-51-40.gh-issue-160386.wYyboz.rst b/Misc/NEWS.d/next/Library/2026-10-22-22-51-40.gh-issue-160386.wYyboz.rst deleted file mode 100644 index 51da0478ee64552..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-22-22-51-40.gh-issue-160386.wYyboz.rst +++ /dev/null @@ -1,11 +0,0 @@ -Applied changes to ``importlib.metadata`` from `importlib_metadata 9.1 -`_: -distribution discovery now prefers the greater of several metadata -directories supplied by one path, so stale metadata left by an unclean -upgrade no longer shadows the version in use; a missing required metadata -field now raises a ``KeyError`` naming both the field and the distribution; -``EntryPoint`` value validation is deferred until the value is used, so one -malformed entry point no longer prevents others from being selected; -``importlib.metadata.diagnose`` no longer crashes on a metadata directory -with no metadata file or no declared name; and ``SimplePath.joinpath`` and -``SimplePath.__truediv__`` take ``other`` positional-only. diff --git a/Misc/NEWS.d/next/Library/2026-10-23-21-52-40.gh-issue-60299.Af6IzL.rst b/Misc/NEWS.d/next/Library/2026-10-23-21-52-40.gh-issue-60299.Af6IzL.rst deleted file mode 100644 index c23e516e131f9cc..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-23-21-52-40.gh-issue-60299.Af6IzL.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :class:`urllib.request.ProxyDigestAuthHandler` failing to authenticate. It -looked up credentials using the target request URL instead of the proxy's -authority, so passwords registered via ``add_password()`` against the proxy -never matched. Patch by Senthil Kumaran. diff --git a/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst b/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst deleted file mode 100644 index ea2082716d64ace..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a crash in :mod:`compression.zstd` when the garbage collector frees a -:class:`~compression.zstd.ZstdCompressor` that uses worker threads together -with its :class:`~compression.zstd.ZstdDict` before the compression of -already submitted data has finished. diff --git a/Misc/NEWS.d/next/Library/2026-10-23-23-21-21.gh-issue-160549.q7ZdMt.rst b/Misc/NEWS.d/next/Library/2026-10-23-23-21-21.gh-issue-160549.q7ZdMt.rst deleted file mode 100644 index f2c21f06f8613e6..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-23-23-21-21.gh-issue-160549.q7ZdMt.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a crash when a :class:`compression.zstd.ZstdCompressor` that uses worker -threads and a :class:`~compression.zstd.ZstdDict` is deallocated, or the -interpreter exits, while data passed to -:meth:`~compression.zstd.ZstdCompressor.compress` is still being compressed. diff --git a/Misc/NEWS.d/next/Library/2026-10-24-01-53-42.gh-issue-160433.DSTlog.rst b/Misc/NEWS.d/next/Library/2026-10-24-01-53-42.gh-issue-160433.DSTlog.rst deleted file mode 100644 index d6779cf60249d00..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-24-01-53-42.gh-issue-160433.DSTlog.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix incorrect backup names around daylight saving time changes in -:class:`logging.handlers.TimedRotatingFileHandler`, including delayed rollovers. diff --git a/Misc/NEWS.d/next/Library/2026-10-24-07-58-00.gh-issue-160561.b3Kq1z.rst b/Misc/NEWS.d/next/Library/2026-10-24-07-58-00.gh-issue-160561.b3Kq1z.rst deleted file mode 100644 index c49cd6a83e90bf4..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-24-07-58-00.gh-issue-160561.b3Kq1z.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a descriptor leak in :func:`socket.fromfd` when the socket constructor rejects its arguments. The duplicated descriptor was never closed. diff --git a/Misc/NEWS.d/next/Library/2026-10-24-22-32-23.gh-issue-160596.RKll-Z.rst b/Misc/NEWS.d/next/Library/2026-10-24-22-32-23.gh-issue-160596.RKll-Z.rst deleted file mode 100644 index c1db616f01d17a4..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-24-22-32-23.gh-issue-160596.RKll-Z.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the error message for invalid data passed to -:meth:`!asyncio.proactor_events.sendto`. diff --git a/Misc/NEWS.d/next/Library/2026-10-26-16-15-29.gh-issue-160427.9G4EBb.rst b/Misc/NEWS.d/next/Library/2026-10-26-16-15-29.gh-issue-160427.9G4EBb.rst deleted file mode 100644 index d57dd02a6d576f5..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-26-16-15-29.gh-issue-160427.9G4EBb.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :attr:`~io.TextIOBase.buffer` of a detached :class:`io.TextIOWrapper` -raising :exc:`ValueError` instead of returning ``None``. diff --git a/Misc/NEWS.d/next/Library/2026-10-26-16-17-45.gh-issue-160708.w7z33g.rst b/Misc/NEWS.d/next/Library/2026-10-26-16-17-45.gh-issue-160708.w7z33g.rst deleted file mode 100644 index 7a88edad93f7c27..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-26-16-17-45.gh-issue-160708.w7z33g.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix out-of-bounds memory access in :func:`bool` on :mod:`ctypes` simple -objects enlarged by :func:`ctypes.resize`, and when passing such objects to -a foreign function. diff --git a/Misc/NEWS.d/next/Library/2026-10-26-21-37-42.gh-issue-158500.zxyuq5.rst b/Misc/NEWS.d/next/Library/2026-10-26-21-37-42.gh-issue-158500.zxyuq5.rst deleted file mode 100644 index 98588df2f8ce83a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-26-21-37-42.gh-issue-158500.zxyuq5.rst +++ /dev/null @@ -1,4 +0,0 @@ -:meth:`datetime.date.fromisoformat` now rejects strings with trailing -characters after a basic-format date, such as ``'2020010112'``. The pure-Python -:meth:`datetime.datetime.fromisoformat` now rejects a date followed by a bare -separator, such as ``'2020-01-01T'``, matching the C implementation. diff --git a/Misc/NEWS.d/next/Library/2026-10-27-15-26-53.gh-issue-160781.4VSeQK.rst b/Misc/NEWS.d/next/Library/2026-10-27-15-26-53.gh-issue-160781.4VSeQK.rst deleted file mode 100644 index f6463a5d5313690..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-27-15-26-53.gh-issue-160781.4VSeQK.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :class:`io.BytesIO` raising :exc:`SystemError` instead of -:exc:`OverflowError` when writing at a position close to -:data:`sys.maxsize`. diff --git a/Misc/NEWS.d/next/Library/2026-10-27-15-27-32.gh-issue-160781.kARuHu.rst b/Misc/NEWS.d/next/Library/2026-10-27-15-27-32.gh-issue-160781.kARuHu.rst deleted file mode 100644 index 1f1e25d8b461b4e..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-27-15-27-32.gh-issue-160781.kARuHu.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :class:`io.StringIO` implementation of :meth:`io.IOBase.truncate` -rejecting sizes larger than ``2**31 - 1`` on Windows. diff --git a/Misc/NEWS.d/next/Library/2026-10-27-15-42-16.gh-issue-160781.VGLB9S.rst b/Misc/NEWS.d/next/Library/2026-10-27-15-42-16.gh-issue-160781.VGLB9S.rst deleted file mode 100644 index 73c8c177fc88282..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-27-15-42-16.gh-issue-160781.VGLB9S.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :class:`io.BytesIO` implementation of :meth:`io.IOBase.truncate` rejecting -sizes larger than ``2**31 - 1`` on Windows. diff --git a/Misc/NEWS.d/next/Library/2026-10-28-07-26-47.gh-issue-160804.R5HeTQ.rst b/Misc/NEWS.d/next/Library/2026-10-28-07-26-47.gh-issue-160804.R5HeTQ.rst deleted file mode 100644 index 36fa06dbb2202b3..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-28-07-26-47.gh-issue-160804.R5HeTQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`asyncio` call graph introspection now includes frames of generator -based awaitables, such as generators returned by ``__await__`` and -:func:`types.coroutine` generators. diff --git a/Misc/NEWS.d/next/Library/2026-10-28-07-34-05.gh-issue-160806.ZZRkhi.rst b/Misc/NEWS.d/next/Library/2026-10-28-07-34-05.gh-issue-160806.ZZRkhi.rst deleted file mode 100644 index 1eb916ce8e98431..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-28-07-34-05.gh-issue-160806.ZZRkhi.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix the pure-Python implementation of :class:`datetime.timedelta` losing -sub-second precision with large float arguments. It now gives the same -results as the C implementation. diff --git a/Misc/NEWS.d/next/Library/2026-10-28-08-12-33.gh-issue-160811.wQ3xT7.rst b/Misc/NEWS.d/next/Library/2026-10-28-08-12-33.gh-issue-160811.wQ3xT7.rst deleted file mode 100644 index 220845de5845a2d..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-28-08-12-33.gh-issue-160811.wQ3xT7.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`io.TextIOWrapper.tell` reading past the end of the buffer after -:meth:`~io.TextIOWrapper.seek`. diff --git a/Misc/NEWS.d/next/Library/2026-10-28-12-00-00.gh-issue-53852.Pr0xy4.rst b/Misc/NEWS.d/next/Library/2026-10-28-12-00-00.gh-issue-53852.Pr0xy4.rst deleted file mode 100644 index 6b6c7fa1e175135..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-28-12-00-00.gh-issue-53852.Pr0xy4.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`urllib.request.ProxyDigestAuthHandler` now raises -:exc:`~urllib.error.HTTPError` with status code 407 (Proxy Authentication -Required) instead of 401 when proxy authentication fails. diff --git a/Misc/NEWS.d/next/Library/2026-10-28-13-23-28.gh-issue-59515.wt31yNN4.rst b/Misc/NEWS.d/next/Library/2026-10-28-13-23-28.gh-issue-59515.wt31yNN4.rst deleted file mode 100644 index 7c4d5b7269aceaa..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-28-13-23-28.gh-issue-59515.wt31yNN4.rst +++ /dev/null @@ -1,4 +0,0 @@ -:class:`urllib.request.HTTPDigestAuthHandler` and -:class:`urllib.request.ProxyDigestAuthHandler` now parse all -``WWW-Authenticate`` (or ``Proxy-Authenticate``) headers and accept -multiple challenges per header: the first Digest challenge is used. diff --git a/Misc/NEWS.d/next/Library/2026-10-28-16-40-00.gh-issue-66333.pdb-sigint.rst b/Misc/NEWS.d/next/Library/2026-10-28-16-40-00.gh-issue-66333.pdb-sigint.rst deleted file mode 100644 index 057caa487e9631b..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-28-16-40-00.gh-issue-66333.pdb-sigint.rst +++ /dev/null @@ -1,3 +0,0 @@ -Allow Ctrl-C to interrupt execution resumed with the :mod:`pdb` -``next``, ``until``, and ``return`` commands without unwinding the debugged -program, as with ``continue``. diff --git a/Misc/NEWS.d/next/Library/2026-10-28-20-36-44.gh-issue-109564.WqoyNE.rst b/Misc/NEWS.d/next/Library/2026-10-28-20-36-44.gh-issue-109564.WqoyNE.rst deleted file mode 100644 index 66821f5cdc6e919..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-28-20-36-44.gh-issue-109564.WqoyNE.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a race in :mod:`asyncio` where closing a server after it accepted a -connection, but before the connection's transport was created, reported an -:exc:`AssertionError` to the exception handler. Such connections are now -closed quietly. diff --git a/Misc/NEWS.d/next/Library/2026-10-29-19-42-53.gh-issue-53089.ewaIBvY6.rst b/Misc/NEWS.d/next/Library/2026-10-29-19-42-53.gh-issue-53089.ewaIBvY6.rst deleted file mode 100644 index 6fe3aace25ffe1f..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-29-19-42-53.gh-issue-53089.ewaIBvY6.rst +++ /dev/null @@ -1,5 +0,0 @@ -:class:`urllib.request.HTTPDigestAuthHandler` now sends ``uri="/"`` in the -Digest ``Authorization`` header for URLs with an empty path, such as -``http://example.com``, matching the request-target sent by :mod:`http.client`. -Previously it sent ``uri=""``, which servers could reject. -Patch by Senthil Kumaran. diff --git a/Misc/NEWS.d/next/Library/2026-10-29-20-52-57.gh-issue-50720.Kd1VMR.rst b/Misc/NEWS.d/next/Library/2026-10-29-20-52-57.gh-issue-50720.Kd1VMR.rst deleted file mode 100644 index 263b7ac89585769..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-29-20-52-57.gh-issue-50720.Kd1VMR.rst +++ /dev/null @@ -1,4 +0,0 @@ -:exc:`urllib.error.URLError` now sets the :attr:`~OSError.errno` and -:attr:`~OSError.strerror` attributes from -:attr:`~urllib.error.URLError.reason` when it is an :exc:`OSError`. On -Windows, :attr:`~OSError.winerror` is set as well. diff --git a/Misc/NEWS.d/next/Library/2026-10-29-21-36-08.gh-issue-160929.mK8n2Q.rst b/Misc/NEWS.d/next/Library/2026-10-29-21-36-08.gh-issue-160929.mK8n2Q.rst deleted file mode 100644 index 09bd1e568e7e9bd..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-29-21-36-08.gh-issue-160929.mK8n2Q.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix reverse union with :class:`~collections.ChainMap` to preserve its lookup -semantics for defaulting mappings such as :class:`~collections.Counter` and -:class:`~collections.defaultdict`. diff --git a/Misc/NEWS.d/next/Library/2026-10-29-21-43-53.gh-issue-160429.eWapMs.rst b/Misc/NEWS.d/next/Library/2026-10-29-21-43-53.gh-issue-160429.eWapMs.rst deleted file mode 100644 index cd8bebd8c09acb5..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-29-21-43-53.gh-issue-160429.eWapMs.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :class:`asyncio.ProactorEventLoop` socket transports leaving the socket -open, and :meth:`asyncio.Server.wait_closed` hanging, when the peer resets -the connection while the transport is being closed. diff --git a/Misc/NEWS.d/next/Library/2026-10-29-23-25-31.gh-issue-160782.1ZwCOr.rst b/Misc/NEWS.d/next/Library/2026-10-29-23-25-31.gh-issue-160782.1ZwCOr.rst deleted file mode 100644 index 57ff63c1f6d3f7e..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-29-23-25-31.gh-issue-160782.1ZwCOr.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`venv`: a backslash before a quote in the prompt or the path no longer -breaks ``activate.fish``. diff --git a/Misc/NEWS.d/next/Library/2026-10-30-10-36-30.gh-issue-160782.n6JCUU.rst b/Misc/NEWS.d/next/Library/2026-10-30-10-36-30.gh-issue-160782.n6JCUU.rst deleted file mode 100644 index d4da24bdca0b5e8..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-30-10-36-30.gh-issue-160782.n6JCUU.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`venv`: a symbolic link or junction in place of a directory inside the -environment now raises :exc:`ValueError` instead of being followed. diff --git a/Misc/NEWS.d/next/Library/2026-10-30-12-00-00.gh-issue-137549.Tm0Bxg.rst b/Misc/NEWS.d/next/Library/2026-10-30-12-00-00.gh-issue-137549.Tm0Bxg.rst deleted file mode 100644 index 32c27840f5b08ba..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-30-12-00-00.gh-issue-137549.Tm0Bxg.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`asyncio.timeout` now inserts :exc:`TimeoutError` into the context -chain of sub-exceptions of a :exc:`BaseExceptionGroup`, as it already did for -:exc:`ExceptionGroup`. diff --git a/Misc/NEWS.d/next/Library/2026-10-31-21-34-00.gh-issue-161039.k7_pTD.rst b/Misc/NEWS.d/next/Library/2026-10-31-21-34-00.gh-issue-161039.k7_pTD.rst deleted file mode 100644 index 3b697fb155b7aca..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-10-31-21-34-00.gh-issue-161039.k7_pTD.rst +++ /dev/null @@ -1 +0,0 @@ -Fix the :mod:`pydoc` pager not detecting ``less`` 706 and newer. diff --git a/Misc/NEWS.d/next/Library/2026-11-01-19-38-33.gh-issue-161083.PM21lv.rst b/Misc/NEWS.d/next/Library/2026-11-01-19-38-33.gh-issue-161083.PM21lv.rst deleted file mode 100644 index 98b3ea827276518..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-01-19-38-33.gh-issue-161083.PM21lv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix stale task nodes left by :func:`asyncio.gather` when argument validation -fails. Patch by Timofei Ivankov diff --git a/Misc/NEWS.d/next/Library/2026-11-01-22-07-24.gh-issue-161124.c2YmcV.rst b/Misc/NEWS.d/next/Library/2026-11-01-22-07-24.gh-issue-161124.c2YmcV.rst deleted file mode 100644 index 32cfb56adb83167..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-01-22-07-24.gh-issue-161124.c2YmcV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`asyncio.print_call_graph` reporting event loop frames when -*depth* exceeds the task stack. Patch by Timofei Ivankov. diff --git a/Misc/NEWS.d/next/Library/2026-11-02-13-11-30.gh-issue-161166.zfFDlW.rst b/Misc/NEWS.d/next/Library/2026-11-02-13-11-30.gh-issue-161166.zfFDlW.rst deleted file mode 100644 index 4d66a399f5e6550..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-02-13-11-30.gh-issue-161166.zfFDlW.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`asyncio.Task.print_stack` displaying an incorrect exception line -after a ``finally`` block. Patch by Timofei Ivankov. diff --git a/Misc/NEWS.d/next/Library/2026-11-02-13-26-36.gh-issue-161179.H8Yyaw.rst b/Misc/NEWS.d/next/Library/2026-11-02-13-26-36.gh-issue-161179.H8Yyaw.rst deleted file mode 100644 index 81d12633029959e..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-02-13-26-36.gh-issue-161179.H8Yyaw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the function names in the error messages of :func:`os.spawnve`, -:func:`os.preadv` and :meth:`tkinter.Wm.wm_attributes`. diff --git a/Misc/NEWS.d/next/Library/2026-11-02-16-28-11.gh-issue-161178.D3-23R.rst b/Misc/NEWS.d/next/Library/2026-11-02-16-28-11.gh-issue-161178.D3-23R.rst deleted file mode 100644 index 1130fbe95ba0190..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-02-16-28-11.gh-issue-161178.D3-23R.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`asyncio.Task.print_stack` to include frames from async generator -operations. Patch by Timofei Ivankov diff --git a/Misc/NEWS.d/next/Library/2026-11-02-21-10-00.gh-issue-161212.pip263.rst b/Misc/NEWS.d/next/Library/2026-11-02-21-10-00.gh-issue-161212.pip263.rst deleted file mode 100644 index af137800454588a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-02-21-10-00.gh-issue-161212.pip263.rst +++ /dev/null @@ -1 +0,0 @@ -Bump the version of :pypi:`pip` bundled in :mod:`ensurepip` to version 26.3. diff --git a/Misc/NEWS.d/next/Library/2026-11-03-10-33-33.gh-issue-161233._9VrfL.rst b/Misc/NEWS.d/next/Library/2026-11-03-10-33-33.gh-issue-161233._9VrfL.rst deleted file mode 100644 index 96e9f5e39df3ec1..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-03-10-33-33.gh-issue-161233._9VrfL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix unraisable :exc:`TypeError` when ``None`` was passed as a *timer* to -:class:`cProfile.Profile `. diff --git a/Misc/NEWS.d/next/Library/2026-11-03-12-00-00.gh-issue-161076.nOdLy1.rst b/Misc/NEWS.d/next/Library/2026-11-03-12-00-00.gh-issue-161076.nOdLy1.rst deleted file mode 100644 index bc286d0a676c024..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-03-12-00-00.gh-issue-161076.nOdLy1.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix :mod:`asyncio` leaving a half-created transport registered with the -event loop and the server when setting ``TCP_NODELAY`` on a socket fails. -Such a failure could make :meth:`asyncio.Server.wait_closed` hang and -prevent new connections from being served. Errors from -``setsockopt(TCP_NODELAY)`` are now ignored. diff --git a/Misc/NEWS.d/next/Library/2026-11-03-12-00-00.gh-issue-93821.aCcRsT.rst b/Misc/NEWS.d/next/Library/2026-11-03-12-00-00.gh-issue-93821.aCcRsT.rst deleted file mode 100644 index a8a0da86e6abdbe..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-03-12-00-00.gh-issue-93821.aCcRsT.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :class:`asyncio.ProactorEventLoop` servers that stopped accepting new -connections on Windows when a client reset its connection before it was -accepted (``WinError 64``, ``ERROR_NETNAME_DELETED``). The listening socket is -now kept open and the server continues to accept connections. diff --git a/Misc/NEWS.d/next/Library/2026-11-03-18-56-28.gh-issue-161259.iyZz6y.rst b/Misc/NEWS.d/next/Library/2026-11-03-18-56-28.gh-issue-161259.iyZz6y.rst deleted file mode 100644 index 400f6e5b716941c..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-03-18-56-28.gh-issue-161259.iyZz6y.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improved the accuracy of :func:`statistics.correlation` for very small -inputs, where the product of the sums of squares is subnormal rather -than zero. diff --git a/Misc/NEWS.d/next/Library/2026-11-03-21-26-09.gh-issue-161295.IJqoJM.rst b/Misc/NEWS.d/next/Library/2026-11-03-21-26-09.gh-issue-161295.IJqoJM.rst deleted file mode 100644 index f2ede87899e9406..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-03-21-26-09.gh-issue-161295.IJqoJM.rst +++ /dev/null @@ -1,2 +0,0 @@ -:class:`reprlib.Repr` objects now have a meaningful :meth:`~object.__repr__` -describing the instance attributes. diff --git a/Misc/NEWS.d/next/Library/2026-11-05-10-05-00.gh-issue-161387.def456.rst b/Misc/NEWS.d/next/Library/2026-11-05-10-05-00.gh-issue-161387.def456.rst deleted file mode 100644 index f445980563853bb..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-05-10-05-00.gh-issue-161387.def456.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash in :mod:`decimal` when the internal ``decimal_context`` -:class:`contextvars.ContextVar` holds a value that is not a context. diff --git a/Misc/NEWS.d/next/Library/2026-11-05-11-56-50.gh-issue-161389.EIl2E6.rst b/Misc/NEWS.d/next/Library/2026-11-05-11-56-50.gh-issue-161389.EIl2E6.rst deleted file mode 100644 index 62fd6fe712f6dfc..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-05-11-56-50.gh-issue-161389.EIl2E6.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixes a crash in :class:`decimal.Context` when passing invalid signaling -dicts as *flags* and *traps*. diff --git a/Misc/NEWS.d/next/Library/2026-11-05-11-59-58.gh-issue-161414.Xnxe8A.rst b/Misc/NEWS.d/next/Library/2026-11-05-11-59-58.gh-issue-161414.Xnxe8A.rst deleted file mode 100644 index 7536c5c25a13f5b..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-05-11-59-58.gh-issue-161414.Xnxe8A.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove the backquote from the symbols listed by :func:`help`, asking for help on it reported that no -documentation was found. diff --git a/Misc/NEWS.d/next/Library/2026-11-05-12-00-24.gh-issue-161415.NJhrAR.rst b/Misc/NEWS.d/next/Library/2026-11-05-12-00-24.gh-issue-161415.NJhrAR.rst deleted file mode 100644 index 22cc4665c253dde..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-05-12-00-24.gh-issue-161415.NJhrAR.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove ``FILES`` and ``unicode`` from the related help topics listed by -:func:`help` for the ``TYPES`` and ``UNICODE`` topics. Neither exists in -Python 3, so asking for help on them failed. diff --git a/Misc/NEWS.d/next/Library/2026-11-06-00-00-00.gh-issue-142782.qU7nE4.rst b/Misc/NEWS.d/next/Library/2026-11-06-00-00-00.gh-issue-142782.qU7nE4.rst deleted file mode 100644 index 4bccc120ab8d33f..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-06-00-00-00.gh-issue-142782.qU7nE4.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a use-after-free in :class:`zoneinfo.ZoneInfo` when a key's ``__eq__`` -clears the cache during a lookup. Strong cache keys are now compared by -string contents. diff --git a/Misc/NEWS.d/next/Library/2026-11-06-05-42-11.gh-issue-71199.qJN4cV.rst b/Misc/NEWS.d/next/Library/2026-11-06-05-42-11.gh-issue-71199.qJN4cV.rst deleted file mode 100644 index fde88d5facc9511..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-06-05-42-11.gh-issue-71199.qJN4cV.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change :mod:`os` module types such as :class:`~os.DirEntry` to report ``os`` -as their module on every platform. :ref:`Pickled ` objects -using the platform-specific qualified name, such as ``nt.times_result``, still load. diff --git a/Misc/NEWS.d/next/Library/2026-11-06-11-59-26.gh-issue-161475.82LWf4.rst b/Misc/NEWS.d/next/Library/2026-11-06-11-59-26.gh-issue-161475.82LWf4.rst deleted file mode 100644 index 3033bc7543b9016..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-11-06-11-59-26.gh-issue-161475.82LWf4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :mod:`ctypes` structures containing arrays of floating-point type -returned by value on 64-bit Linux SPARC. diff --git a/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCFSN.rst b/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCFSN.rst deleted file mode 100644 index 3220f4613a91af6..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-08-31-16-47-33.gh-issue-157953.KxCFSN.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled `libexpat `_ to version 2.9.0. diff --git a/Misc/NEWS.d/next/Security/2026-10-06-12-30-00.gh-issue-158928.Zd9Ybv.rst b/Misc/NEWS.d/next/Security/2026-10-06-12-30-00.gh-issue-158928.Zd9Ybv.rst deleted file mode 100644 index 2b9597c5ed94f32..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-06-12-30-00.gh-issue-158928.Zd9Ybv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a buffer overflow when initializing a :class:`bytearray` from an -iterator. diff --git a/Misc/NEWS.d/next/Security/2026-10-15-03-47-08.gh-issue-159428.XHRKwY.rst b/Misc/NEWS.d/next/Security/2026-10-15-03-47-08.gh-issue-159428.XHRKwY.rst deleted file mode 100644 index d481ed2b74a20ea..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-15-03-47-08.gh-issue-159428.XHRKwY.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`mimetypes` no longer reads the files listed in :data:`mimetypes.knownfiles` -by default on Windows, and uses only the registry. diff --git a/Misc/NEWS.d/next/Security/2026-10-20-13-15-00.gh-issue-159894.8Kp4mN.rst b/Misc/NEWS.d/next/Security/2026-10-20-13-15-00.gh-issue-159894.8Kp4mN.rst deleted file mode 100644 index 0f27a475b5883eb..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-20-13-15-00.gh-issue-159894.8Kp4mN.rst +++ /dev/null @@ -1 +0,0 @@ -Bump the version of :pypi:`pip` bundled in :mod:`ensurepip` to version 26.2.2. diff --git a/Misc/NEWS.d/next/Security/2026-10-21-16-55-07.gh-issue-160040.9PjFjC.rst b/Misc/NEWS.d/next/Security/2026-10-21-16-55-07.gh-issue-160040.9PjFjC.rst deleted file mode 100644 index 7bcec7f5808cac6..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-21-16-55-07.gh-issue-160040.9PjFjC.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. cve: CVE-2026-115557 - -:mod:`ssl` now rejects TLS certificate chains that contain any of a set of -compromised eMudhra intermediate CA keys. This is an extraordinary measure and -a last line of defense: removing trust is primarily the job of the platform root -store, and the built-in denylist only blocks connections while that update has -not yet reached the OS, a container image, or a bundled trust store in various -Python packages. Entries are added to the denylist only under a very high -threshold. -The denylist matches on the public key (the SHA-256 digest of the :rfc:`5280` -``SubjectPublicKeyInfo``), so it also covers the cross-signed certificates that -reuse one of the distrusted keys. diff --git a/Misc/NEWS.d/next/Security/2026-10-21-19-58-53.gh-issue-160097.-opfAL.rst b/Misc/NEWS.d/next/Security/2026-10-21-19-58-53.gh-issue-160097.-opfAL.rst deleted file mode 100644 index b48c8f7213ee826..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-21-19-58-53.gh-issue-160097.-opfAL.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. cve: CVE-2026-114583 - -:func:`tarfile.tar_filter` now rejects link members whose path resolves to -the destination directory itself, as :func:`tarfile.data_filter` already -did. diff --git a/Misc/NEWS.d/next/Security/2026-10-22-08-04-33.gh-issue-160300.q7Zk2B.rst b/Misc/NEWS.d/next/Security/2026-10-22-08-04-33.gh-issue-160300.q7Zk2B.rst deleted file mode 100644 index 3cadc6c2c4fd1fb..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-22-08-04-33.gh-issue-160300.q7Zk2B.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :class:`http.cookiejar.DefaultCookiePolicy` accepting and returning -cookies with a negative ``Version`` attribute without checking their domain -against the request host. Such cookies are now ignored. diff --git a/Misc/NEWS.d/next/Security/2026-10-22-08-05-32.gh-issue-160420.tZ9dRv.rst b/Misc/NEWS.d/next/Security/2026-10-22-08-05-32.gh-issue-160420.tZ9dRv.rst deleted file mode 100644 index 2255056dac577e7..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-22-08-05-32.gh-issue-160420.tZ9dRv.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :class:`zoneinfo.ZoneInfo` on Windows accepting keys in which a component -after a ``/`` starts with a drive letter, such as ``America/C:\file``. Such a -key could open a file outside the ``tzdata`` package; it now raises -:exc:`ValueError`. diff --git a/Misc/NEWS.d/next/Security/2026-10-22-08-07-52.gh-issue-161323.q7Drv2.rst b/Misc/NEWS.d/next/Security/2026-10-22-08-07-52.gh-issue-161323.q7Drv2.rst deleted file mode 100644 index 95949e93576c6cb..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-22-08-07-52.gh-issue-161323.q7Drv2.rst +++ /dev/null @@ -1,5 +0,0 @@ -On Windows, the ``'tar'`` and ``'data'`` :mod:`tarfile` extraction filters -now refuse member names and link targets that have a drive but are not -absolute, such as ``C:foo``, with :exc:`~tarfile.AbsolutePathError` or -:exc:`~tarfile.AbsoluteLinkError`. Such members could be extracted outside -the destination directory. Fixes :cve:`2026-107784`. diff --git a/Misc/NEWS.d/next/Security/2026-10-22-08-08-23.gh-issue-160506.rN4dLq.rst b/Misc/NEWS.d/next/Security/2026-10-22-08-08-23.gh-issue-160506.rN4dLq.rst deleted file mode 100644 index 07ba8fdbff2cef4..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-22-08-08-23.gh-issue-160506.rN4dLq.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :func:`os.path.realpath` on Windows returning the wrong path for a broken -relative symbolic link reached through a symbolic link to a directory. This -allowed :mod:`tarfile` extraction with the ``'data'`` or ``'tar'`` filter to -create a file outside the destination directory. diff --git a/Misc/NEWS.d/next/Security/2026-10-22-08-08-57.gh-issue-160499.pf9H9g.rst b/Misc/NEWS.d/next/Security/2026-10-22-08-08-57.gh-issue-160499.pf9H9g.rst deleted file mode 100644 index 4fcfc9edd90b782..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-22-08-08-57.gh-issue-160499.pf9H9g.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix unbounded memory growth in :mod:`asyncio` TLS connections when a peer -stops reading but keeps sending records that make OpenSSL generate its own -replies (such as TLS 1.2 renegotiation alerts). Reading from the peer is now -paused when this output accumulates, instead of only when application data -does. diff --git a/Misc/NEWS.d/next/Security/2026-10-22-11-09-30.gh-issue-160279.9BtxeF.rst b/Misc/NEWS.d/next/Security/2026-10-22-11-09-30.gh-issue-160279.9BtxeF.rst deleted file mode 100644 index d143e742f52b0e1..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-22-11-09-30.gh-issue-160279.9BtxeF.rst +++ /dev/null @@ -1,2 +0,0 @@ -Prevent :mod:`venv` from importing modules from the root of an existing -virtual environment when installing or upgrading pip. diff --git a/Misc/NEWS.d/next/Security/2026-10-22-21-59-39.gh-issue-160369.imapTLS.rst b/Misc/NEWS.d/next/Security/2026-10-22-21-59-39.gh-issue-160369.imapTLS.rst deleted file mode 100644 index 35c0a7527457935..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-22-21-59-39.gh-issue-160369.imapTLS.rst +++ /dev/null @@ -1,4 +0,0 @@ -.. cve: CVE-2026-115559 - -:meth:`imaplib.IMAP4.starttls` now rejects unexpected plaintext buffered after -the STARTTLS response, preventing it from being processed as TLS-protected data. diff --git a/Misc/NEWS.d/next/Security/2026-10-27-16-25-40.gh-issue-160898.epi53X.rst b/Misc/NEWS.d/next/Security/2026-10-27-16-25-40.gh-issue-160898.epi53X.rst deleted file mode 100644 index 2eb04454cbd9a7f..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-10-27-16-25-40.gh-issue-160898.epi53X.rst +++ /dev/null @@ -1,4 +0,0 @@ -.. cve: CVE-2026-116566 - -:func:`socket.getaddrinfo` now rejects null bytes within *host* and *port* -parameters as this previously caused the values to be silently truncated. diff --git a/Misc/NEWS.d/next/Security/2026-11-02-18-30-09.gh-issue-161192.2ScSRO.rst b/Misc/NEWS.d/next/Security/2026-11-02-18-30-09.gh-issue-161192.2ScSRO.rst deleted file mode 100644 index 14682058abb2351..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-11-02-18-30-09.gh-issue-161192.2ScSRO.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update bundled `libexpat `_ to version 2.10.0 -for the fixes to :cve:`2026-113351`, :cve:`2026-115506`, :cve:`2026-117610` -and :cve:`2026-118370`. diff --git a/Misc/NEWS.d/next/Security/2026-11-06-16-22-47.gh-issue-161503.FkIl87.rst b/Misc/NEWS.d/next/Security/2026-11-06-16-22-47.gh-issue-161503.FkIl87.rst deleted file mode 100644 index 308ba0c83f3efe4..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-11-06-16-22-47.gh-issue-161503.FkIl87.rst +++ /dev/null @@ -1,4 +0,0 @@ -.. cve: CVE-2026-19552 - -:func:`ssl.get_server_certificate` now verifies that the server hostname matches -its certificate when the caller specifies *ca_certs*. diff --git a/Misc/NEWS.d/next/Security/2026-11-07-01-43-49.gh-issue-161569.cookiej.rst b/Misc/NEWS.d/next/Security/2026-11-07-01-43-49.gh-issue-161569.cookiej.rst deleted file mode 100644 index b16f6caf90f1b06..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-11-07-01-43-49.gh-issue-161569.cookiej.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. cve: CVE-2026-124249 - -Fix :class:`http.cookiejar.CookieJar` deleting a stored cookie when a -response from an unrelated host sets an already expired cookie with a -``Domain`` or ``Path`` attribute pointing at that cookie. The cookie policy -is now consulted before an expired cookie is allowed to delete an existing -one. diff --git a/Misc/NEWS.d/next/Security/2026-11-07-15-56-47.gh-issue-161705.OjDK2d.rst b/Misc/NEWS.d/next/Security/2026-11-07-15-56-47.gh-issue-161705.OjDK2d.rst deleted file mode 100644 index d9a862ed858e012..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-11-07-15-56-47.gh-issue-161705.OjDK2d.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. cve: CVE-2026-115791 - -Remove ``sys.path[0]`` in :mod:`gzip`, :mod:`tarfile` and :mod:`zipfile` -command line interface to prevent shadowing standard library modules from -extracted files. Issue reported by Jelle Zijlstra, patch by Gregory P. -Smith and Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst b/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst deleted file mode 100644 index a1c9e648e5c744e..000000000000000 --- a/Misc/NEWS.d/next/Tests/2026-10-05-22-36-06.gh-issue-158895.3Dmc4J.rst +++ /dev/null @@ -1,3 +0,0 @@ -When run in GitHub Actions, :mod:`test.regrtest` now annotates each test -failure in the job log and writes a summary of the failures to the job -summary. diff --git a/Misc/NEWS.d/next/Tests/2026-10-07-12-00-58.gh-issue-158975.yR8q2L.rst b/Misc/NEWS.d/next/Tests/2026-10-07-12-00-58.gh-issue-158975.yR8q2L.rst deleted file mode 100644 index 450af29aea272b7..000000000000000 --- a/Misc/NEWS.d/next/Tests/2026-10-07-12-00-58.gh-issue-158975.yR8q2L.rst +++ /dev/null @@ -1,3 +0,0 @@ -Replace fixed sleeps used for synchronization in multiprocessing tests with -bounded waits to avoid failures on slow or overloaded systems. Ensure pool -cleanup runs even when assertions fail in the imap buffer tests. diff --git a/Misc/NEWS.d/next/Tests/2026-10-13-13-34-00.gh-issue-159293.Lr8q2N.rst b/Misc/NEWS.d/next/Tests/2026-10-13-13-34-00.gh-issue-159293.Lr8q2N.rst deleted file mode 100644 index 2a122343a3fe2af..000000000000000 --- a/Misc/NEWS.d/next/Tests/2026-10-13-13-34-00.gh-issue-159293.Lr8q2N.rst +++ /dev/null @@ -1,2 +0,0 @@ -Restore the recursion limit after the free-threading recursion-limit race test -to avoid failures in subsequent tests run in the same process. diff --git a/Misc/NEWS.d/next/Tests/2026-10-19-17-05-00.gh-issue-107862.qH3sTa.rst b/Misc/NEWS.d/next/Tests/2026-10-19-17-05-00.gh-issue-107862.qH3sTa.rst deleted file mode 100644 index 624fd82d50041c1..000000000000000 --- a/Misc/NEWS.d/next/Tests/2026-10-19-17-05-00.gh-issue-107862.qH3sTa.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``hypothesis.stateful`` and the ``timezones()`` and ``ip_addresses()`` -strategies to the Hypothesis stubs used when Hypothesis is not installed. diff --git a/Misc/NEWS.d/next/Tests/2026-10-28-20-40-00.gh-issue-158646.k2Xw7q.rst b/Misc/NEWS.d/next/Tests/2026-10-28-20-40-00.gh-issue-158646.k2Xw7q.rst deleted file mode 100644 index 93a7c26c6321405..000000000000000 --- a/Misc/NEWS.d/next/Tests/2026-10-28-20-40-00.gh-issue-158646.k2Xw7q.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a race in ``test_asyncio.test_sslproto`` where the server side of -``test_shutdown_corrupted_ssl_sends_close_notify`` could fail with a -connection reset on Windows. diff --git a/Misc/NEWS.d/next/Tests/2026-11-03-13-12-50.gh-issue-161263.Voe8VM.rst b/Misc/NEWS.d/next/Tests/2026-11-03-13-12-50.gh-issue-161263.Voe8VM.rst deleted file mode 100644 index 98c631cc2f663a3..000000000000000 --- a/Misc/NEWS.d/next/Tests/2026-11-03-13-12-50.gh-issue-161263.Voe8VM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Skip tests which use MD5 algorithm for WWW-Authenticate challenges when -libssl is in FIPS mode. diff --git a/Misc/NEWS.d/next/Tests/2026-11-05-16-20-23.gh-issue-161430.M7b8N6.rst b/Misc/NEWS.d/next/Tests/2026-11-05-16-20-23.gh-issue-161430.M7b8N6.rst deleted file mode 100644 index b53d2f01d5ce141..000000000000000 --- a/Misc/NEWS.d/next/Tests/2026-11-05-16-20-23.gh-issue-161430.M7b8N6.rst +++ /dev/null @@ -1,2 +0,0 @@ -No longer list tests expected to be skipped in the final report. Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2026-11-06-23-15-23.gh-issue-161517.LkAlDt.rst b/Misc/NEWS.d/next/Tests/2026-11-06-23-15-23.gh-issue-161517.LkAlDt.rst deleted file mode 100644 index fc76d235d207f0e..000000000000000 --- a/Misc/NEWS.d/next/Tests/2026-11-06-23-15-23.gh-issue-161517.LkAlDt.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix test_gdb when Python is built out-of-tree and Python is run from a -virtual environment. Previously, test_gdb was just skipped. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Tests/2026-11-07-01-45-00.gh-issue-160334.F2n8qA.rst b/Misc/NEWS.d/next/Tests/2026-11-07-01-45-00.gh-issue-160334.F2n8qA.rst deleted file mode 100644 index 458f192ce0be77d..000000000000000 --- a/Misc/NEWS.d/next/Tests/2026-11-07-01-45-00.gh-issue-160334.F2n8qA.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the sampling profiler integration test for special frames without file and -line information. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-03-25-18-00-00.gh-issue-131725.qwfh321.rst b/Misc/NEWS.d/next/Tools-Demos/2025-03-25-18-00-00.gh-issue-131725.qwfh321.rst deleted file mode 100644 index c7a8a3e492ad82b..000000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-03-25-18-00-00.gh-issue-131725.qwfh321.rst +++ /dev/null @@ -1,2 +0,0 @@ -``Tools/i18n/msgfmt.py`` now writes a hash table to the generated ``.mo`` -file, as GNU :program:`msgfmt` does. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-10-14-09-48-26.gh-issue-159385.0aWOA9.rst b/Misc/NEWS.d/next/Tools-Demos/2026-10-14-09-48-26.gh-issue-159385.0aWOA9.rst deleted file mode 100644 index e76a5e666df2364..000000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2026-10-14-09-48-26.gh-issue-159385.0aWOA9.rst +++ /dev/null @@ -1 +0,0 @@ -iOS builds can now be configured to generate free-threading or debug output. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-10-20-12-50-00.gh-issue-114258.slotNx.rst b/Misc/NEWS.d/next/Tools-Demos/2026-10-20-12-50-00.gh-issue-114258.slotNx.rst deleted file mode 100644 index afaeda209e1bd26..000000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2026-10-20-12-50-00.gh-issue-114258.slotNx.rst +++ /dev/null @@ -1,3 +0,0 @@ -Argument Clinic now supports the ``__next__`` type slot (``tp_iternext``), -including ``@critical_section``. - diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-10-21-11-53-40.gh-issue-87613.tmKBlg.rst b/Misc/NEWS.d/next/Tools-Demos/2026-10-21-11-53-40.gh-issue-87613.tmKBlg.rst deleted file mode 100644 index b52d7ea7fe5e8d1..000000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2026-10-21-11-53-40.gh-issue-87613.tmKBlg.rst +++ /dev/null @@ -1,2 +0,0 @@ -Extend Argument Clinic :ref:`vectorcall` to support :ref:`heap-types` with -zero argument construction. diff --git a/Misc/NEWS.d/next/WASI/2026-10-21-15-26-12.gh-issue-160135.UP-DlO.rst b/Misc/NEWS.d/next/WASI/2026-10-21-15-26-12.gh-issue-160135.UP-DlO.rst deleted file mode 100644 index 2f3b76136afd24e..000000000000000 --- a/Misc/NEWS.d/next/WASI/2026-10-21-15-26-12.gh-issue-160135.UP-DlO.rst +++ /dev/null @@ -1 +0,0 @@ -Support wasm32-wasip3. diff --git a/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst b/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst deleted file mode 100644 index 7d3e16444338007..000000000000000 --- a/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add a ``test`` command to :file:`Platforms/WASI` with ``--fast-ci`` and -``--slow-ci`` to select fast CI or slow Buildbot tests and ``--target`` to -test the build, gathered distribution, or release archive. diff --git a/Misc/NEWS.d/next/Windows/2026-09-14-16-49-44.gh-issue-157418.r2IggA.rst b/Misc/NEWS.d/next/Windows/2026-09-14-16-49-44.gh-issue-157418.r2IggA.rst deleted file mode 100644 index b1a048c46ab83cc..000000000000000 --- a/Misc/NEWS.d/next/Windows/2026-09-14-16-49-44.gh-issue-157418.r2IggA.rst +++ /dev/null @@ -1,3 +0,0 @@ -Correct the third parameter name of ``IocpProactor.recvfrom_into()`` from -*flags* to *nbytes* to reflect that it specifies the number of bytes to -receive. diff --git a/Misc/NEWS.d/next/Windows/2026-10-15-18-45-00.gh-issue-159118.wmiMTA.rst b/Misc/NEWS.d/next/Windows/2026-10-15-18-45-00.gh-issue-159118.wmiMTA.rst deleted file mode 100644 index 561a5005b759755..000000000000000 --- a/Misc/NEWS.d/next/Windows/2026-10-15-18-45-00.gh-issue-159118.wmiMTA.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a crash (``RPC_NT_INTERNAL_ERROR``) and intermittent window class failures in :mod:`!_wmi` under high concurrency and load by initializing COM as multi-threaded (MTA) instead of apartment-threaded (STA). diff --git a/Misc/NEWS.d/next/Windows/2026-10-20-21-37-34.gh-issue-159880.eKBiLr.rst b/Misc/NEWS.d/next/Windows/2026-10-20-21-37-34.gh-issue-159880.eKBiLr.rst deleted file mode 100644 index ec3c28c4c4303a5..000000000000000 --- a/Misc/NEWS.d/next/Windows/2026-10-20-21-37-34.gh-issue-159880.eKBiLr.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a key handle leak in :func:`winreg.CreateKey`, -:func:`winreg.CreateKeyEx` and :func:`winreg.OpenKey` when the -``winreg.OpenKey/result`` audit hook fails. diff --git a/Misc/NEWS.d/next/Windows/2026-10-22-14-04-17.gh-issue-160239.A1b2C3.rst b/Misc/NEWS.d/next/Windows/2026-10-22-14-04-17.gh-issue-160239.A1b2C3.rst deleted file mode 100644 index 6ca8aace90e676e..000000000000000 --- a/Misc/NEWS.d/next/Windows/2026-10-22-14-04-17.gh-issue-160239.A1b2C3.rst +++ /dev/null @@ -1 +0,0 @@ -Remove dead and broken ``_repr_info()`` methods from ``asyncio.windows_events``; they called ``Future._repr_info()``, which no longer exists. diff --git a/Misc/NEWS.d/next/macOS/2026-10-19-19-33-35.gh-issue-159739.aaf575.rst b/Misc/NEWS.d/next/macOS/2026-10-19-19-33-35.gh-issue-159739.aaf575.rst deleted file mode 100644 index 0e29daaa307e221..000000000000000 --- a/Misc/NEWS.d/next/macOS/2026-10-19-19-33-35.gh-issue-159739.aaf575.rst +++ /dev/null @@ -1 +0,0 @@ -Propagate errors that occur while converting macOS system proxy settings. diff --git a/README.rst b/README.rst index d4bea5edcac674c..0afef7499257965 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.16.0 alpha 1+dev -========================================= +This is Python version 3.16.0 alpha 2 +===================================== .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push :alt: CPython build status on GitHub Actions From f66165c64a8e3d24906b06a888a1f54bb172069a Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Tue, 10 Nov 2026 17:42:32 +0000 Subject: [PATCH 078/229] gh-161693: Fix crash in `dict.setdefault` with a `str` subclass key on a split-table dict (#161704) --- Lib/test/test_dict.py | 20 +++++++++++++++++++ ...-11-09-12-00-00.gh-issue-161693.Sp1tKy.rst | 3 +++ Objects/dictobject.c | 2 +- 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-12-00-00.gh-issue-161693.Sp1tKy.rst diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index e8b2aeb1d53125c..6179ad053e7a2a3 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1020,6 +1020,26 @@ def test_splittable_setdefault(self): self.assertEqual(list(a), ['x', 'y', 'z', 'a', 'b']) self.assertEqual(list(b), ['x', 'y', 'z', 'b', 'a']) + @support.cpython_only + def test_splittable_setdefault_str_subclass(self): + class C: + pass + + class S(str): + pass + + a = C() + a.x = 1 + b = C() + b.y = 2 + d = b.__dict__ + + self.assertEqual(d.setdefault(S('x'), 5), 5) + self.assertEqual(d, {'y': 2, 'x': 5}) + self.assertEqual(d.setdefault(S('x'), 6), 5) + self.assertEqual(list(d), ['y', 'x']) + self.assertEqual(a.__dict__, {'x': 1}) + @support.cpython_only def test_splittable_del(self): """split table must be combined when del d[k]""" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-12-00-00.gh-issue-161693.Sp1tKy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-12-00-00.gh-issue-161693.Sp1tKy.rst new file mode 100644 index 000000000000000..ed820f6d29efe9a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-09-12-00-00.gh-issue-161693.Sp1tKy.rst @@ -0,0 +1,3 @@ +Fix a crash in :meth:`dict.setdefault` when called on an instance +dictionary with a :class:`str` subclass key equal to an attribute name +that is not set on that instance. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b8ca74aedc2adfa..ce6f4a61ad3c262 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4979,7 +4979,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } } - if (ix == DKIX_EMPTY) { + if (ix == DKIX_EMPTY || value == NULL) { value = default_value; // See comment to this function in insertdict. From 44ff6c9c3c64c880bfef3288e57fa1a8a724883a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ahmed=20Semih=20=C3=96zmekik?= Date: Tue, 10 Nov 2026 21:04:11 +0300 Subject: [PATCH 079/229] gh-161364: Mark Setup.bootstrap's modules with a `*bootstrap*` line (#161734) An existing Makefile can regenerate itself using an older recipe that invokes `makesetup` without `-b`. This leaves `BOOTSTRAP_MODOBJS` empty and causes `Programs/_freeze_module` to fail to link. Add a `*bootstrap*` marker to `Setup.bootstrap.in` so both old and new recipes identify the bootstrap modules correctly. --- Modules/Setup.bootstrap.in | 4 ++++ Modules/makesetup | 3 ++- Tools/build/check_extension_modules.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 65a1fefe72e92ec..1d1de43fd8f05b4 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -1,5 +1,9 @@ # -*- makefile -*- +# Mark bootstrap modules explicitly so older Makefile recipes work +# without passing -b to makesetup (gh-161364). +*bootstrap* + # --- # Built-in modules required to get a functioning interpreter; # cannot be built as shared! diff --git a/Modules/makesetup b/Modules/makesetup index 932c0952e4781da..4b2d4fad02e10b4 100755 --- a/Modules/makesetup +++ b/Modules/makesetup @@ -19,7 +19,8 @@ set -e # The Setup file following a -b option holds the *bootstrap* modules: # the static modules that the build helper programs (_freeze_module, # _bootstrap_python) are linked with. They are also regular modules of -# the final build. +# the final build. A *bootstrap* line in a Setup file does the same for +# the rest of that file. # Setup files after a -n option are used for their variables, modules # and libraries but not for their .o files. # diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index c619a9a0c1c5a1b..e907442b61ea22f 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -421,7 +421,7 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: state = ModuleState.BUILTIN case ["*disabled*"]: state = ModuleState.DISABLED - case ["*noconfig*"]: + case ["*noconfig*"] | ["*bootstrap*"]: continue case [*items]: if state == ModuleState.DISABLED: From 28f2dd900145683ce98000bae35b0ad866b9e6ee Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 10 Nov 2026 10:54:23 -0800 Subject: [PATCH 080/229] Post 3.16.0a2 --- Include/patchlevel.h | 2 +- README.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index bde6ca17c3ff0c5..5723d1be301af45 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -27,7 +27,7 @@ #define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.16.0a2" +#define PY_VERSION "3.16.0a2+dev" /*--end constants--*/ diff --git a/README.rst b/README.rst index 0afef7499257965..b3058f63c27a120 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.16.0 alpha 2 -===================================== +This is Python version 3.16.0 alpha 2+dev +========================================= .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push :alt: CPython build status on GitHub Actions From e0d1ef25476875317be6e5ee74e1f254eeb49126 Mon Sep 17 00:00:00 2001 From: Pengyu Date: Wed, 11 Nov 2026 03:30:12 +0800 Subject: [PATCH 081/229] gh-159054: Check `basicblock_addop()` failure in `push_cold_blocks_to_end()` (#159057) --- Lib/test/test_compile.py | 21 +++++++++++++++++++ ...-10-09-18-27-12.gh-issue-159054.YUCTvG.rst | 2 ++ Python/flowgraph.c | 5 +++-- 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-18-27-12.gh-issue-159054.YUCTvG.rst diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index d2069a0ac4214d0..b17626252fb537d 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -64,6 +64,27 @@ def test_assemble_init_allocation_failure(self): """) script_helper.assert_python_ok('-c', code) + @support.requires_subprocess() + @support.nomemtest + def test_push_cold_blocks_to_end_allocation_failure(self): + code = textwrap.dedent(""" + from test import support + + source = "async def f():\\n await x\\n" + for n in range(1, 1000): + with support.inject_memory_error_cm(n, n + 1): + try: + compile(source, "", "exec") + except MemoryError: + continue + break + else: + raise AssertionError("compile() never succeeded") + if n == 1: + raise AssertionError("no MemoryError raised") + """) + script_helper.assert_python_ok('-c', code) + def test_other_newlines(self): compile("\r\n", "", "exec") compile("\r", "", "exec") diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-18-27-12.gh-issue-159054.YUCTvG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-18-27-12.gh-issue-159054.YUCTvG.rst new file mode 100644 index 000000000000000..eaa01aa05435a72 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-18-27-12.gh-issue-159054.YUCTvG.rst @@ -0,0 +1,2 @@ +Fixed a crash when an allocation failure occurred while compiling code +containing cold blocks. diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 7b8a3e9cf9b57bd..fc676dc6feb1674 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -3557,8 +3557,9 @@ push_cold_blocks_to_end(cfg_builder *g) { if (!IS_LABEL(b->b_next->b_label)) { b->b_next->b_label.id = next_lbl++; } - basicblock_addop(explicit_jump, JUMP_NO_INTERRUPT, b->b_next->b_label.id, - NO_LOCATION, NULL); + RETURN_IF_ERROR( + basicblock_addop(explicit_jump, JUMP_NO_INTERRUPT, + b->b_next->b_label.id, NO_LOCATION, NULL)); explicit_jump->b_cold = 1; explicit_jump->b_next = b->b_next; explicit_jump->b_predecessors = 1; From 6781fe743ae9df4be510568ad9b676fd20e7b843 Mon Sep 17 00:00:00 2001 From: Thejesh Date: Tue, 10 Nov 2026 12:19:05 -0800 Subject: [PATCH 082/229] gh-161007: Add a glossary entry for "virtual subclass" (#161182) --- Doc/glossary.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index a31bd53f06294db..f0934dc2e138771 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -29,9 +29,10 @@ Glossary Abstract base classes complement :term:`duck-typing` by providing a way to define interfaces when other techniques like :func:`hasattr` would be clumsy or subtly wrong (for example with - :ref:`magic methods `). ABCs introduce virtual - subclasses, which are classes that don't inherit from a class but are - still recognized by :func:`isinstance` and :func:`issubclass`; see the + :ref:`magic methods `). ABCs introduce + :term:`virtual subclasses `, which are classes that + don't inherit from a class but are still recognized by + :func:`isinstance` and :func:`issubclass`; see the :mod:`abc` module documentation. Python comes with many built-in ABCs for data structures (in the :mod:`collections.abc` module), numbers (in the :mod:`numbers` module), streams (in the :mod:`io` module), import finders @@ -1727,6 +1728,12 @@ Glossary A computer defined entirely in software. Python's virtual machine executes the :term:`bytecode` emitted by the bytecode compiler. + virtual subclass + A class that :func:`issubclass` and :func:`isinstance` treat as a + subclass of an :term:`abstract base class` even though it does not + inherit from it. A class is made a virtual subclass by registering + it with the ABC's :meth:`~abc.ABCMeta.register` method. + walrus operator A light-hearted way to refer to the :ref:`assignment expression ` operator ``:=`` because it looks a bit like a From fe1c75e2efa7220141b454f356e3b1879bddafa8 Mon Sep 17 00:00:00 2001 From: scrieve <86650480+scrieve@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:32:35 +0000 Subject: [PATCH 083/229] gh-160501: Test `adaptive_find` in `fastsearch.h` for all string kinds (#160502) --- Lib/test/test_str.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index b4b4572256c8f23..e0632db94b9a847 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -267,6 +267,30 @@ class MyStr(str): pass self.checkequal(3, MyStr('aaa'), 'count', 'a') + def test_find_adaptive_widths(self): + # The adaptive path in Objects/stringlib/fastsearch.h needs a haystack + # of at least 2500 characters and a needle of at least a third of it, + # and it is compiled once per character width. Exercise all four. + cases = [ + ('ascii', 'a', 'b', 'c'), + ('latin1', '\xe9', '\xea', '\xeb'), + ('ucs2', '\u0100', '\u0101', '\u0102'), + ('ucs4', '\U0001d11e', '\U0001d11f', '\U0001d120'), + ] + for kind, c, d, x in cases: + # A needle of 2000 characters, none of which is x. + needle = c * 1000 + d * 1000 + with self.subTest(kind=kind): + # 4000 characters, mostly x: the skip branches jump by m. + haystack = c * 1000 + x * 3000 + self.assertEqual(haystack.find(needle), -1) + self.assertEqual(haystack.count(needle), 0) + # A match at 3000 in a haystack of 5000, so count() reaches + # the match branch inside adaptive_find rather than two-way. + haystack = c * 1000 + x * 2000 + needle + self.assertEqual(haystack.find(needle), 3000) + self.assertEqual(haystack.count(needle), 1) + def test_find(self): string_tests.StringLikeTest.test_find(self) # test implementation details of the memchr fast path From 226813c0f7834b55be51e912a3d8555e652d9026 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 10 Nov 2026 22:40:40 +0100 Subject: [PATCH 084/229] gh-161724, PCbuild: Build also xxlimited in Debug configuration (#161725) Previously, xxlimited extension modules were only built in the Release configuration. --- PCbuild/pcbuild.proj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 53aec0276beaee2..95e135705dac2ff 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -78,9 +78,9 @@ - - - + + + false From acab53212d89b298ba0dd299e8e55c62bf71db80 Mon Sep 17 00:00:00 2001 From: Abhishek Tiwari Date: Wed, 11 Nov 2026 04:28:40 +0530 Subject: [PATCH 085/229] gh-151763: Fix crash in PyList_New() on allocation failure under low memory (#154019) Co-authored-by: Sergey Miryanov --- .../2026-07-18-22-56-27.gh-issue-152125.sEPvJf.rst | 3 +++ Objects/listobject.c | 11 ++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-18-22-56-27.gh-issue-152125.sEPvJf.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-18-22-56-27.gh-issue-152125.sEPvJf.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-18-22-56-27.gh-issue-152125.sEPvJf.rst new file mode 100644 index 000000000000000..ae66f69f206217f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-18-22-56-27.gh-issue-152125.sEPvJf.rst @@ -0,0 +1,3 @@ +Fix a crash in the :term:`free-threaded build` where :c:func:`PyList_New` could leave +a newly allocated list's item buffer pointer uninitialized when the backing +array allocation failed under low memory, leading to an invalid free. diff --git a/Objects/listobject.c b/Objects/listobject.c index 4384a850780b7fd..c5fe704df0e23de 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -273,10 +273,11 @@ PyList_New(Py_ssize_t size) return NULL; } } - if (size <= 0) { - op->ob_item = NULL; - } - else { + op->ob_item = NULL; + op->allocated = 0; + Py_SET_SIZE(op, size); + + if (size > 0) { #ifdef Py_GIL_DISABLED _PyListArray *array = list_allocate_array(size); if (array == NULL) { @@ -293,7 +294,7 @@ PyList_New(Py_ssize_t size) return PyErr_NoMemory(); } } - Py_SET_SIZE(op, size); + op->allocated = size; _PyObject_GC_TRACK(op); return (PyObject *) op; From 3cf810dc084b64b4d6a78bde02ed19d3f4015ab3 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Tue, 10 Nov 2026 21:26:26 -0800 Subject: [PATCH 086/229] gh-52330 Document how a proxy auto-config (PAC) file can be used with urllib.request.ProxyHandler (#161729) gh-52330 Document how a proxy auto-config (PAC) file can be used with urllib.request.ProxyHandler We will point to the pypi package pypac which supports PAC discovery and can be extended as a custom ProxyHandler to be used with urllib.request. The documentation will point to the pypi package and share high-level example of using this. --- Doc/library/urllib.request.rst | 50 ++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 9a75f4986e1a3a9..8234be98a2d6596 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -378,6 +378,11 @@ The following classes are provided: ``HTTP_PROXY`` will be ignored if a variable ``REQUEST_METHOD`` is set; see the documentation on :func:`~urllib.request.getproxies`. + Because *proxies* holds at most one proxy for each protocol, it cannot + use a proxy auto-config (PAC) file, which selects a proxy for each + URL. See :ref:`this example ` for one way to use a + PAC file. + .. class:: HTTPPasswordMgr() @@ -1382,6 +1387,51 @@ programmatically supplied proxy URLs, and adds proxy authorization support with with opener.open('http://www.example.com/login.html') as f: print(f.read().decode('utf-8')) +.. _urllib-request-pac: + +Proxy auto-config (PAC) files select a proxy for each URL, so they cannot be +described by the *proxies* mapping that :class:`ProxyHandler` expects. The +:pypi:`pypac` package finds and evaluates PAC files; its ``ProxyResolver`` +can be used to write a handler that consults the PAC file once per request:: + + import urllib.error + import urllib.request + from pypac import get_pac + from pypac.resolver import ProxyResolver + + class PACProxyHandler(urllib.request.ProxyHandler): + + def __init__(self, resolver): + self.resolver = resolver + # An empty mapping suppresses the _open methods that + # ProxyHandler would make from a static proxies mapping. + super().__init__({}) + + def _pac_open(self, req, protocol): + proxy = self.resolver.get_proxy(req.full_url) + if proxy is None: + raise urllib.error.URLError('no proxy available') + if proxy == 'DIRECT': + return None # continue with the remaining handlers + if proxy.startswith('socks'): + raise urllib.error.URLError('SOCKS proxies are not supported') + return self.proxy_open(req, proxy, protocol) + + def http_open(self, req): + return self._pac_open(req, 'http') + + def https_open(self, req): + return self._pac_open(req, 'https') + + opener = urllib.request.build_opener(PACProxyHandler(ProxyResolver(get_pac()))) + with opener.open('http://www.example.com/') as f: + print(f.read().decode('utf-8')) + +Note that a PAC file may also select a SOCKS proxy, which :mod:`urllib.request` +cannot use, and that the :envvar:`no_proxy` environment variable and the +operating system's proxy bypass list still apply to requests the PAC file +assigned to a proxy. + Adding HTTP headers: Use the *headers* argument to the :class:`Request` constructor, or:: From 4b79d90e935398442d65c59243f8015f062da0f8 Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Wed, 11 Nov 2026 06:31:22 +0000 Subject: [PATCH 087/229] gh-155003: Don't try to close mmap in SharedMemory.__del__ (GH-155070) * Don't try to close mmap in SharedMemory.__del__ * Add a test for SharedMemory finalization with a live buffer, expand NEWS --------- Co-authored-by: Gregory P. Smith --- Lib/multiprocessing/shared_memory.py | 11 +++--- Lib/test/_test_multiprocessing.py | 36 +++++++++++++++++++ ...-08-02-08-43-20.gh-issue-155003.VVy9Z_.rst | 5 +++ 3 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-02-08-43-20.gh-issue-155003.VVy9Z_.rst diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index b5396451d0e64a5..de74a0cbbba8533 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -186,7 +186,7 @@ def __init__(self, name=None, create=False, size=0, *, track=True): def __del__(self): try: - self.close() + self._closefd() except OSError: pass @@ -222,6 +222,11 @@ def size(self): "Size in bytes." return self._size + def _closefd(self): + if _USE_POSIX and self._fd >= 0: + os.close(self._fd) + self._fd = -1 + def close(self): """Closes access to the shared memory from this instance but does not destroy the shared memory block.""" @@ -231,9 +236,7 @@ def close(self): if self._mmap is not None: self._mmap.close() self._mmap = None - if _USE_POSIX and self._fd >= 0: - os.close(self._fd) - self._fd = -1 + self._closefd() def unlink(self): """Requests that the underlying shared memory block be destroyed. diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 44e66707607ab57..540189147cb4d7b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4997,6 +4997,42 @@ def test_shared_memory_pickle_unpickle_dead_object(self): with self.assertRaises(FileNotFoundError): pickle.loads(pickled_sms) + def test_shared_memory_del_with_live_buffer(self): + # gh-155003: Garbage collecting a SharedMemory must not release a + # buffer that is still in use, and must still close the file + # descriptor. + cases = { + 'buf': lambda sms: sms.buf, + 'cast': lambda sms: sms.buf.cast('i'), + } + if c_int is not None: + # Holds a buffer on the memoryview object itself (as a NumPy + # array would), so releasing the memoryview fails, not just + # closing the mmap. + cases['ctypes'] = lambda sms: (c_int * 4).from_buffer(sms.buf) + for case, make_view in cases.items(): + with self.subTest(case=case): + sms = shared_memory.SharedMemory(create=True, size=16) + name = sms.name + fd = sms._fd + sms.buf[0] = 42 + kept = make_view(sms) + with support.catch_unraisable_exception() as cm: + del sms + support.gc_collect() + self.assertIsNone(cm.unraisable) + with memoryview(kept) as mv, mv.cast('B') as view: + self.assertEqual(view[0], 42) + view[1] = 43 + if shared_memory._USE_POSIX: + with self.assertRaises(OSError): + os.fstat(fd) + other = shared_memory.SharedMemory(name) + self.assertEqual(bytes(other.buf[:2]), b'\x2a\x2b') + other.unlink() + other.close() + del kept + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_shared_memory_across_processes(self): # bpo-40135: don't define shared memory block's name in case of diff --git a/Misc/NEWS.d/next/Library/2026-08-02-08-43-20.gh-issue-155003.VVy9Z_.rst b/Misc/NEWS.d/next/Library/2026-08-02-08-43-20.gh-issue-155003.VVy9Z_.rst new file mode 100644 index 000000000000000..c41ce1879488b78 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-02-08-43-20.gh-issue-155003.VVy9Z_.rst @@ -0,0 +1,5 @@ +Fix :class:`multiprocessing.shared_memory.SharedMemory` leaking its file +descriptor and printing an ignored :exc:`BufferError` when garbage collected +while its :attr:`~multiprocessing.shared_memory.SharedMemory.buf` was still in +use, for example by a NumPy array. :attr:`!buf` now also stays usable after +the :class:`!SharedMemory` object is garbage collected. From de9a4fb5586cc9e60e8b75a3f69033c4a2d6c61b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 11 Nov 2026 11:12:36 +0100 Subject: [PATCH 088/229] gh-154643: Only use _Py_STDC_VERSION when defined (GH-154650) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gh-154643: Only use _Py_STDC_VERSION when defined This fixes: $ g++ -I/usr/include/python3.15 -c -Werror=undef test.cpp In file included from /usr/include/python3.15/Python.h:68, from test.cpp:1: /usr/include/python3.15/pyport.h:494:9: error: ‘_Py_STDC_VERSION’ is not defined, evaluates to ‘0’ [-Werror=undef] 494 | # elif _Py_STDC_VERSION >= 201112L && !defined(__STDC_NO_THREADS__) | ^~~~~~~~~~~~~~~~ cc1plus: some warnings being treated as errors For any test.cpp that has #include . For locating the place for the added test, I used LLM. The main issue I could reproduce was in Include/pyport.h. Include/cpython/pyatomic.h was updated for completeness. Include/internal/mimalloc/mimalloc/atomic.h also has this for __STDC_VERSION__ but was left untouched as it is code from a bundled project and is likely not exposed publicly. Assisted-By: Claude Opus 4.6 * Also add test for C * Add -Werror=undef to PUBLIC_CFLAGS & PEDANTIC_CXXFLAGS in setup.py * Drop tests, we use PUBLIC_CFLAGS & PEDANTIC_CXXFLAGS instead --- Include/cpython/pyatomic.h | 2 +- Include/pyport.h | 2 +- Lib/test/test_cext/setup.py | 6 ++++++ .../C_API/2026-07-24-22-30-40.gh-issue-154643.z-ZutK.rst | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-07-24-22-30-40.gh-issue-154643.z-ZutK.rst diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 71a76bdc82ce475..0a11f7781f07e00 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -585,7 +585,7 @@ static inline void _Py_atomic_fence_release(void); # define Py_ATOMIC_GCC_H # include "pyatomic_gcc.h" # undef Py_ATOMIC_GCC_H -#elif _Py_STDC_VERSION >= 201112L && !defined(__STDC_NO_ATOMICS__) +#elif defined(_Py_STDC_VERSION) && _Py_STDC_VERSION >= 201112L && !defined(__STDC_NO_ATOMICS__) # define Py_ATOMIC_STD_H # include "pyatomic_std.h" # undef Py_ATOMIC_STD_H diff --git a/Include/pyport.h b/Include/pyport.h index 718a9b50bae1039..de1240be76b3dad 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -490,7 +490,7 @@ extern "C" { #ifdef WITH_THREAD # ifdef thread_local # define _Py_thread_local thread_local -# elif _Py_STDC_VERSION >= 201112L && !defined(__STDC_NO_THREADS__) +# elif defined(_Py_STDC_VERSION) && _Py_STDC_VERSION >= 201112L && !defined(__STDC_NO_THREADS__) # define _Py_thread_local _Thread_local # elif defined(_MSC_VER) /* AKA NT_THREADS */ # define _Py_thread_local __declspec(thread) diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 10a3887931932ac..3b613ee729f52c4 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -42,6 +42,9 @@ # Ask for strict(er) compliance with the standard '-pedantic-errors', + + # gh-154643: Headers should not use undefined macros in #if + '-Werror=undef', ] if not support.Py_GIL_DISABLED: PUBLIC_CFLAGS.append( @@ -94,6 +97,9 @@ # We also use `long long`, a C++11 feature we can enable individually. '-Wno-long-long', + + # gh-154643: Headers should not use undefined macros in #if + '-Werror=undef', ] INTERNAL_CXXFLAGS = [ # Do not complain about unused parameteres in the internal C API diff --git a/Misc/NEWS.d/next/C_API/2026-07-24-22-30-40.gh-issue-154643.z-ZutK.rst b/Misc/NEWS.d/next/C_API/2026-07-24-22-30-40.gh-issue-154643.z-ZutK.rst new file mode 100644 index 000000000000000..17c2a58555e7ae9 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-07-24-22-30-40.gh-issue-154643.z-ZutK.rst @@ -0,0 +1,2 @@ +In ``pyport.h`` and ``pyatomic.h`` don't use ``_Py_STDC_VERSION`` when it is +not defined, fixing build errors for C or C++ code built with ``-Werror=undef``. From 0ada998ebbf392238719358432cd723a622fcd5e Mon Sep 17 00:00:00 2001 From: Timofei Ivankov <128279579+deadlovelll@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:37:51 +0300 Subject: [PATCH 089/229] gh-157299: Fix missing awaited-by edge in eager tasks (#157300) --- Doc/library/asyncio-task.rst | 4 + Lib/asyncio/tasks.py | 3 + .../test_asyncio/test_eager_task_factory.py | 75 +++++++++++++++++++ Lib/test/test_asyncio/test_graph.py | 17 +++++ ...-09-11-12-43-04.gh-issue-157299.HQ3I4F.rst | 4 + Modules/_asynciomodule.c | 25 ++++++- 6 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-11-12-43-04.gh-issue-157299.HQ3I4F.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 6df87d4d71a3657..bc79927b9856270 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -666,6 +666,10 @@ Eager task factory changes to existing applications. For example, the application's task execution order is likely to change. + During the eager step, call graph introspection reports the creating + task as an awaiter, even for a plain :func:`create_task` call. + This relationship is removed when the eager step finishes. + .. versionadded:: 3.12 .. function:: create_eager_task_factory(custom_task_constructor) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 70da1e25ebaf081..86b64880b37c9bd 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -257,9 +257,12 @@ def __eager_start(self): prev_task = _py_swap_current_task(self._loop, self) try: _py_register_eager_task(self) + # gh-157299: the eager step must record the awaited_by edge + futures.future_add_to_awaited_by(self, prev_task) try: self._context.run(self.__step_run_and_handle_result, None) finally: + futures.future_discard_from_awaited_by(self, prev_task) _py_unregister_eager_task(self) finally: try: diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 73496de81a58efe..05646ba373abd86 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -69,6 +69,20 @@ async def run(): self.assertEqual(self.run_coro(run()), 'my message') + def test_awaited_by_during_eager_step(self): + async def run(): + parent = asyncio.current_task() + + async def coro(): + self.assertEqual(asyncio.current_task()._asyncio_awaited_by, + frozenset({parent})) + + t = self.loop.create_task(coro()) + self.assertFalse(t._asyncio_awaited_by) + await t + + self.run_coro(run()) + def test_eager_completion(self): async def coro(): @@ -292,11 +306,29 @@ def setUp(self): self._current_task = asyncio.current_task asyncio.current_task = asyncio.tasks.current_task = asyncio.tasks._py_current_task asyncio.all_tasks = asyncio.tasks.all_tasks = asyncio.tasks._py_all_tasks + + futures = asyncio.futures + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._py_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._py_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by return super().setUp() def tearDown(self): asyncio.current_task = asyncio.tasks.current_task = self._current_task asyncio.all_tasks = asyncio.tasks.all_tasks = self._all_tasks + + futures = asyncio.futures + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by return super().tearDown() @@ -318,6 +350,49 @@ def tearDown(self): asyncio.all_tasks = asyncio.tasks.all_tasks = self._all_tasks return super().tearDown() + def test_eager_step_with_non_native_parent(self): + async def child(): + self.assertFalse(asyncio.current_task()._asyncio_awaited_by) + + async def run(): + parent = tasks._c_swap_current_task(self.loop, object()) + try: + task = self.loop.create_task(child()) + finally: + tasks._c_swap_current_task(self.loop, parent) + await task + + self.run_coro(run()) + + def test_eager_step_preserves_exception(self): + # gh-157299: check that the exception is saved and cleared before + # discarding the awaited-by edge, and restored afterwards. + # A default task can still pass with the wrong order since its hash + # is in C. Overriding __hash__ makes cleanup call back into Python. + # If the exception is still pending, this raises SystemError. + class ParentTask(self.Task): + def __hash__(self): + return super().__hash__() + + child_tasks = [] + exc = SystemExit("eager step") + + async def child(): + task = asyncio.current_task() + child_tasks.append(task) + asyncio.future_add_to_awaited_by(task, self.loop.create_future()) + raise exc + + async def parent(): + with self.assertRaises(SystemExit) as cm: + self.loop.create_task(child()) + task, = child_tasks + self.assertIs(cm.exception, exc) + self.assertIs(task.exception(), exc) + self.assertNotIn(asyncio.current_task(), task._asyncio_awaited_by) + + self.loop.run_until_complete(ParentTask(parent(), loop=self.loop)) + def test_issue105987(self): code = """if 1: from _asyncio import _swap_current_task, _set_running_loop diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 8af42a851707a56..1f1c76e4aef69f8 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -969,6 +969,23 @@ async def main(): await main() self.assertEqual(stack[:3], ['gen', 'middle', 'main']) + def set_eager_task_factory(self): + loop = asyncio.get_running_loop() + loop.set_task_factory(asyncio.create_eager_task_factory(asyncio.Task)) + self.addCleanup(loop.set_task_factory, None) + + async def test_stack_eager_task(self): + self.set_eager_task_factory() + + async def child(): + nonlocal stack + stack = capture_test_stack() + + stack = None + await asyncio.gather(child()) + + self.assertEqual(stack[0][2], [['T', ['a test_stack_eager_task'], []]]) + @unittest.skipIf( not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), diff --git a/Misc/NEWS.d/next/Library/2026-09-11-12-43-04.gh-issue-157299.HQ3I4F.rst b/Misc/NEWS.d/next/Library/2026-09-11-12-43-04.gh-issue-157299.HQ3I4F.rst new file mode 100644 index 000000000000000..fe9483325829c44 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-11-12-43-04.gh-issue-157299.HQ3I4F.rst @@ -0,0 +1,4 @@ +Fix eagerly started :class:`asyncio.Task` objects missing their awaited-by +edge during the first step. During that step, even a plain +:func:`asyncio.create_task` call reports the creating task as an awaiter. +Patch by Timofey Ivankov. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index a8293f39a7f8c7b..d7415aef91532a7 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3491,17 +3491,36 @@ task_eager_start(_PyThreadStateImpl *ts, asyncio_state *state, TaskObj *task) int retval = 0; + // gh-157299: the eager step must record the awaited_by edge + int eager_edge = TaskOrFuture_Check(state, prevtask); PyObject *stepres; Py_BEGIN_CRITICAL_SECTION(task); - stepres = task_step_impl(state, task, NULL); - Py_END_CRITICAL_SECTION(); + if (eager_edge && + future_awaited_by_add(state, (FutureObj *)task, prevtask) < 0) + { + eager_edge = 0; + task->task_log_destroy_pending = 0; + unregister_task(task); + stepres = NULL; + } + else { + stepres = task_step_impl(state, task, NULL); + } + + PyObject *exc = stepres == NULL ? PyErr_GetRaisedException() : NULL; + if (eager_edge) { + int res = future_awaited_by_discard(state, (FutureObj *)task, prevtask); + if (res) { + retval = -1; + } + } if (stepres == NULL) { - PyObject *exc = PyErr_GetRaisedException(); _PyErr_ChainExceptions1(exc); retval = -1; } else { Py_DECREF(stepres); } + Py_END_CRITICAL_SECTION(); PyObject *curtask = swap_current_task(ts, task->task_loop, prevtask); Py_DECREF(prevtask); From f34992bfab227932675d491a32bb44835aeee089 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 11 Nov 2026 11:42:07 -0600 Subject: [PATCH 090/229] Remove duplicated '_zstd' section from PCbuild/readme.txt (GH-161819) --- PCbuild/readme.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index d54c0ed3e3c6333..32ea8921c667c32 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -274,10 +274,6 @@ zlib-ng Sources for zlib-ng are imported unmodified into our source repository at https://github.com/python/cpython-source-deps. -_zstd - Python wrapper for version 1.5.7 of the Zstandard compression library - Homepage: - https://facebook.github.io/zstd/ Getting External Sources From aecef70048dc56c7ac270c7d9e3c112347ea347e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Delfino?= Date: Wed, 11 Nov 2026 15:54:18 -0300 Subject: [PATCH 091/229] Fix test assertions that never compared against the expected value (#161774) Co-authored-by: Stan Ulbrych --- Lib/test/test_capi/test_function.py | 2 +- Lib/test/test_ctypes/test_memfunctions.py | 2 +- Lib/test/test_ctypes/test_returnfuncptrs.py | 6 +++--- Lib/test/test_enum.py | 6 +++--- Lib/test/test_frame.py | 6 +++--- Lib/test/test_minidom.py | 2 +- Lib/test/test_optparse.py | 2 +- Lib/test/test_pathlib/test_pathlib.py | 4 ++-- Lib/test/test_ssl.py | 4 +--- Lib/test/test_stringprep.py | 8 ++++---- Lib/test/test_ttk_textonly.py | 2 +- Lib/test/test_zipfile/test_core.py | 4 ++-- 12 files changed, 23 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_capi/test_function.py b/Lib/test/test_capi/test_function.py index 9a07e3106f3a22a..5024c58679bb2b9 100644 --- a/Lib/test/test_capi/test_function.py +++ b/Lib/test/test_capi/test_function.py @@ -235,7 +235,7 @@ def inner(arg5, arg6): self.assertEqual(len(closure), len(func.__code__.co_freevars)) for cell in closure: self.assertIsInstance(cell, CellType) - self.assertTrue(closure[0].cell_contents, 5) + self.assertEqual(closure[0].cell_contents, 5) func = with_two_levels(1, 2)(3, 4) closure = _testcapi.function_get_closure(func) diff --git a/Lib/test/test_ctypes/test_memfunctions.py b/Lib/test/test_ctypes/test_memfunctions.py index e3cb5db775ef622..4edf819ca98ebaa 100644 --- a/Lib/test/test_ctypes/test_memfunctions.py +++ b/Lib/test/test_ctypes/test_memfunctions.py @@ -60,7 +60,7 @@ def test_cast(self): @support.refcount_test def test_string_at(self): s = string_at(b"foo bar") - self.assertTrue(s, "foo bar") + self.assertEqual(s, b"foo bar") self.assertEqual(string_at(b"foo bar", 7), b"foo bar") self.assertEqual(string_at(b"foo bar", 3), b"foo") diff --git a/Lib/test/test_ctypes/test_returnfuncptrs.py b/Lib/test/test_ctypes/test_returnfuncptrs.py index 337801b226ab06b..8f0a61be4c5a80b 100644 --- a/Lib/test/test_ctypes/test_returnfuncptrs.py +++ b/Lib/test/test_ctypes/test_returnfuncptrs.py @@ -28,7 +28,7 @@ def test_without_prototype(self): # _CFuncPtr instances are now callable with an integer argument # which denotes a function address: strchr = CFUNCTYPE(c_char_p, c_char_p, c_char)(addr) - self.assertTrue(strchr(b"abcdef", b"b"), "bcdef") + self.assertEqual(strchr(b"abcdef", b"b"), b"bcdef") self.assertEqual(strchr(b"abcdef", b"x"), None) self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0) self.assertRaises(TypeError, strchr, b"abcdef") @@ -38,7 +38,7 @@ def test_from_dll(self): # _CFuncPtr instances are now callable with a tuple argument # which denotes a function name and a dll: strchr = CFUNCTYPE(c_char_p, c_char_p, c_char)(("my_strchr", dll)) - self.assertTrue(strchr(b"abcdef", b"b"), "bcdef") + self.assertEqual(strchr(b"abcdef", b"b"), b"bcdef") self.assertEqual(strchr(b"abcdef", b"x"), None) self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0) self.assertRaises(TypeError, strchr, b"abcdef") @@ -57,7 +57,7 @@ def __getitem__(self, key): # which denotes a function name and a dll: strchr = CFUNCTYPE(c_char_p, c_char_p, c_char)( BadSequence(("my_strchr", CDLL(_ctypes_test.__file__)))) - self.assertTrue(strchr(b"abcdef", b"b"), "bcdef") + self.assertEqual(strchr(b"abcdef", b"b"), b"bcdef") self.assertEqual(strchr(b"abcdef", b"x"), None) self.assertRaises(ArgumentError, strchr, b"abcdef", 3.0) self.assertRaises(TypeError, strchr, b"abcdef") diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 447f847f33da938..4484749eac449e3 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -1032,7 +1032,7 @@ class OpenXYZ(self.enum_type): self.assertIs(~(Y|Z), OpenXYZ(28)) self.assertIs(~(X|Y|Z), OpenXYZ(24)) self.assertIs(~XYZ_MASK, OpenXYZ(0)) - self.assertTrue(~OpenXYZ(0), XYZ_MASK) + self.assertIs(~OpenXYZ(0), XYZ_MASK) else: self.assertIs(~X, Y|Z) self.assertIs(~Y, X|Z) @@ -1045,7 +1045,7 @@ class OpenXYZ(self.enum_type): self.assertIs(~(Y|Z), X) self.assertIs(~(X|Y|Z), OpenXYZ(0)) self.assertIs(~XYZ_MASK, OpenXYZ(0)) - self.assertTrue(~OpenXYZ(0), (X|Y|Z)) + self.assertIs(~OpenXYZ(0), (X|Y|Z)) def test_assigned_negative_value(self): class X(self.enum_type): @@ -1895,7 +1895,7 @@ class Season(Enum): @reraise_if_not_enum(Name) def test_subclassing(self): self.assertEqual(Name.BDFL, 'Guido van Rossum') - self.assertTrue(Name.BDFL, Name('Guido van Rossum')) + self.assertIs(Name.BDFL, Name('Guido van Rossum')) self.assertIs(Name.BDFL, getattr(Name, 'BDFL')) test_pickle_dump_load(self.assertIs, Name.BDFL) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index a0a11966ebf3c17..2b41ef33a30b446 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -867,13 +867,13 @@ def PyFrame_GetLocals(frame: ctypes.py_object) -> ctypes.py_object: pass frame_locals = PyEval_GetFrameLocals() - self.assertTrue(type(frame_locals), dict) + self.assertIs(type(frame_locals), dict) self.assertEqual(frame_locals['x'], 1) frame_locals['x'] = 2 self.assertEqual(x, 1) frame_globals = PyEval_GetFrameGlobals() - self.assertTrue(type(frame_globals), dict) + self.assertIs(type(frame_globals), dict) self.assertIs(frame_globals, globals()) frame_builtins = PyEval_GetFrameBuiltins() @@ -881,7 +881,7 @@ def PyFrame_GetLocals(frame: ctypes.py_object) -> ctypes.py_object: frame = sys._getframe() f_locals = PyFrame_GetLocals(frame) - self.assertTrue(f_locals['x'], 1) + self.assertEqual(f_locals['x'], 1) f_locals['x'] = 2 self.assertEqual(x, 2) diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 274c6acab4a6512..0c418587afe8ed2 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -1274,7 +1274,7 @@ def testCloneElementDeep(self): self.assertEqual(len(clone.childNodes), 1) self.assertEqual(clone.childNodes.length, 1) self.assertIsNone(clone.parentNode) - self.assertTrue(clone.toxml(), '') + self.assertEqual(clone.toxml(), '') dom.unlink() def testCloneDocumentShallow(self): diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 10ecdd3bccc23f7..f393e67ecbff594 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -1018,7 +1018,7 @@ def test_group_manipulate(self): description="Some more options") group.set_title("Bacon") group.add_option("--bacon", type="int") - self.assertTrue(self.parser.get_option_group("--bacon"), group) + self.assertIs(self.parser.get_option_group("--bacon"), group) # -- Test extending and parser.parse_args() ---------------------------- diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index aff66c8efedbbc0..788547c49e43e33 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1697,8 +1697,8 @@ def test_move_dir(self): self.assertEqual(target.joinpath('dirD', 'fileD').read_text(), "this is file D\n") self.assertTrue(target.joinpath('fileC').is_file()) - self.assertTrue(target.joinpath('fileC').read_text(), - "this is file C\n") + self.assertEqual(target.joinpath('fileC').read_text(), + "this is file C\n") @patch_replace def test_move_dir_other_fs(self): diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index b0e3cbaa548e8d5..19a9f98c316ab3a 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4217,8 +4217,6 @@ def test_ecc_cert(self): s.connect((HOST, server.port)) cert = s.getpeercert() self.assertTrue(cert, "Can't get peer certificate.") - cipher = s.cipher()[0].split('-') - self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) @unittest.skipUnless(IS_OPENSSL_3_0_0, "test requires RFC 5280 check added in OpenSSL 3.0+") @@ -4276,7 +4274,7 @@ def test_dual_rsa_ecc(self): cert = s.getpeercert() self.assertTrue(cert, "Can't get peer certificate.") cipher = s.cipher()[0].split('-') - self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) + self.assertEqual(cipher[:2], ['ECDHE', 'ECDSA']) def test_check_hostname_idn(self, warnings_filters=True): if support.verbose: diff --git a/Lib/test/test_stringprep.py b/Lib/test/test_stringprep.py index d4b4a13d0de2645..a4ddc8b33e2570b 100644 --- a/Lib/test/test_stringprep.py +++ b/Lib/test/test_stringprep.py @@ -13,11 +13,11 @@ def test(self): self.assertTrue(in_table_b1("\u00ad")) self.assertFalse(in_table_b1("\u00ae")) - self.assertTrue(map_table_b2("\u0041"), "\u0061") - self.assertTrue(map_table_b2("\u0061"), "\u0061") + self.assertEqual(map_table_b2("\u0041"), "\u0061") + self.assertEqual(map_table_b2("\u0061"), "\u0061") - self.assertTrue(map_table_b3("\u0041"), "\u0061") - self.assertTrue(map_table_b3("\u0061"), "\u0061") + self.assertEqual(map_table_b3("\u0041"), "\u0061") + self.assertEqual(map_table_b3("\u0061"), "\u0061") self.assertTrue(in_table_c11("\u0020")) self.assertFalse(in_table_c11("\u0021")) diff --git a/Lib/test/test_ttk_textonly.py b/Lib/test/test_ttk_textonly.py index e6525c4d6c69822..7fecbb6305442be 100644 --- a/Lib/test/test_ttk_textonly.py +++ b/Lib/test/test_ttk_textonly.py @@ -169,7 +169,7 @@ def test_format_mapdict(self): def test_format_elemcreate(self): - self.assertTrue(ttk._format_elemcreate(None), (None, ())) + self.assertEqual(ttk._format_elemcreate(None), ((),)) ## Testing type = image # image type expects at least an image name, so this should raise diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 2f8fef2a006c381..5e098720df77010 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -5607,14 +5607,14 @@ def test_write_dir(self): self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) zipf.write(dirpath, "y") zinfo = zipf.filelist[1] - self.assertTrue(zinfo.filename, "y/") + self.assertEqual(zinfo.filename, "y/") self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) with zipfile.ZipFile(TESTFN, "r") as zipf: zinfo = zipf.filelist[0] self.assertEndsWith(zinfo.filename, "/x/") self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) zinfo = zipf.filelist[1] - self.assertTrue(zinfo.filename, "y/") + self.assertEqual(zinfo.filename, "y/") self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) target = os.path.join(TESTFN2, "target") os.mkdir(target) From d100c642752d88e202081156525c906c6d20470d Mon Sep 17 00:00:00 2001 From: sidlovescoding Date: Wed, 11 Nov 2026 11:17:02 -0800 Subject: [PATCH 092/229] gh-141998: Do not syntax highlight text read by `input()` in the REPL (#161790) Co-authored-by: Stan Ulbrych --- Lib/_pyrepl/readline.py | 2 +- Lib/test/test_pyrepl/test_pyrepl.py | 10 ++++++++++ .../2026-11-08-19-29-54.gh-issue-141998.8vum0w.rst | 2 ++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-08-19-29-54.gh-issue-141998.8vum0w.rst diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 4232a868a34e9ad..fbc0f5926ea16b4 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -377,7 +377,7 @@ def input(self, prompt: object = "") -> str: prompt_str = str(prompt) reader.ps1 = prompt_str sys.audit("builtins.input", prompt_str) - with reader.suspend_history_writes(): + with reader.suspend_history_writes(), reader.suspend_colorization(): result = reader.readline(startup_hook=self.startup_hook) sys.audit("builtins.input/result", result) return result diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 1828fd735669d54..c11c759f559affa 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -732,6 +732,16 @@ def test_input_does_not_modify_history(self): self.assertEqual(wrapper.input(), "new input") self.assertEqual(reader.history, ["previous input"]) + def test_input_is_not_syntax_highlighted(self): + reader = self.prepare_reader(code_to_events("def f(): pass\n")) + reader.can_colorize = True + reader.gen_colors = Mock(return_value=()) + wrapper = _ReadlineWrapper(f_in=None, f_out=None, reader=reader) + + self.assertEqual(wrapper.input(), "def f(): pass") + reader.gen_colors.assert_not_called() + self.assertTrue(reader.can_colorize) + def test_input_error_does_not_modify_history(self): for text, exception in ( ("partial input\x03", KeyboardInterrupt), diff --git a/Misc/NEWS.d/next/Library/2026-11-08-19-29-54.gh-issue-141998.8vum0w.rst b/Misc/NEWS.d/next/Library/2026-11-08-19-29-54.gh-issue-141998.8vum0w.rst new file mode 100644 index 000000000000000..633c27d55175a3d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-08-19-29-54.gh-issue-141998.8vum0w.rst @@ -0,0 +1,2 @@ +Do not syntax highlight text typed in response to :func:`input` in the +:term:`REPL`. From a8720240f12eb60f760e5d32735286720be00bdf Mon Sep 17 00:00:00 2001 From: David Ellis Date: Wed, 11 Nov 2026 19:17:51 +0000 Subject: [PATCH 093/229] gh-158797: Update docs for `test.support.import_helper` (#158875) Co-authored-by: Stan Ulbrych --- Doc/library/test.rst | 196 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 157 insertions(+), 39 deletions(-) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 20810a537d4fe19..e6786a441c66f83 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1706,22 +1706,111 @@ The :mod:`!test.support.import_helper` module provides support for import tests. .. versionadded:: 3.10 +Creating module objects +----------------------- -.. function:: forget(module_name) +.. function:: create_module(name, loader=None, *, ispkg=False) - Remove the module named *module_name* from ``sys.modules`` and delete any + Create a new, empty :term:`module` or :term:`package` and return it. + + *name*, *loader* and *ispkg* (as ``is_package``) are passed to + :class:`importlib.machinery.ModuleSpec`. + + +.. function:: add_module(spec, *, parents=True) + + Create a :term:`module` from a name or spec and add it to :data:`sys.modules`. + + If *parents* is ``True``, create any missing parent modules. + + Return the new module. + + +.. function:: add_package(spec, *, parents=True) + + Create a :term:`package` from a name or spec and add it to :data:`sys.modules`. + + If *parents* is ``True``, create any missing parent modules. + + Return the new package. + + +Manipulating sys.modules +------------------------ + + +.. function:: unload(name) + + Remove the module named *name* from :data:`sys.modules`. + + +.. function:: forget(modname) + + Remove the module named *modname* from :data:`sys.modules` and delete any byte-compiled files of the module. -.. function:: import_fresh_module(name, fresh=(), blocked=(), deprecated=False) +.. function:: modules_setup() + + Return a tuple containing a copy of :data:`sys.modules`. + + +.. function:: modules_cleanup(oldmodules) + + Remove modules except for *oldmodules* and ``encodings`` from :data:`sys.modules` + in order to preserve the internal cache. + + +.. function:: isolated_modules() + + A context manager that makes a copy of :data:`sys.modules` on entry and restores + it on exit. + + +.. class:: CleanImport(*module_names, usefrozen=False) + + A context manager to force imports to return new module references. This + is useful for testing module-level behaviors, such as the emission of a + :exc:`DeprecationWarning` on import. + + When created, this makes a copy of :data:`sys.modules` and removes names from + *module_names* from the original. On exit, the original module references are + restored. + + If *usefrozen* is False (the default) then the frozen importer is + disabled (except for essential modules like ``importlib._bootstrap``). + + Example usage:: + + with CleanImport('foo'): + importlib.import_module('foo') # New reference. + + +Importing modules +----------------- + +.. function:: import_module(name, deprecated=False, *, required_on=()) + + This function imports and returns the named module. Unlike a normal + import, this function raises :exc:`unittest.SkipTest` if the module + cannot be imported. + + Module and package deprecation messages are suppressed during this import + if *deprecated* is ``True``. If a module is required on a platform but + optional for others, set *required_on* to an iterable of platform prefixes + which will be compared against :data:`sys.platform`. + + .. versionadded:: 3.1 + +.. function:: import_fresh_module(name, fresh=(), blocked=(), *, deprecated=False, usefrozen=False) This function imports and returns a fresh copy of the named Python module - by removing the named module from ``sys.modules`` before doing the import. - Note that unlike :func:`importlib.reload`, the original module is not + by removing the named module from :data:`sys.modules` before doing the import. + Note that unlike :func:`reload `, the original module is not affected by this operation. *fresh* is an iterable of additional module names that are also removed - from the ``sys.modules`` cache before doing the import. + from the :data:`sys.modules` cache before doing the import. *blocked* is an iterable of module names that are replaced with ``None`` in the module cache during the import to ensure that attempts to import @@ -1729,11 +1818,14 @@ The :mod:`!test.support.import_helper` module provides support for import tests. The named module and any modules named in the *fresh* and *blocked* parameters are saved before starting the import and then reinserted into - ``sys.modules`` when the fresh import is complete. + :data:`sys.modules` when the fresh import is complete. Module and package deprecation messages are suppressed during this import if *deprecated* is ``True``. + If *usefrozen* is False (the default) then the frozen importer is + disabled (except for essential modules like ``importlib._bootstrap``). + This function will raise :exc:`ImportError` if the named module cannot be imported. @@ -1749,65 +1841,91 @@ The :mod:`!test.support.import_helper` module provides support for import tests. .. versionadded:: 3.1 -.. function:: import_module(name, deprecated=False, *, required_on=()) +.. function:: ensure_module_imported(name, *, clearnone=True) - This function imports and returns the named module. Unlike a normal - import, this function raises :exc:`unittest.SkipTest` if the module - cannot be imported. + Import and return the named module. - Module and package deprecation messages are suppressed during this import - if *deprecated* is ``True``. If a module is required on a platform but - optional for others, set *required_on* to an iterable of platform prefixes - which will be compared against :data:`sys.platform`. + If the module is already imported, this is returned. Otherwise, import + the module and return it. If the module is not found a new, empty module + will be created. - .. versionadded:: 3.1 + If *clearnone* is ``True``, this will first remove any ``None`` values that + would block the import. -.. function:: modules_setup() +Manipulating sys.path +--------------------- - Return a copy of :data:`sys.modules`. +.. class:: DirsOnSysPath(*paths) + A context manager to temporarily add directories to :data:`sys.path`. -.. function:: modules_cleanup(oldmodules) + This makes a copy of :data:`sys.path`, appends any directories given + as positional arguments, then reverts :data:`sys.path` to the copied + settings when the context ends. - Remove modules except for *oldmodules* and ``encodings`` in order to - preserve internal cache. + Note that *all* :data:`sys.path` modifications in the body of the + context manager, including replacement of the object, + will be reverted at the end of the block. -.. function:: unload(name) +.. function:: ready_to_import(name=None, source="") + + A context manager that will create a new Python module *name* with *source* + source code in a temporary directory and inserts this directory at the start + of :data:`sys.path`. The context manager yields a tuple with the name and the path + to the script in the temporary directory. + + If the name matches an existing module, this will be cleared from :data:`sys.modules` + on entry and restored on exit. - Delete *name* from ``sys.modules``. + If no name is given, the module will be named "spam". -.. function:: make_legacy_pyc(source) + +Manipulating internals +---------------------- + +.. function:: make_legacy_pyc(source, allow_compile=False) Move a :pep:`3147`/:pep:`488` pyc file to its legacy pyc location and return the file system path to the legacy pyc file. The *source* value is the file system path to the source file. It does not need to exist, however the PEP - 3147/488 pyc file must exist. + 3147/488 pyc file must exist or *allow_compile* must be set. + If *allow_compile* is ``True`` this will create a .pyc file if it does not exist. -.. class:: CleanImport(*module_names) - A context manager to force import to return a new module reference. This - is useful for testing module-level behaviors, such as the emission of a - :exc:`DeprecationWarning` on import. Example usage:: +.. function:: frozen_modules(enabled=True) - with CleanImport('foo'): - importlib.import_module('foo') # New reference. + A context manager that forces frozen modules to be used or excluded. + If *enabled* is ``True`` frozen modules are used, otherwise they are excluded. -.. class:: DirsOnSysPath(*paths) + This only applies to modules that have not yet been imported. Some essential + modules will always be imported frozen. - A context manager to temporarily add directories to :data:`sys.path`. - This makes a copy of :data:`sys.path`, appends any directories given - as positional arguments, then reverts :data:`sys.path` to the copied - settings when the context ends. +.. function:: multi_interp_extensions_check(enabled=True) - Note that *all* :data:`sys.path` modifications in the body of the - context manager, including replacement of the object, - will be reverted at the end of the block. + A context manager that forces (if *enabled* is ``True``) or prevents legacy (single-phase init) + modules from being allowed in subinterpreters. + + This only applies to modules that haven't been imported yet. + It overrides the :c:member:`PyInterpreterConfig.check_multi_interp_extensions` + setting. + + +Lazy imports +------------ + +.. function:: ensure_lazy_imports(imported_module, modules_to_block, *, additional_code=None) + + Test that when *imported_module* is imported, none of the modules named in *modules_to_block* + are imported as a side effect. + + *additional_code* if given should be additional Python source code to execute before + checking that the blocked modules still haven't been imported. :mod:`!test.support.warnings_helper` --- Utilities for warnings tests From 043ddefd2bc7585c967c081b179ecbf9ada4d65f Mon Sep 17 00:00:00 2001 From: Iason Krommydas Date: Wed, 11 Nov 2026 22:55:16 +0100 Subject: [PATCH 094/229] gh-161827: Mark `_PyEval_EvalFrameDefault` as `Py_NO_INLINE` (GH-161829) gh-161827: Mark _PyEval_EvalFrameDefault as Py_NO_INLINE Signed-off-by: Iason Krommydas --- .../2026-11-11-21-52-02.gh-issue-161827.3PD3RK.rst | 4 ++++ Python/ceval.c | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-21-52-02.gh-issue-161827.3PD3RK.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-21-52-02.gh-issue-161827.3PD3RK.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-21-52-02.gh-issue-161827.3PD3RK.rst new file mode 100644 index 000000000000000..2ae6517db14f157 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-21-52-02.gh-issue-161827.3PD3RK.rst @@ -0,0 +1,4 @@ +Keep ``_PyEval_EvalFrameDefault`` as a separate frame in the C call stack +when Python is built with the tail-calling interpreter +(:option:`--with-tail-call-interp`), so that stack unwinders, profilers and +other tools that walk the stack can find the interpreter loop. diff --git a/Python/ceval.c b/Python/ceval.c index 0d053cdc7a63e34..ccc0edca2fc7940 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1200,7 +1200,13 @@ dtrace_function_return(_PyInterpreterFrame *frame) } #endif -PyObject* _Py_HOT_FUNCTION DONT_SLP_VECTORIZE +/* + * _PyEval_EvalFrameDefault must remain a separate frame in the C call stack: + * stack unwinders, profilers and other tools that walk the stack use it to + * find the interpreter loop. With the tail-calling interpreter it is small + * enough to be inlined into its callers, e.g. by Clang with PGO. + */ +Py_NO_INLINE PyObject* _Py_HOT_FUNCTION DONT_SLP_VECTORIZE _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { _Py_EnsureTstateNotNULL(tstate); From 9f03afd3700cb65390b388e33eec241e4628553e Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 11 Nov 2026 14:28:10 -0800 Subject: [PATCH 095/229] gh-51540: Let proxy auth handlers answer a 407 to CONNECT for https URLs (#161596) urllib.request could not authenticate with a proxy for an https URL using ProxyBasicAuthHandler or ProxyDigestAuthHandler: http.client raised a plain OSError when the proxy answered the CONNECT request with 407, so the http_error_407 handlers never ran. * http.client: add ProxyTunnelError, an OSError subclass carrying the code, reason and headers of the proxy's response. It holds no socket and can be pickled. * urllib.request: do_open() passes a 407 to CONNECT on to the error handlers, so the request is retried with Proxy-Authorization on the CONNECT. Other CONNECT failures still raise URLError. * Request.set_proxy() keeps the tunnel when a request is re-opened, so the retried request still sends the path, not the absolute URL. * ProxyDigestAuthHandler computes the digest over the CONNECT request (CONNECT host:port) when tunnelling. Co-authored-by: Claude Opus 5.5 --- Doc/library/http.client.rst | 26 +++ Doc/library/urllib.request.rst | 24 +++ Doc/whatsnew/3.16.rst | 22 +++ Lib/http/client.py | 17 +- Lib/test/test_httplib.py | 31 ++++ Lib/test/test_urllib2.py | 173 ++++++++++++++++++ Lib/urllib/request.py | 45 ++++- ...6-11-07-12-00-00.gh-issue-51540.Tunn07.rst | 8 + 8 files changed, 339 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-07-12-00-00.gh-issue-51540.Tunn07.rst diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 055971ec21aee37..22448c14f7a1888 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -230,6 +230,29 @@ The following exceptions are raised as appropriate: Previously, :exc:`BadStatusLine`\ ``('')`` was raised. +.. exception:: ProxyTunnelError + + A subclass of :exc:`OSError`. Raised when the proxy server answers the + CONNECT request sent for :meth:`HTTPConnection.set_tunnel` with a status + other than 200. + + .. attribute:: code + + The HTTP status code of the proxy's response, such as ``407``. + + .. attribute:: reason + + The reason phrase of the proxy's response. + + .. attribute:: headers + + The headers of the proxy's response, as an + :class:`http.client.HTTPMessage` instance. + + .. versionadded:: next + Previously, :exc:`OSError` was raised. + + The constants defined in this module are: .. data:: HTTP_PORT @@ -398,6 +421,9 @@ HTTPConnection Objects one will be automatically generated and transmitted if not provided in the headers argument. + .. versionchanged:: next + :exc:`ProxyTunnelError` is raised if the proxy rejects the CONNECT request. + .. method:: HTTPConnection.get_proxy_response_headers() diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 8234be98a2d6596..c3f749774b869c0 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -445,6 +445,10 @@ The following classes are provided: :ref:`http-password-mgr` for information on the interface that must be supported. + .. versionchanged:: next + Also handles a proxy that requires authentication for the CONNECT + request used to tunnel ``https`` URLs. + .. class:: AbstractDigestAuthHandler(password_mgr=None) @@ -482,6 +486,10 @@ The following classes are provided: :ref:`http-password-mgr` for information on the interface that must be supported. + .. versionchanged:: next + Also handles a proxy that requires authentication for the CONNECT + request used to tunnel ``https`` URLs. + .. class:: HTTPHandler() @@ -1102,6 +1110,14 @@ ProxyBasicAuthHandler Objects Retry the request with authentication information, if available. + For an ``https`` URL, the request goes through a tunnel that the proxy + sets up in reply to a CONNECT request. A 407 response can then come + either from the proxy, rejecting the CONNECT, or from the origin server, + through the tunnel. Both are handled the same way: the request is retried + with the credentials for the proxy, and those credentials are sent only in + the CONNECT request, never to the origin server. If the origin server + keeps answering 407, :exc:`~urllib.error.HTTPError` is raised. + .. _abstract-digest-auth-handler: @@ -1141,6 +1157,14 @@ ProxyDigestAuthHandler Objects Retry the request with authentication information, if available. + For an ``https`` URL, the request goes through a tunnel that the proxy + sets up in reply to a CONNECT request. A 407 response can then come + either from the proxy, rejecting the CONNECT, or from the origin server, + through the tunnel. Both are handled the same way: the request is retried + with the credentials for the proxy, and those credentials are sent only in + the CONNECT request, never to the origin server. If the origin server + keeps answering 407, :exc:`~urllib.error.HTTPError` is raised. + .. _http-handler-objects: diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 4a1fb6b6b87e131..da81d8513da99d7 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -508,6 +508,16 @@ gzip (Contributed by Marin Misur in :gh:`91372`.) +http.client +----------- + +* Add the :exc:`~http.client.ProxyTunnelError` exception, a subclass of + :exc:`OSError` raised when a proxy rejects the CONNECT request sent for + :meth:`~http.client.HTTPConnection.set_tunnel`. It exposes the status + code, reason and headers of the proxy's response. + (Contributed by Senthil Kumaran in :gh:`51540`.) + + imaplib ------- @@ -956,6 +966,18 @@ unicodedata * The Unicode database has been updated to Unicode 18.0.0. +urllib.request +-------------- + +* :class:`~urllib.request.ProxyBasicAuthHandler` and + :class:`~urllib.request.ProxyDigestAuthHandler` now authenticate with a + proxy that answers the CONNECT request for an ``https`` URL with + ``407 Proxy Authentication Required``. When no credentials are available, + :exc:`~urllib.error.HTTPError` with code 407 is raised instead of + :exc:`~urllib.error.URLError`. + (Contributed by Senthil Kumaran in :gh:`51540`.) + + uuid ---- diff --git a/Lib/http/client.py b/Lib/http/client.py index fe67d0f7963f786..96f266f32346ab9 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -87,8 +87,8 @@ "UnknownTransferEncoding", "UnimplementedFileMode", "IncompleteRead", "InvalidURL", "ImproperConnectionState", "CannotSendRequest", "CannotSendHeader", "ResponseNotReady", - "BadStatusLine", "LineTooLong", "RemoteDisconnected", "error", - "responses"] + "BadStatusLine", "LineTooLong", "RemoteDisconnected", + "ProxyTunnelError", "error", "responses"] HTTP_PORT = 80 HTTPS_PORT = 443 @@ -1045,7 +1045,8 @@ def _tunnel(self): if code != http.HTTPStatus.OK: self.close() - raise OSError(f"Tunnel connection failed: {code} {message.strip()}") + raise ProxyTunnelError(code, message.strip(), + _parse_header_lines(self._raw_proxy_headers)) finally: response.close() @@ -1617,5 +1618,15 @@ def __init__(self, *pos, **kw): BadStatusLine.__init__(self, "") ConnectionResetError.__init__(self, *pos, **kw) +class ProxyTunnelError(OSError): + def __init__(self, code, reason, headers): + OSError.__init__(self, f"Tunnel connection failed: {code} {reason}") + self.code = code + self.reason = reason + self.headers = headers + + def __reduce__(self): + return self.__class__, (self.code, self.reason, self.headers) + # for backwards compatibility error = HTTPException diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 1e5ec0d4e43c271..57eedee25a89e02 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -4,6 +4,7 @@ import io import itertools import os +import pickle import array import re import socket @@ -2786,6 +2787,36 @@ def test_no_proxy_response_headers(self): headers = self.conn.get_proxy_response_headers() self.assertIsNone(headers) + def test_tunnel_error(self): + response_text = ( + 'HTTP/1.1 407 Proxy Authentication Required\r\n' + 'Proxy-Authenticate: Basic realm="proxy"\r\n\r\n' + ) + self.conn._create_connection = self._create_connection(response_text) + self.conn.set_tunnel('destination.com') + + with self.assertRaises(client.ProxyTunnelError) as cm: + self.conn.request('HEAD', '/', '') + exc = cm.exception + self.assertIsInstance(exc, OSError) + self.assertEqual(exc.code, 407) + self.assertEqual(exc.reason, 'Proxy Authentication Required') + self.assertEqual(exc.headers['Proxy-Authenticate'], + 'Basic realm="proxy"') + self.assertEqual(str(exc), 'Tunnel connection failed: ' + '407 Proxy Authentication Required') + self.assertIsNone(self.conn.sock) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + copy = pickle.loads(pickle.dumps(exc, proto)) + self.assertIs(type(copy), client.ProxyTunnelError) + self.assertEqual(copy.code, 407) + self.assertEqual(copy.reason, exc.reason) + self.assertEqual(copy.headers['Proxy-Authenticate'], + 'Basic realm="proxy"') + self.assertEqual(str(copy), str(exc)) + def test_tunnel_leak(self): sock = None diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index f01b94c581e87ac..a198a9de68b64f3 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -8,6 +8,7 @@ import os import io +import hashlib import ftplib import socket import array @@ -617,6 +618,42 @@ def https_open(self, req): return self.do_open(self.httpconn, req) +class MockHTTPTunnelClass(MockHTTPClass): + # Stands in for a proxy that requires authentication: CONNECT fails + # with tunnel_code until the credentials arrive in the tunnel headers + # (tunnel_code=None: no authentication). The origin then answers + # through the tunnel with origin_code. + def __init__(self, tunnel_code=407, challenge='Basic realm="proxy"', + origin_code=200): + super().__init__() + self.tunnel_code = tunnel_code + self.challenge = challenge + self.origin_code = origin_code + self.tunnel_requests = [] + + def challenge_headers(self): + return http.client.parse_headers(io.BytesIO( + b'Proxy-Authenticate: %s\r\n\r\n' % self.challenge.encode())) + + def request(self, method, url, body=None, headers=None, *, + encode_chunked=False): + self.tunnel_requests.append(dict(self._tunnel_headers)) + if (self.tunnel_code is not None and + "Proxy-Authorization" not in self._tunnel_headers): + raise http.client.ProxyTunnelError( + self.tunnel_code, http.client.responses[self.tunnel_code], + self.challenge_headers()) + super().request(method, url, body, headers, + encode_chunked=encode_chunked) + + def getresponse(self): + response = super().getresponse() + response.code = self.origin_code + if self.origin_code != 200: + response.info = self.challenge_headers + return response + + class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler): # useful for testing auth # sends supplied code response @@ -1645,6 +1682,142 @@ def test_proxy_https_proxy_authorization(self): self.assertEqual(req.host, "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") + def _open_through_tunnel(self, httpconn, password_mgr, + auth_handler=urllib.request.ProxyBasicAuthHandler, + url="https://www.example.com/path"): + https_handler = MockHTTPSHandler() + https_handler.httpconn = httpconn + opener = urllib.request.build_opener( + urllib.request.ProxyHandler(dict(https="proxy.example.com:3128")), + auth_handler(password_mgr), + https_handler) + return opener.open(url) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_proxy_https_basic_auth_handler(self): + # gh-51540: a 407 from the proxy to CONNECT is passed to + # ProxyBasicAuthHandler, which retries with credentials + password_mgr = urllib.request.HTTPPasswordMgr() + password_mgr.add_password("proxy", "proxy.example.com:3128", + "user", "pass") + httpconn = MockHTTPTunnelClass() + response = self._open_through_tunnel(httpconn, password_mgr) + self.assertEqual(response.code, 200) + + self.assertEqual(len(httpconn.tunnel_requests), 2) + self.assertNotIn("Proxy-Authorization", httpconn.tunnel_requests[0]) + self.assertEqual(httpconn.tunnel_requests[1]["Proxy-Authorization"], + "Basic dXNlcjpwYXNz") + # The credentials go to the proxy only, not to the origin. + self.assertEqual(httpconn._tunnel_host, "www.example.com") + self.assertEqual(httpconn.selector, "/path") + self.assertNotIn("Proxy-Authorization", + dict(httpconn.req_headers)) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + @skip_libssl_fips_mode # Uses MD5 algorithm in challenge + def test_proxy_https_digest_auth_handler(self): + # gh-51540: a 407 from the proxy to CONNECT is passed to + # ProxyDigestAuthHandler, which retries with credentials + password_mgr = urllib.request.HTTPPasswordMgr() + password_mgr.add_password("proxy", "proxy.example.com:3128", + "user", "pass") + httpconn = MockHTTPTunnelClass( + challenge='Digest realm="proxy", nonce="abcdef0123456789", ' + 'qop="auth"') + response = self._open_through_tunnel( + httpconn, password_mgr, urllib.request.ProxyDigestAuthHandler) + self.assertEqual(response.code, 200) + + self.assertEqual(len(httpconn.tunnel_requests), 2) + self.assertNotIn("Proxy-Authorization", httpconn.tunnel_requests[0]) + self.assertStartsWith( + httpconn.tunnel_requests[1]["Proxy-Authorization"], + 'Digest username="user", realm="proxy", ') + self.assertEqual(httpconn.selector, "/path") + self.assertNotIn("Proxy-Authorization", + dict(httpconn.req_headers)) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + @skip_libssl_fips_mode # Uses MD5 algorithm in challenge + def test_proxy_https_digest_auth_connect(self): + # Through an https tunnel, the digest covers the CONNECT request + # (method CONNECT, request-target host:port), not the request + # later sent to the origin + def md5(text): + return hashlib.md5(text.encode("ascii")).hexdigest() + + nonce = "abcdef0123456789" + for url, target in [ + ("https://www.example.com/path", "www.example.com:443"), + ("https://www.example.com:8443/path", "www.example.com:8443"), + ]: + with self.subTest(url=url): + password_mgr = urllib.request.HTTPPasswordMgr() + password_mgr.add_password("proxy", "proxy.example.com:3128", + "user", "pass") + httpconn = MockHTTPTunnelClass( + challenge='Digest realm="proxy", nonce="%s"' % nonce) + self._open_through_tunnel( + httpconn, password_mgr, + urllib.request.ProxyDigestAuthHandler, url) + + expected = md5("%s:%s:%s" % (md5("user:proxy:pass"), nonce, + md5("CONNECT:" + target))) + auth = httpconn.tunnel_requests[1]["Proxy-Authorization"] + self.assertIn('uri="%s"' % target, auth) + self.assertIn('response="%s"' % expected, auth) + self.assertEqual(httpconn.selector, "/path") + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_proxy_https_origin_407(self): + # A 407 sent by the origin through the tunnel is retried like one + # from the proxy, but the credentials only go on the CONNECT + password_mgr = urllib.request.HTTPPasswordMgr() + password_mgr.add_password("proxy", "proxy.example.com:3128", + "user", "pass") + httpconn = MockHTTPTunnelClass(tunnel_code=None, origin_code=407) + with self.assertRaises(urllib.error.HTTPError) as cm: + self._open_through_tunnel(httpconn, password_mgr) + with cm.exception as err: + self.assertEqual(err.code, 407) + + self.assertEqual(len(httpconn.tunnel_requests), 2) + self.assertNotIn("Proxy-Authorization", httpconn.tunnel_requests[0]) + self.assertEqual(httpconn.tunnel_requests[1]["Proxy-Authorization"], + "Basic dXNlcjpwYXNz") + # Neither request to the origin carried the credentials. + self.assertNotIn("Proxy-Authorization", + dict(httpconn.req_headers)) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_proxy_https_basic_auth_handler_no_password(self): + password_mgr = urllib.request.HTTPPasswordMgr() + httpconn = MockHTTPTunnelClass() + with self.assertRaises(urllib.error.HTTPError) as cm: + self._open_through_tunnel(httpconn, password_mgr) + with cm.exception as err: + self.assertEqual(err.code, 407) + self.assertEqual(err.reason, "Proxy Authentication Required") + self.assertEqual(err.headers["Proxy-Authenticate"], + 'Basic realm="proxy"') + self.assertEqual(err.read(), b"") + self.assertEqual(len(httpconn.tunnel_requests), 1) + + @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') + def test_proxy_https_tunnel_error(self): + # Other failures to CONNECT are still raised as URLError + password_mgr = urllib.request.HTTPPasswordMgr() + password_mgr.add_password("proxy", "proxy.example.com:3128", + "user", "pass") + httpconn = MockHTTPTunnelClass(tunnel_code=403) + with self.assertRaises(urllib.error.URLError) as cm: + self._open_through_tunnel(httpconn, password_mgr) + self.assertNotIsInstance(cm.exception, urllib.error.HTTPError) + self.assertIsInstance(cm.exception.reason, + http.client.ProxyTunnelError) + self.assertEqual(cm.exception.reason.code, 403) + @unittest.skipUnless(os.name == "nt", "only relevant for Windows") def test_winreg_proxy_bypass(self): proxy_override = "www.example.com;*.example.net; 192.168.0.1" diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index da634a205c195e4..a5166caed586d74 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -357,8 +357,10 @@ def get_full_url(self): return self.full_url def set_proxy(self, host, type): - if self.type == 'https' and not self._tunnel_host: - self._tunnel_host = self.host + if self.type == 'https': + # A retried request keeps the tunnel host set on the first pass. + if not self._tunnel_host: + self._tunnel_host = self.host else: self.type= type self.selector = self.full_url @@ -1228,10 +1230,19 @@ def get_authorization(self, req, chal): # HTTP proxy. RFC 7616, Section 3.4 allows both. An empty path is # sent as '/' by http.client, so use the same here. - uri = req.selector or '/' + if self.auth_header == 'Proxy-Authorization' and req._tunnel_host: + # For an https URL the proxy authenticates the CONNECT request, + # whose request-target is host:port, as sent by http.client. + method = 'CONNECT' + host, port = _splitport(req._tunnel_host) + uri = '%s:%s' % (host.encode('idna').decode('ascii'), + port or http.client.HTTPS_PORT) + else: + method = req.get_method() + uri = req.selector or '/' A1 = "%s:%s:%s" % (user, realm, pw) - A2 = "%s:%s" % (req.get_method(), uri) + A2 = "%s:%s" % (method, uri) # NOTE: As per RFC 2617, when server sends "auth,auth-int", the client could use either `auth` # or `auth-int` to the response back. we use `auth` to send the response back. if qop is None: @@ -1314,6 +1325,26 @@ def http_error_407(self, req, fp, code, msg, headers): self.reset_retry_count() return retry +class _ProxyTunnelResponse(io.BytesIO): + """The proxy's error response to CONNECT, as an empty-bodied response.""" + + def __init__(self, err, url): + super().__init__() + self.url = url + self.code = self.status = err.code + self.msg = self.reason = err.reason + self.headers = err.headers + + def info(self): + return self.headers + + def geturl(self): + return self.url + + def getcode(self): + return self.code + + class AbstractHTTPHandler(BaseHandler): def __init__(self, debuglevel=None): @@ -1407,6 +1438,12 @@ def do_open(self, http_class, req, **http_conn_args): try: h.request(req.get_method(), req.selector, req.data, headers, encode_chunked=req.has_header('Transfer-encoding')) + except http.client.ProxyTunnelError as err: + if err.code != 407: + raise URLError(err) + # Pass the proxy's challenge to CONNECT on to the + # http_error_407 handlers, like any other error response. + return _ProxyTunnelResponse(err, req.full_url) except OSError as err: # timeout error raise URLError(err) r = h.getresponse() diff --git a/Misc/NEWS.d/next/Library/2026-11-07-12-00-00.gh-issue-51540.Tunn07.rst b/Misc/NEWS.d/next/Library/2026-11-07-12-00-00.gh-issue-51540.Tunn07.rst new file mode 100644 index 000000000000000..9efd6dd0cca7db7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-07-12-00-00.gh-issue-51540.Tunn07.rst @@ -0,0 +1,8 @@ +:class:`urllib.request.ProxyBasicAuthHandler` and +:class:`urllib.request.ProxyDigestAuthHandler` now handle a ``407 Proxy +Authentication Required`` response to the CONNECT request for an ``https`` +URL. :mod:`http.client` raises the new :exc:`~http.client.ProxyTunnelError`, +a subclass of :exc:`OSError`, when a proxy rejects a CONNECT request. +For such a proxy, :class:`~urllib.request.ProxyDigestAuthHandler` computes +the digest over the CONNECT request (``CONNECT host:port``) instead of the +request sent to the origin server. From 35e2c5abc940e616d84542d32bef80c4ae1faf2c Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:54:21 -0800 Subject: [PATCH 096/229] gh-160303: Fix bytearray state after a failed del b[:n] (GH-160304) * gh-160303: Fix bytearray state after a failed del b[:n] When shrinking the storage of a bytearray which has a logical offset, allocate the new buffer before modifying the object, so that bytearray_setslice_linear() can restore ob_start if the allocation fails. * gh-160303: Keep ob_start instead of allocating in del b[:n] Move ob_start and the size before the shrinking resize, so a failed shrink leaves the bytes removed and the larger block in place, as the non-front path already does. Split the checks out of bytearray_resize_lock_held() so the slice code can resize directly. Move the tests into test_resize_error. * gh-160303: Test failed shrinks not at the front; narrow the unchecked resize Only the front path of bytearray_setslice_linear() calls bytearray_resize_unchecked(): it has already advanced ob_start and reduced the size, so bytearray_resize_lock_held() would return early without releasing memory. Other slices go through bytearray_resize_lock_held() again and keep the failure fixup to themselves. test_resize_error now also covers slices in the middle and at the end, and checks the null byte after the data with float(). --- Lib/test/test_bytes.py | 31 +++++++ ...-10-22-15-40-00.gh-issue-160303.q7Zb2K.rst | 5 + Objects/bytearrayobject.c | 93 +++++++++++-------- 3 files changed, 91 insertions(+), 38 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-15-40-00.gh-issue-160303.q7Zb2K.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 67960f5709b08e8..8a3b60dca42d6ca 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1661,6 +1661,10 @@ def test_resize(self): def test_resize_error(self): # gh-157242: If bytearray.resize() fails (MemoryError), # the bytearray must be left unchanged. + # + # gh-160303: If a slice is deleted or replaced by shorter data and + # shrinking the memory block fails (MemoryError), the bytearray + # must be modified anyway. offset = 3 for logical_offset in (False, True): @@ -1687,6 +1691,33 @@ def test_resize_error(self): ba.resize(1) self.assertEqual(ba, expected) + # shrink bytearray with a slice at the front, in the middle + # and at the end + data = b'0123456789' + if logical_offset: + data = data[offset:] + for start, stop in ((0, -2), (1, -1), (2, len(data))): + for replacement in (b'', b'.'): + with self.subTest(start=start, stop=stop, + replacement=replacement): + ba = bytearray(b'0123456789') + if logical_offset: + del ba[:offset] + index = slice(start, stop) + value = bytearray(replacement) + expected = data[:start] + replacement + data[stop:] + with inject_memory_error(self): + if replacement: + ba[index] = value + else: + del ba[index] + self.assertEqual(ba, expected) + # float() relies on the null byte after the data. + self.assertEqual(float(ba), float(expected)) + ba.append(ord('5')) + del ba[:1] + self.assertEqual(ba, expected[1:] + b'5') + def test_take_bytes(self): ba = bytearray(b'ab') self.assertEqual(ba.take_bytes(), b'ab') diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-15-40-00.gh-issue-160303.q7Zb2K.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-15-40-00.gh-issue-160303.q7Zb2K.rst new file mode 100644 index 000000000000000..ba72afc400bd0ab --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-15-40-00.gh-issue-160303.q7Zb2K.rst @@ -0,0 +1,5 @@ +Fix :class:`bytearray` reading and writing memory outside of its buffer +after ``del b[:n]`` or ``b[:n] = shorter`` failed with :exc:`MemoryError`. +The bytes are now removed even if shrinking the memory block fails; +:exc:`MemoryError` is still raised. Also fix the missing null byte after the +data when shrinking fails for a slice which is not at the front. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 317534c99f038d4..b9d547575bd2eba 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -266,13 +266,22 @@ bytearray_resize_storage(PyByteArrayObject *self, } +/* Resize without the checks of bytearray_resize_lock_held(): the caller must + have checked that the bytearray can be resized. + + Unlike bytearray_resize_lock_held(), this does not return early if + requested_size is already the size of the bytearray. A caller which + removed bytes by advancing ob_start and reducing the size uses that to + release the memory. */ static int -bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) +bytearray_resize_unchecked(PyByteArrayObject *obj, Py_ssize_t requested_size) { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - PyByteArrayObject *obj = ((PyByteArrayObject *)self); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); + PyObject *self = (PyObject *)obj; assert(obj->ob_bytes_object != NULL); + assert(requested_size >= 0); + assert(obj->ob_exports == 0); /* All computations are done unsigned to avoid integer overflows (see issue #22335). */ @@ -280,23 +289,8 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) size_t logical_offset = (size_t) (obj->ob_start - obj->ob_bytes); size_t size = (size_t) requested_size; - assert(self != NULL); - assert(PyByteArray_Check(self)); assert(logical_offset <= alloc); - if (requested_size < 0) { - PyErr_Format(PyExc_ValueError, - "Can only resize to positive sizes, got %zd", requested_size); - return -1; - } - - if (requested_size == Py_SIZE(self)) { - return 0; - } - if (!_canresize(obj)) { - return -1; - } - /* Resize to 0 resets to empty bytes (see issue #153419). */ if (requested_size == 0) { Py_SETREF(obj->ob_bytes_object, @@ -348,6 +342,30 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) return 0; } +static int +bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + PyByteArrayObject *obj = ((PyByteArrayObject *)self); + + assert(self != NULL); + assert(PyByteArray_Check(self)); + + if (requested_size < 0) { + PyErr_Format(PyExc_ValueError, + "Can only resize to positive sizes, got %zd", requested_size); + return -1; + } + + if (requested_size == Py_SIZE(self)) { + return 0; + } + if (!_canresize(obj)) { + return -1; + } + return bytearray_resize_unchecked(obj, requested_size); +} + int PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) { @@ -612,14 +630,22 @@ bytearray_setslice_linear(PyByteArrayObject *self, return -1; if (lo == 0) { - /* Shrink the buffer by advancing its logical start */ + /* Shrink the buffer by advancing its logical start. The bytes + are removed at this point; the resize below only releases + memory. */ self->ob_start -= growth; + Py_SET_SIZE(self, Py_SIZE(self) + growth); /* 0 lo hi old_size | |<----avail----->|<-----tail------>| | |<-bytes_len->|<-----tail------>| 0 new_lo new_hi new_size */ + if (bytearray_resize_unchecked(self, Py_SIZE(self)) < 0) { + /* The operation is completed, but a MemoryError is still + raised and the memory block is not shrunk. */ + res = -1; + } } else { /* @@ -630,26 +656,17 @@ bytearray_setslice_linear(PyByteArrayObject *self, */ memmove(buf + lo + bytes_len, buf + hi, Py_SIZE(self) - hi); - } - if (bytearray_resize_lock_held((PyObject *)self, - Py_SIZE(self) + growth) < 0) { - /* Issue #19578: Handling the memory allocation failure here is - tricky here because the bytearray object has already been - modified. Depending on growth and lo, the behaviour is - different. - - If growth < 0 and lo != 0, the operation is completed, but a - MemoryError is still raised and the memory block is not - shrunk. Otherwise, the bytearray is restored in its previous - state and a MemoryError is raised. */ - if (lo == 0) { - self->ob_start += growth; - return -1; + if (bytearray_resize_lock_held((PyObject *)self, + Py_SIZE(self) + growth) < 0) { + /* Issue #19578: memmove() removed bytes, the bytearray + object cannot be restored in its previous state. + + The operation is completed, but a MemoryError is still + raised and the memory block is not shrunk. */ + Py_SET_SIZE(self, Py_SIZE(self) + growth); + bytearray_write_trailing_null_byte(self); + res = -1; } - /* memmove() removed bytes, the bytearray object cannot be - restored in its previous state. */ - Py_SET_SIZE(self, Py_SIZE(self) + growth); - res = -1; } buf = PyByteArray_AS_STRING(self); } From f112215dd7ef054655093e880384dbaed0fbfc66 Mon Sep 17 00:00:00 2001 From: Damian Shaw Date: Wed, 11 Nov 2026 19:01:45 -0500 Subject: [PATCH 097/229] gh-161763: Correct SSL docs for unconnected client sockets (#GH-61772) * gh-161763: Correct SSL docs for unconnected client sockets * gh-161763: Qualify SSL socket method references --- Doc/library/ssl.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index d1c099bd77e47ea..868ef0667597435 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -101,6 +101,7 @@ Client socket example with custom context and IPv4:: with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock: with context.wrap_socket(sock, server_hostname=hostname) as ssock: + ssock.connect((hostname, 443)) print(ssock.version()) @@ -2236,13 +2237,12 @@ to speed up repeated connections from the same clients. The parameter ``server_side`` is a boolean which identifies whether server-side or client-side behavior is desired from this socket. - For client-side sockets, the context construction is lazy; if the - underlying socket isn't connected yet, the context construction will be - performed after :meth:`connect` is called on the socket. For - server-side sockets, if the socket has no remote peer, it is assumed + For client-side sockets, if *sock* is not yet connected, the underlying + SSL object is created when :meth:`~socket.socket.connect` is called. + For server-side sockets, if the socket has no remote peer, it is assumed to be a listening socket, and the server-side SSL wrapping is automatically performed on client connections accepted via the - :meth:`accept` method. The method may raise :exc:`SSLError`. + :meth:`~socket.socket.accept` method. The method may raise :exc:`SSLError`. On client connections, the optional parameter *server_hostname* specifies the hostname of the service which we are connecting to. This allows a From eb21dea301df801352ac6b4b21117fe6ef5cae4e Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:22:10 -0800 Subject: [PATCH 098/229] gh-161060: Fix pyexpat crash when the default handler is cleared (#161061) Expat calls the default handler once per 1024-byte piece of a token that needs encoding conversion. Install a no-op instead of NULL when the handler is cleared after an error or unset from within a callback, as is already done for the character data handler. ANT-2026-0SJB3WWC --- Lib/test/test_pyexpat.py | 97 +++++++++++++++++++ ...-11-01-05-05-38.gh-issue-161060.YOZsVl.rst | 5 + Modules/pyexpat.c | 44 +++++++-- 3 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-01-05-05-38.gh-issue-161060.YOZsVl.rst diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index c52c640bd4b26f8..b962f47bc844c4d 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -834,6 +834,103 @@ def handler(text): self.assertRaises(SpecificException, parser.Parse, xml.encode('iso8859')) + +class DefaultHandlerExceptionTest(unittest.TestCase): + # In a document that is not UTF-8, Expat passes a long piece of text such + # as this comment to the default handler in chunks of at most 1024 bytes, + # re-reading the handler between chunks. Clearing the handler from within + # the callback, by raising or by setting it to None, must not crash. + xml = ("" + % ('a' * 2000)).encode('iso8859') + + class SpecificException(Exception): + pass + + def check(self, name): + def handler(text): + if text.startswith('" + % ('a' * 2000)).encode('iso8859') + parser.Parse(xml, True) + + def test_CharacterDataHandler_with_buffer_text(self): + # With buffer_text, buffered character data is passed to the + # CharacterDataHandler just before the default handler is called; + # an exception there clears the handlers while Expat is still + # passing the comment to the default handler. + def handler(text): + raise self.SpecificException + + parser = expat.ParserCreate() + parser.buffer_text = True + parser.CharacterDataHandler = handler + parser.DefaultHandler = lambda text: None + xml = ("a" + % ('a' * 2000)).encode('iso8859') + self.assertRaises(self.SpecificException, parser.Parse, xml, True) + + class ChardataBufferTest(unittest.TestCase): """ test setting of chardata buffer size diff --git a/Misc/NEWS.d/next/Library/2026-11-01-05-05-38.gh-issue-161060.YOZsVl.rst b/Misc/NEWS.d/next/Library/2026-11-01-05-05-38.gh-issue-161060.YOZsVl.rst new file mode 100644 index 000000000000000..8d55e0162c452ed --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-01-05-05-38.gh-issue-161060.YOZsVl.rst @@ -0,0 +1,5 @@ +Fix a crash in :mod:`xml.parsers.expat` when a handler raises an exception, +or the :meth:`~xml.parsers.expat.xmlparser.DefaultHandler` or +:meth:`~xml.parsers.expat.xmlparser.DefaultHandlerExpand` handler is unset, +while a token longer than 1024 bytes, such as a comment, is being reported to +the default handler from a document that is not encoded in UTF-8. diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index fb9d333babf70b4..a6063f034463c82 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -293,10 +293,13 @@ error_external_entity_ref_handler(XML_Parser parser, /* Dummy character data handler used when an error (exception) has been detected, and the actual parsing can be terminated early. - This is needed since character data handler can't be safely removed - from within the character data handler, but can be replaced. It is - used only from the character data handler trampoline, and must be - used right after `flag_error()` is called. */ + This is needed since the character data and default handlers can't be + safely removed from within a callback, only replaced: Expat may call + them several times for one piece of text (a long comment in a document + that is not UTF-8, for example). The default handler has the same + signature, so this function serves for both. clear_handlers() installs + it after `flag_error()`, and the handler setter installs it when a + handler is set to None from within a callback. */ static void noop_character_data_handler(void *userData, const XML_Char *data, int len) { @@ -382,15 +385,18 @@ call_character_handler(xmlparseobject *self, const XML_Char *buffer, int len) temp = call_with_frame("CharacterData", __LINE__, self->handlers[CharacterData], args, self); /* temp is an owned reference again, or NULL */ - self->in_callback = 0; Py_DECREF(args); if (temp == NULL) { + self->in_callback = 0; flag_error(self); XML_SetCharacterDataHandler(self->itself, noop_character_data_handler); return -1; } + /* Releasing the result can run arbitrary code, which must still be + treated as being within the callback by the handler setter. */ Py_DECREF(temp); + self->in_callback = 0; return 0; } @@ -579,14 +585,18 @@ my_ ## NAME ## Handler PARAMS { \ PyObject *rv = call_with_frame( \ #NAME, __LINE__, \ self->handlers[NAME], args, self); \ - self->in_callback = 0; \ Py_DECREF(args); \ if (rv == NULL) { \ + self->in_callback = 0; \ flag_error(self); \ return RETURN_VARIABLE; \ } \ CONVERSION \ + /* Releasing the result can run arbitrary code, \ + which must still be treated as being within \ + the callback by the handler setter. */ \ Py_DECREF(rv); \ + self->in_callback = 0; \ if (PyErr_Occurred()) { \ invalid_expat_handler_rv(#NAME); \ } \ @@ -1838,8 +1848,14 @@ xmlparse_handler_setter(PyObject *op, PyObject *v, void *closure) active, so Expat does a little more work than it otherwise would, but that's really an odd case. A more elaborate system of handlers and state could remove the - C handler more effectively. */ - if (handlernum == CharacterData && self->in_callback) { + C handler more effectively. The same applies to the + default handler, which Expat can call several times for one + piece of text, such as a long comment in a document that is + not UTF-8. */ + if (self->in_callback + && (handlernum == CharacterData + || handlernum == Default + || handlernum == DefaultHandlerExpand)) { c_handler = (xmlhandler)noop_character_data_handler; } v = NULL; @@ -2786,7 +2802,17 @@ clear_handlers(xmlparseobject *self, int initial) } else { Py_CLEAR(self->handlers[i]); - CALL_XML_HANDLER_SETTER(&handler_info[i], self->itself, NULL); + xmlhandler c_handler = NULL; + if (i == Default || i == DefaultHandlerExpand) { + /* This may be called from within a callback, where the + default handler can't be removed: for a document that is + not UTF-8, Expat passes a long piece of text to it in + several calls and re-reads the handler before each one + (reportDefault() in xmlparse.c). The no-op handler is + harmless otherwise. */ + c_handler = (xmlhandler)noop_character_data_handler; + } + CALL_XML_HANDLER_SETTER(&handler_info[i], self->itself, c_handler); } } } From 9736a841a1f11065aeb7792cd5f1b00ea7b0c893 Mon Sep 17 00:00:00 2001 From: IngridMorstrad Date: Wed, 11 Nov 2026 22:45:13 -0800 Subject: [PATCH 099/229] gh-42550: Add 'Expect: 100-Continue' support to httplib (#133276) * Add 'Expect: 100-Continue' support to httplib Previously, http.client would always send content body immediately and ignore any 100 responses. This change makes HTTPClient.request() wait for a `Continue` response if the `Expect: 100-Continue` header is set, and adds a parameter to HTTPClient.getresponse() that will cause it to return 100 responses instead of eating them. --------- Co-authored-by: Peter Bierma Co-authored-by: Senthil Kumaran --- Doc/library/http.client.rst | 94 +++++- Doc/whatsnew/3.16.rst | 11 + Lib/http/client.py | 136 +++++++-- Lib/test/test_httplib.py | 274 ++++++++++++++++++ Misc/ACKS | 2 + ...6-09-27-13-04-02.gh-issue-42550.sqYlvB.rst | 3 + 6 files changed, 487 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-27-13-04-02.gh-issue-42550.sqYlvB.rst diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 22448c14f7a1888..20fd357c56ac3ef 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -282,7 +282,7 @@ HTTPConnection Objects .. method:: HTTPConnection.request(method, url, body=None, headers={}, *, \ - encode_chunked=False) + encode_chunked=False, continue_timeout=2.5) This will send a request to the server using the HTTP request method *method* and the request URI *url*. The provided *url* must be @@ -358,11 +358,33 @@ HTTPConnection Objects No attempt is made to determine the Content-Length for file objects. -.. method:: HTTPConnection.getresponse() + .. versionchanged:: next + The *continue_timeout* parameter was added. + + If the headers include ``Expect: 100-Continue`` and *body* is set, the + body will not be sent until either a ``100 Continue`` response is + received from the server or *continue_timeout* seconds have elapsed; if a + response code other than 100 is received, the body will not be sent at all. + + *continue_timeout* value must be greater than zero, otherwise + :exc:`ValueError` is raised. + +.. method:: HTTPConnection.getresponse(support_continue=False) Should be called after a request is sent to get the response from the server. Returns an :class:`HTTPResponse` instance. + By default, ``100 Continue`` responses are skipped and the method returns + the first response with a status other than 100. If *support_continue* + is true, a ``100 Continue`` response is returned to the caller, which must + then send the request body and call :meth:`getresponse` again to obtain + the final response. + + .. note:: + + The whole response must be read before a new request can be sent to + the server. + .. versionchanged:: 3.5 If a :exc:`ConnectionError` or subclass is raised, the :class:`HTTPConnection` object will be ready to reconnect when @@ -372,6 +394,10 @@ HTTPConnection Objects socket. Instead the caller is responsible to call :meth:`close` on the existing connection. + .. versionchanged:: next + Added the *support_continue* parameter. (In prior versions + a ``Continue`` response was always ignored.) + .. method:: HTTPConnection.set_debuglevel(level) @@ -470,7 +496,8 @@ also send your request step by step, by using the four functions below. .. method:: HTTPConnection.putrequest(method, url, skip_host=False, \ - skip_accept_encoding=False) + skip_accept_encoding=False, \ + continue_timeout=2.5) This should be the first call after the connection to the server has been made. It sends a line to the server consisting of the *method* string, @@ -479,6 +506,14 @@ also send your request step by step, by using the four functions below. additional content encodings), specify *skip_host* or *skip_accept_encoding* with non-False values. + If an ``Expect: 100-Continue`` header is sent, the optional + *continue_timeout* parameter sets how many seconds the client will wait for + a ``100 Continue`` response before sending the message body, It must be + greater than zero, otherwise :exc:`ValueError` is raised. + + .. versionchanged:: next + *continue_timeout* parameter was added. + .. method:: HTTPConnection.putheader(header, argument[, ...]) @@ -492,7 +527,12 @@ also send your request step by step, by using the four functions below. Send a blank line to the server, signalling the end of the headers. The optional *message_body* argument can be used to pass a message body - associated with the request. + associated with the request. If a body is provided and an + ``Expect: 100-Continue`` header has been sent, the body will not be sent + until a ``100 Continue`` response is received from the server; if another + response code is received, the body is not sent at all. If no response + is received within the *continue_timeout* given to :meth:`putrequest`, + the body will be sent regardless. If *encode_chunked* is ``True``, the result of each iteration of *message_body* will be chunk-encoded as specified in :rfc:`7230`, @@ -513,6 +553,12 @@ also send your request step by step, by using the four functions below. .. versionchanged:: 3.6 Added chunked encoding support and the *encode_chunked* parameter. + .. versionchanged:: next + If an ``Expect: 100-Continue`` header has been sent, the body will not be + sent until a ``100 Continue`` response is received from the server, or a + contine_timeout has elapsed. If the response code is not 100, the body + will not be sent at all. + .. method:: HTTPConnection.send(data) @@ -691,6 +737,46 @@ method attribute. Here is an example session that uses the ``PUT`` method:: >>> print(response.status, response.reason) 200 OK +Conditional transmission of the body is supported when an ``Expect: 100-Continue`` +header is set. To use this in a simple case, just set the header, and +optionally the time for which the client should wait for a ``100 Continue`` +response before sending the body regardless:: + + >>> import http.client + >>> BODY = "***filecontents***" + >>> conn = http.client.HTTPConnection("localhost", 8080) + >>> conn.request("PUT", "/file", BODY, headers={'Expect': '100-Continue'}, + ... continue_timeout=1.0) + >>> response = conn.getresponse() + >>> # You will not see the '100' response, as it is handled internally + >>> print(response.status, response.reason) + 200, OK + +Here is a more complex example in which we manually check the response and +decide whether to send the body. This may be useful if the body must be +generated by some resource-intensive process which should be skipped if the +server will not accept it. *continue_timeout* does not apply here, so this +should only be used with servers known to support ``Expect: 100-Continue``; +otherwise :meth:`~HTTPConnection.getresponse` may block until the +connection's *timeout* expires. :: + + >>> import http.client + >>> conn = http.client.HTTPConnection("localhost", 8080) + >>> conn.putrequest("PUT", "/file") + >>> conn.putheader('Expect', '100-Continue') + >>> # Assuming you know in advance what the length will be + >>> # If not, you will need to encode it as chunked + >>> conn.putheader('Content-Length', '42') + >>> conn.endheaders() + >>> response = conn.getresponse(support_continue=True) + >>> print(response.status, response.reason) + 100, Continue + >>> BODY = resource_intensive_calculation() + >>> conn.send(BODY) + >>> response = conn.getresponse() + >>> print(response.status, response.reason) + 200, OK + .. _httpmessage-objects: HTTPMessage Objects diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index da81d8513da99d7..f86a6ec69172d8a 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -511,6 +511,17 @@ gzip http.client ----------- +* :class:`~http.client.HTTPConnection` now honors an ``Expect: 100-Continue`` + request header: the request body is sent only after the server responds + with ``100 Continue``, or once the new *continue_timeout* of + :meth:`~http.client.HTTPConnection.request` and + :meth:`~http.client.HTTPConnection.putrequest` expires. If the server + responds with a final status instead, the body is not sent. + :meth:`~http.client.HTTPConnection.getresponse` gains a *support_continue* + parameter to return ``100 Continue`` responses to the caller. + (Contributed by Ashwin Menon, Tim Bartlett and Senthil Kumaran in + :gh:`42550`.) + * Add the :exc:`~http.client.ProxyTunnelError` exception, a subclass of :exc:`OSError` raised when a proxy rejects the CONNECT request sent for :meth:`~http.client.HTTPConnection.set_tunnel`. It exposes the status diff --git a/Lib/http/client.py b/Lib/http/client.py index 96f266f32346ab9..4bdb8f72f979b16 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -75,11 +75,17 @@ import http import io import re +import select import socket import sys import collections.abc from urllib.parse import urlsplit +try: + import ssl +except ImportError: + ssl = None + # HTTPMessage, parse_headers(), and the HTTP status code constants are # intentionally omitted for simplicity __all__ = ["HTTPResponse", "HTTPConnection", @@ -119,6 +125,10 @@ # timeout window. _MAXINTERIMRESPONSES = 100 +# default number of seconds to wait for a "100 Continue" response before +# sending the body of a request with an "Expect: 100-Continue" header. +_DEFAULT_CONTINUE_TIMEOUT = 2.5 + # Data larger than this will be read in chunks, to prevent extreme # overallocation. _MIN_READ_BUF_SIZE = 1 << 20 @@ -341,7 +351,7 @@ def _read_status(self): raise BadStatusLine(line) return version, status, reason - def begin(self, *, _max_headers=None): + def begin(self, *, support_continue=False, _max_headers=None): if self.headers is not None: # we've already started reading the response return @@ -354,8 +364,13 @@ def begin(self, *, _max_headers=None): # read until we get a non-100 response for _ in range(_MAXINTERIMRESPONSES): version, status, reason = self._read_status() - if status != CONTINUE: + + if support_continue or status != CONTINUE: + # This is the response to return to the caller: either a + # final response (status != CONTINUE), or a 100 Continue + # the caller asked to see. break + # skip the header from the 100 response skipped_headers = _read_headers(self.fp, _max_headers) if self.debuglevel > 0: @@ -929,7 +944,9 @@ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, self.sock = None self._buffer = [] self.__response = None + self.__pending_response = None self.__state = _CS_IDLE + self._continue_timeout = _DEFAULT_CONTINUE_TIMEOUT self._method = None self._tunnel_host = None self._tunnel_port = None @@ -941,9 +958,10 @@ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, self._validate_host(self.host) - # This is stored as an instance variable to allow unit - # tests to replace it with a suitable mockup + # These are stored as instance variables to allow unit + # tests to replace them with a suitable mockup self._create_connection = socket.create_connection + self._select = select.select def set_tunnel(self, host, port=None, headers=None): """Set up host and port for HTTP CONNECT tunnelling. @@ -1089,6 +1107,10 @@ def close(self): self.sock = None sock.close() # close it manually... there may be other refs finally: + pending_response = self.__pending_response + if pending_response: + self.__pending_response = None + pending_response.close() response = self.__response if response: self.__response = None @@ -1149,7 +1171,9 @@ def _read_readable(self, readable): datablock = datablock.encode("iso-8859-1") yield datablock - def _send_output(self, message_body=None, encode_chunked=False): + def _send_output(self, message_body=None, encode_chunked=False, + expect_continue=False, + continue_timeout=_DEFAULT_CONTINUE_TIMEOUT): """Send the currently buffered request and clear the buffer. Appends an extra \\r\\n to the buffer. @@ -1161,6 +1185,34 @@ def _send_output(self, message_body=None, encode_chunked=False): self.send(msg) if message_body is not None: + if expect_continue and not self.__response: + # For a TLS socket, check for already decrypted data in the + # internal buffer of the SSLSocket. + if (ssl is not None and isinstance(self.sock, ssl.SSLSocket) + and self.sock.pending() > 0): + read_ready = True + else: + read_ready, _, _ = self._select([self.sock], [], [], + continue_timeout) + if read_ready: + if self.debuglevel > 0: + response = self.response_class(self.sock, + self.debuglevel, + method=self._method) + else: + response = self.response_class(self.sock, + method=self._method) + if self.max_response_headers is None: + response.begin(support_continue=True) + else: + response.begin( + support_continue=True, + _max_headers=self.max_response_headers) + if response.code != CONTINUE: + # Break without sending the body + self.__pending_response = response + return + response.close() # create a consistent interface to message_body if hasattr(message_body, 'read'): @@ -1204,7 +1256,8 @@ def _send_output(self, message_body=None, encode_chunked=False): self.send(b'0\r\n\r\n') def putrequest(self, method, url, skip_host=False, - skip_accept_encoding=False): + skip_accept_encoding=False, + continue_timeout=_DEFAULT_CONTINUE_TIMEOUT): """Send a request to the server. 'method' specifies an HTTP request method, e.g. 'GET'. @@ -1213,12 +1266,14 @@ def putrequest(self, method, url, skip_host=False, 'skip_accept_encoding' if True does not add automatically an 'Accept-Encoding:' header """ + if continue_timeout <= 0.0: + raise ValueError("continue_timeout must be greater than 0") + self._continue_timeout = continue_timeout # if a prior response has been completed, then forget about it. if self.__response and self.__response.isclosed(): self.__response = None - # in certain cases, we cannot issue another request on this connection. # this occurs when: # 1) we are in the process of sending a request. (_CS_REQ_STARTED) @@ -1398,14 +1453,31 @@ def endheaders(self, message_body=None, *, encode_chunked=False): self.__state = _CS_REQ_SENT else: raise CannotSendHeader() - self._send_output(message_body, encode_chunked=encode_chunked) + + expect_continue = False + if message_body is not None and not self.__response: + prefix = b'expect:' + for line in self._buffer: + if line.lower().startswith(prefix): + value = line[len(prefix):].strip().lower() + expect_continue = value == b'100-continue' + break + + self._send_output(message_body, encode_chunked=encode_chunked, + expect_continue=expect_continue, + continue_timeout=self._continue_timeout) def request(self, method, url, body=None, headers={}, *, - encode_chunked=False): + encode_chunked=False, + continue_timeout=_DEFAULT_CONTINUE_TIMEOUT): """Send a complete request to the server.""" - self._send_request(method, url, body, headers, encode_chunked) + if continue_timeout <= 0.0: + raise ValueError("continue_timeout must be greater than 0") + self._send_request(method, url, body, headers, encode_chunked, + continue_timeout) - def _send_request(self, method, url, body, headers, encode_chunked): + def _send_request(self, method, url, body, headers, encode_chunked, + continue_timeout): # Honor explicitly requested Host: and Accept-Encoding: headers. header_names = frozenset(k.lower() for k in headers) skips = {} @@ -1414,7 +1486,8 @@ def _send_request(self, method, url, body, headers, encode_chunked): if 'accept-encoding' in header_names: skips['skip_accept_encoding'] = 1 - self.putrequest(method, url, **skips) + self.putrequest(method, url, **skips, + continue_timeout=continue_timeout) # chunked encoding will happen if HTTP/1.1 is used and either # the caller passes encode_chunked=True or the following @@ -1451,11 +1524,13 @@ def _send_request(self, method, url, body, headers, encode_chunked): body = _encode(body, 'body') self.endheaders(body, encode_chunked=encode_chunked) - def getresponse(self): + def getresponse(self, support_continue=False): """Get the response from the server. If the HTTPConnection is in the correct state, returns an - instance of HTTPResponse. + instance of HTTPResponse or of whatever object is returned by the + response_class variable. The connection will wait for a response other + than code 100 ('Continue') unless support_continue is set to True. If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP @@ -1486,7 +1561,10 @@ def getresponse(self): if self.__state != _CS_REQ_SENT or self.__response: raise ResponseNotReady(self.__state) - if self.debuglevel > 0: + if self.__pending_response: + response = self.__pending_response + self.__pending_response = None + elif self.debuglevel > 0: response = self.response_class(self.sock, self.debuglevel, method=self._method) else: @@ -1495,32 +1573,32 @@ def getresponse(self): try: try: if self.max_response_headers is None: - response.begin() + response.begin(support_continue=support_continue) else: - response.begin(_max_headers=self.max_response_headers) + response.begin(support_continue=support_continue, + _max_headers=self.max_response_headers) except ConnectionError: self.close() raise assert response.will_close != _UNKNOWN - self.__state = _CS_IDLE + if response.code != 100: + # Code 100 is effectively 'not a response' for this purpose + self.__state = _CS_IDLE - if response.will_close: - # this effectively passes the connection to the response - self.close() - else: - # remember this, so we can tell when it is complete - self.__response = response + if response.will_close: + # this effectively passes the connection to the response + self.close() + else: + # remember this, so we can tell when it is complete + self.__response = response return response except: response.close() raise -try: - import ssl -except ImportError: - pass -else: +if ssl is not None: + class HTTPSConnection(HTTPConnection): "This class allows communication via SSL." diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 57eedee25a89e02..849c44e17aab0a2 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -7,6 +7,7 @@ import pickle import array import re +import select import socket import threading @@ -17,6 +18,7 @@ from test import support from test.support import os_helper from test.support import socket_helper +from test.support import threading_helper support.requires_working_socket(module=True) @@ -126,6 +128,7 @@ def __init__(self, *args): super().__init__('example.com') self.fake_socket_args = args self._create_connection = self.create_connection + self._select = self.fake_select def connect(self): """Count the number of times connect() is invoked""" @@ -135,6 +138,23 @@ def connect(self): def create_connection(self, *pos, **kw): return FakeSocket(*self.fake_socket_args) + @staticmethod + def fake_select(rlist, wlist, xlist, timeout=None): + """Real select.select() won't work on our fake socket""" + # Only needs to support read select for current tests + ret = [] + for r in rlist: + assert isinstance(r, FakeSocket) + if r.data and not hasattr(r, 'file'): + # Has data to read, hasn't called makefile() yet + ret.append(r) + elif getattr(r, 'file', None) and not r.file_closed: + if r.file.read(1): + r.file.seek(r.file.tell() - 1) + ret.append(r) + return ret, [], [] + + class HeaderTests(TestCase): def test_auto_headers(self): # Some headers are added automatically, but should not be added by @@ -2504,6 +2524,260 @@ def test_binary_file_body(self): self.assertEqual(b'5\r\nbody\xc1\r\n0\r\n\r\n', f.read()) +class ExpectContinueTests(TestCase): + + def test_data_sent_on_continue(self): + # Prior versions would pass this too, as they send body immediately + # and ignore the Continue, but make sure it still behaves as expected. + conn = FakeSocketHTTPConnection( + b'HTTP/1.1 100 Continue\r\n' + b'\r\n' + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + conn.request('PUT', '/', headers={'Expect': '100-continue'}, + body=b'Body content') + resp = conn.getresponse() + self.assertEqual(resp.code, 200) + self.assertIn(b'Expect: 100-continue', conn.sock.data) + self.assertIn(b'Body content', conn.sock.data) + + def test_data_not_sent_on_error(self): + conn = FakeSocketHTTPConnection( + b'HTTP/1.1 429 Too Many Requests\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + conn.request('PUT', '/', headers={'Expect': '100-continue'}, + body=b'Body content') + resp = conn.getresponse() + self.assertEqual(resp.code, 429) + self.assertIn(b'Expect: 100-continue', conn.sock.data) + self.assertNotIn(b'Body content', conn.sock.data) + + def test_request_continue_timeout(self): + conn = FakeSocketHTTPConnection( + b'HTTP/1.1 100 Continue\r\n' + b'\r\n' + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + conn._select = mock.Mock(wraps=conn.fake_select) + conn.request('PUT', '/', headers={'Expect': '100-continue'}, + body=b'Body content', continue_timeout=0.25) + conn._select.assert_called_once_with([conn.sock], [], [], 0.25) + self.assertEqual(conn.getresponse().code, 200) + + def test_putrequest_continue_timeout(self): + conn = FakeSocketHTTPConnection( + b'HTTP/1.1 100 Continue\r\n' + b'\r\n' + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + conn._select = mock.Mock(wraps=conn.fake_select) + conn.putrequest('PUT', '/', continue_timeout=0.25) + conn.putheader('Expect', '100-continue') + conn.putheader('Content-Length', '12') + conn.endheaders(b'Body content') + conn._select.assert_called_once_with([conn.sock], [], [], 0.25) + self.assertIn(b'Body content', conn.sock.data) + self.assertEqual(conn.getresponse().code, 200) + + def test_invalid_continue_timeout(self): + conn = FakeSocketHTTPConnection(b'') + for timeout in (0, -1): + with self.subTest(timeout=timeout): + with self.assertRaises(ValueError): + conn.request('PUT', '/', body=b'Body content', + continue_timeout=timeout) + with self.assertRaises(ValueError): + conn.putrequest('PUT', '/', continue_timeout=timeout) + + @unittest.skipUnless(hasattr(client, 'HTTPSConnection'), + 'ssl support required') + def test_https_pending_data_skips_select(self): + # Data already decrypted and buffered in the SSLSocket is not + # visible to select(), so it must be checked with pending(). + class FakeSSLSocket(FakeSocket): + def pending(self): + return len(self.text) + + conn = client.HTTPSConnection('example.com') + conn.sock = FakeSSLSocket( + b'HTTP/1.1 417 Expectation Failed\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + conn._select = mock.Mock(side_effect=AssertionError('select called')) + with mock.patch.object(client, 'ssl', + mock.Mock(SSLSocket=FakeSSLSocket)): + conn.request('PUT', '/', headers={'Expect': '100-continue'}, + body=b'Body content') + conn._select.assert_not_called() + self.assertEqual(conn.getresponse().code, 417) + self.assertNotIn(b'Body content', conn.sock.data) + + @unittest.skipUnless(hasattr(client, 'HTTPSConnection'), + 'ssl support required') + def test_https_no_pending_data_uses_select(self): + class FakeSSLSocket(FakeSocket): + def pending(self): + return 0 + + conn = client.HTTPSConnection('example.com') + conn.sock = FakeSSLSocket( + b'HTTP/1.1 100 Continue\r\n' + b'\r\n' + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + conn._select = mock.Mock(wraps=FakeSocketHTTPConnection.fake_select) + with mock.patch.object(client, 'ssl', + mock.Mock(SSLSocket=FakeSSLSocket)): + conn.request('PUT', '/', headers={'Expect': '100-continue'}, + body=b'Body content', continue_timeout=0.25) + conn._select.assert_called_once_with([conn.sock], [], [], 0.25) + self.assertEqual(conn.getresponse().code, 200) + self.assertIn(b'Body content', conn.sock.data) + + def test_pending_response_kept_on_invalid_putrequest(self): + # A real socket is needed: FakeSocket.makefile() returns the whole + # response again, which would hide a lost early response. + def server_thread(sock): + conn, _ = sock.accept() + with conn: + conn.recv(4096) + conn.sendall( + b'HTTP/1.1 417 Expectation Failed\r\n' + b'Content-Length: 4\r\n' + b'\r\n' + b'nope' + ) + # Keep the connection open until the client is done. + conn.recv(4096) + + with socket.socket() as sock: + sock.bind(('localhost', 0)) + sock.listen(1) + t = threading.Thread(target=server_thread, args=(sock,)) + t.start() + conn = client.HTTPConnection('localhost', sock.getsockname()[1], + timeout=support.SHORT_TIMEOUT) + try: + conn.request('PUT', '/', + headers={'Expect': '100-continue'}, + body=b'Body content') + with self.assertRaises(client.CannotSendRequest): + conn.putrequest('GET', '/') + with conn.getresponse() as resp: + self.assertEqual(resp.status, 417) + self.assertEqual(resp.read(), b'nope') + finally: + conn.close() + t.join() + + def test_body_sent_on_timeout(self): + def client_thread(port): + conn = client.HTTPConnection('localhost', port) + conn.request('PUT', '/', + headers={'Expect': '100-continue'}, + body=b'Body content', continue_timeout=0.75) + resp = conn.getresponse() + self.assertEqual(resp.code, 200) + resp.close() + conn.close() + + with threading_helper.catch_threading_exception() as cm: + with socket.socket() as sock: + sock.bind(('localhost', 0)) + sock.listen(1) + t = threading.Thread(target=client_thread, + args=(sock.getsockname()[1],)) + t.start() + conn, _ = sock.accept() + req_data = conn.recv(4096) + self.assertTrue(req_data.startswith(b'PUT / HTTP/1.1\r\n')) + self.assertIn(b'Expect: 100-continue\r\n', req_data) + self.assertIn(b'Content-Length: 12\r\n', req_data) + # Client should not send body data yet + self.assertNotIn(b'Body content', req_data) + + # Server does not respond to the continue request with 100 Continue + # so the client will time out and send the body data + # Wait for 0.5 seconds to see if the client sends the body data + rr, _, _ = select.select([conn], [], [], 0.5) + self.assertEqual(rr, []) + # Client should time out and send body data in another ~0.25s + rr, _, _ = select.select([conn], [], [], 0.5) + self.assertEqual(rr, [conn]) + body = conn.recv(4096) + self.assertEqual(body, b'Body content') + conn.sendall( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + conn.close() + sock.close() + t.join() + if cm.exc_value is not None: + raise cm.exc_value + + def test_manual_getresponse(self): + def client_thread(port): + conn = client.HTTPConnection('localhost', port) + conn.putrequest("PUT", "/file", continue_timeout=0.1) + conn.putheader('Expect', '100-Continue') + conn.putheader('Content-Length', '42') + conn.endheaders() + with conn.getresponse(support_continue=True) as resp: + self.assertEqual(resp.status, 100) + conn.send(b'Go away or I shall taunt you a second time') + with conn.getresponse() as resp: + self.assertEqual(resp.status, 200) + conn.close() + + with threading_helper.catch_threading_exception() as cm: + with socket.socket() as sock: + sock.bind(('localhost', 0)) + sock.listen(1) + t = threading.Thread(target=client_thread, + args=(sock.getsockname()[1],)) + t.start() + conn, _ = sock.accept() + req_data = conn.recv(4096) + self.assertTrue(req_data.startswith(b'PUT /file HTTP/1.1\r\n')) + self.assertIn(b'Expect: 100-Continue\r\n', req_data) + self.assertIn(b'Content-Length: 42\r\n', req_data) + self.assertNotIn(b'I shall taunt you', req_data) + # Client is not expected to send body data until we respond, + # regardless of continue_timeout + rr, _, _ = select.select([conn], [], [], 1.0) + self.assertEqual(rr, []) + conn.sendall(b'HTTP/1.1 100 Continue\r\n\r\n') + rr, _, _ = select.select([conn], [], [], 0.5) + self.assertEqual(rr, [conn]) + b = conn.recv(42) + self.assertEqual( + b, b'Go away or I shall taunt you a second time') + conn.sendall( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + conn.close() + sock.close() + t.join() + if cm.exc_value is not None: + raise cm.exc_value + + + class HTTPResponseTest(TestCase): def setUp(self): diff --git a/Misc/ACKS b/Misc/ACKS index 4178ae3108128aa..c3ecfc1e315a595 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -117,6 +117,7 @@ Cesar Eduardo Barros Des Barry Emanuel Barry Ulf Bartelt +Tim Bartlett Campbell Barton Don Bashford Pior Bastida @@ -1270,6 +1271,7 @@ Rihaan Meher Bill van Melle Lucas Prado Melo Ezio Melotti +Ashwin Menon Doug Mennella Dimitri Merejkowsky Brian Merrell diff --git a/Misc/NEWS.d/next/Library/2026-09-27-13-04-02.gh-issue-42550.sqYlvB.rst b/Misc/NEWS.d/next/Library/2026-09-27-13-04-02.gh-issue-42550.sqYlvB.rst new file mode 100644 index 000000000000000..af8fe2630ba440f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-27-13-04-02.gh-issue-42550.sqYlvB.rst @@ -0,0 +1,3 @@ +:class:`http.client.HTTPConnection` now waits for a ``100 Continue`` +response before sending body content if the ``Expect: 100-Continue`` header +is set. From dc192869b67e5fef086970dc82cbdb0caeb32a1a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Nov 2026 16:23:42 +0900 Subject: [PATCH 100/229] gh-140456: Doc: Clarify traceback source lines and tuple inputs (#161801) --- Doc/library/traceback.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 8f167eb3524db21..9e1be62c568ea14 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -157,16 +157,19 @@ Module-Level Functions .. function:: print_list(extracted_list, file=None) - Print the list of tuples as returned by :func:`extract_tb` or - :func:`extract_stack` as a formatted stack trace to the given file. + Print a list of :class:`FrameSummary` objects as returned by + :func:`extract_tb` or :func:`extract_stack`, or a list of old-style tuples + (see :meth:`StackSummary.from_list`), as a formatted stack trace to the + given file. If *file* is ``None``, the output is written to :data:`sys.stderr`. .. function:: format_list(extracted_list) - Given a list of tuples or :class:`FrameSummary` objects as returned by - :func:`extract_tb` or :func:`extract_stack`, return a list of strings ready - for printing. Each string in the resulting list corresponds to the item with + Given a list of :class:`FrameSummary` objects as returned by + :func:`extract_tb` or :func:`extract_stack`, or a list of old-style tuples + (see :meth:`StackSummary.from_list`), return a list of strings ready for + printing. Each string in the resulting list corresponds to the item with the same index in the argument list. Each string ends in a newline; the strings may contain internal newlines as well, for those items whose source text line is not ``None``. @@ -536,8 +539,8 @@ in a :ref:`traceback `. .. attribute:: FrameSummary.line - A string representing the source code for this frame, with leading and - trailing whitespace stripped. + The source line at :attr:`~FrameSummary.lineno`, + with leading and trailing whitespace stripped. If the source is not available, it is ``None``. .. attribute:: FrameSummary.end_lineno From 10862056f51eed3a332be3eefd51d4d6e871dade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Thu, 12 Nov 2026 15:45:22 +0800 Subject: [PATCH 101/229] gh-127172: Honor umask and default ACL when installing venv activation scripts (GH-154850) venv wrote the activation scripts with a fixed mode copied from the packaged templates, so they ignored the process umask and any default ACL, unlike every other file venv creates. Copy only the data and let the new files inherit the umask and ACL. --- Lib/test/test_venv.py | 21 +++++++++++++++++++ Lib/venv/__init__.py | 3 +-- ...-07-29-12-20-39.gh-issue-127172.Kf3Xa9.rst | 4 ++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-29-12-20-39.gh-issue-127172.Kf3Xa9.rst diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 43c2a552ee90b7f..bcb782a8d10c3ae 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -628,6 +628,27 @@ def test_install_scripts_selinux(self): venv.create(self.env_dir) listxattr_mock.assert_not_called() + @unittest.skipIf(os.name == 'nt', 'POSIX-specific permission test') + def test_install_scripts_honors_umask(self): + """ + gh-127172: Activation scripts must inherit the process umask (and any + default ACL) like every other created file, rather than a fixed mode + copied from the source template. + """ + old_umask = os.umask(0o002) + try: + venv.create(self.env_dir, with_pip=False) + bindir = os.path.join(self.env_dir, self.bindir) + control = os.path.join(bindir, 'control') + with open(control, 'w'): + pass + expected = os.stat(control).st_mode & 0o777 + for name in ('activate', 'activate.csh', 'activate.fish'): + mode = os.stat(os.path.join(bindir, name)).st_mode & 0o777 + self.assertEqual(mode, expected) + finally: + os.umask(old_umask) + def test_overwrite_existing(self): """ Test creating environment in an existing directory. diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 60fe070b1f9f4b1..0e9ee18892b9338 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -607,11 +607,10 @@ def skip_file(f): 'may be binary: %s', srcfile, e) continue if new_data == data: - shutil.copy(srcfile, dstfile) + shutil.copyfile(srcfile, dstfile) else: with open(dstfile, 'wb') as f: f.write(new_data) - shutil.copymode(srcfile, dstfile) def upgrade_dependencies(self, context): logger.debug( diff --git a/Misc/NEWS.d/next/Library/2026-07-29-12-20-39.gh-issue-127172.Kf3Xa9.rst b/Misc/NEWS.d/next/Library/2026-07-29-12-20-39.gh-issue-127172.Kf3Xa9.rst new file mode 100644 index 000000000000000..c4bf1f07a2ea577 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-29-12-20-39.gh-issue-127172.Kf3Xa9.rst @@ -0,0 +1,4 @@ +Creating a virtual environment with :mod:`venv` now writes the activation +scripts using permissions derived from the process umask (and any default +ACL), consistent with the other files it creates, instead of a fixed mode +copied from the packaged templates. Patch by tonghuaroot. From 9f386367333f7845095d764fad3ad9b397d04672 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 12 Nov 2026 17:39:32 +0900 Subject: [PATCH 102/229] Doc: Correct FAQ explanations (#161663) Doc: Correct FAQ explanations and extension debugging instructions --- Doc/faq/design.rst | 2 +- Doc/faq/extending.rst | 25 ++++++++++--------------- Doc/faq/programming.rst | 33 ++++----------------------------- 3 files changed, 15 insertions(+), 45 deletions(-) diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 3a3e16b3bcf3db4..ca5fcc154f73127 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -401,7 +401,7 @@ library that are impossible to free (e.g. a tool like Purify will complain about these). Python is, however, aggressive about cleaning up memory on exit and does try to destroy every single object. -If you want to force Python to delete certain things on deallocation use the +If you want to force Python to delete certain things when Python exits, use the :mod:`atexit` module to run a function that will force those deletions. diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index e0e6ddd04af76df..555c312b9137690 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -209,25 +209,20 @@ minor that it doesn't seem worth the effort.) How do I debug an extension? ---------------------------- -When using GDB with dynamically loaded extensions, you can't set a breakpoint in -your extension until your extension is loaded. - -In your ``.gdbinit`` file (or interactively), add the command: - -.. code-block:: none - - br _PyImport_LoadDynamicModule - -Then, when you run GDB: +When using GDB with dynamically loaded extensions, enable pending breakpoints +to set a breakpoint before the extension is loaded. For example, to stop in +a function named ``myfunction``: .. code-block:: shell-session $ gdb /local/bin/python - gdb) run myscript.py - gdb) continue # repeat until your extension is loaded - gdb) finish # so that your extension is loaded - gdb) br myfunction.c:50 - gdb) continue + (gdb) set breakpoint pending on + (gdb) break myfunction + (gdb) run myscript.py + +GDB resolves the breakpoint when the extension is loaded and stops when the +function is called. You can also set a pending breakpoint directly in your +extension's source code, for example with ``break myfunction.c:50``. I want to compile a Python module on my Linux system, but some files are missing. Why? -------------------------------------------------------------------------------------- diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 5eb8dff1ee8052b..fe2e8587eb201b3 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1198,8 +1198,7 @@ Use the :func:`reversed` built-in function:: for x in reversed(sequence): ... # do something with x ... -This won't touch your original sequence, but build a new copy with reversed -order to iterate over. +This returns a reverse iterator without modifying your original sequence. How do you remove duplicates from a list? @@ -1642,8 +1641,8 @@ Use the built-in :func:`super` function:: In the example, :func:`super` will automatically determine the instance from which it was called (the ``self`` value), look up the :term:`method resolution -order` (MRO) with ``type(self).__mro__``, and return the next in line after -``Derived`` in the MRO: ``Base``. +order` (MRO) with ``type(self).__mro__``, and return a proxy that searches for +methods starting with the next class after ``Derived`` in the MRO: ``Base``. How can I organize my code to make it easier to change the base class? @@ -1686,7 +1685,7 @@ C)`` holds, unless overridden by ``c`` itself or by some class on the base-class search path from ``c.__class__`` back to ``C``. Caution: within a method of C, an assignment like ``self.count = 42`` creates a -new and unrelated instance named "count" in ``self``'s own dict. Rebinding of a +new and unrelated attribute named "count" in ``self``'s own dict. Rebinding of a class-static data name must always specify the class whether inside a method or not:: @@ -2055,30 +2054,6 @@ relevant instance attributes are mutable, the ``cached_property`` approach can't be made to work because it cannot detect changes to the attributes. -To make the ``lru_cache`` approach work when the *station_id* is mutable, -the class needs to define the :meth:`~object.__eq__` and :meth:`~object.__hash__` -methods so that the cache can detect relevant attribute updates:: - - class Weather: - "Example with a mutable station identifier" - - def __init__(self, station_id): - self.station_id = station_id - - def change_station(self, station_id): - self.station_id = station_id - - def __eq__(self, other): - return self.station_id == other.station_id - - def __hash__(self): - return hash(self.station_id) - - @lru_cache(maxsize=20) - def historic_rainfall(self, date, units='cm'): - 'Rainfall on a given date' - # Depends on the station_id, date, and units. - Modules ======= From 72ba9549898f219e70a0dd2ec848a092d6e4a9c9 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 12 Nov 2026 13:07:48 +0300 Subject: [PATCH 103/229] gh-161617: Fix `test_clinic`, `test_tools`, `test_generated_cases` with lazy imports (#161618) --- Lib/test/lazy_imports_all_exclude.txt | 3 --- Lib/test/support/import_helper.py | 14 ++++++++++++++ Lib/test/test_tools/__init__.py | 5 ++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/Lib/test/lazy_imports_all_exclude.txt b/Lib/test/lazy_imports_all_exclude.txt index 864d723fe56b11a..e5519323e88e38a 100644 --- a/Lib/test/lazy_imports_all_exclude.txt +++ b/Lib/test/lazy_imports_all_exclude.txt @@ -19,10 +19,8 @@ test.test_inspect.test_inspect test___all__ test__interpreters -test_clinic test_crossinterp test_datetime -test_generated_cases test_import test_importlib test_json @@ -32,6 +30,5 @@ test_profiling test_pyrepl test_subprocess test_symtable -test_tools test_type_annotations test_unittest diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index e8a3d176ad6943f..962f65d8dcd14a2 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -119,6 +119,20 @@ def frozen_modules(enabled=True): _imp._override_frozen_modules_for_tests(0) +@contextlib.contextmanager +def force_eager_imports(): + """Force eager imports inside the context manager.""" + original_lazy_imports_filter = sys.get_lazy_imports_filter() + sys.set_lazy_imports_filter( + # Tools must be imported eagerly under this context manager: + lambda importing, imported, fromlist: False + ) + try: + yield + finally: + sys.set_lazy_imports_filter(original_lazy_imports_filter) + + @contextlib.contextmanager def multi_interp_extensions_check(enabled=True): """Force legacy modules to be allowed in subinterpreters (or not). diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py index c4395c7c0ad0c98..310bf48c2ef03a1 100644 --- a/Lib/test/test_tools/__init__.py +++ b/Lib/test/test_tools/__init__.py @@ -32,7 +32,10 @@ def skip_if_missing(tool=None): @contextlib.contextmanager def imports_under_tool(name, *subdirs): tooldir = os.path.join(toolsdir, name, *subdirs) - with import_helper.DirsOnSysPath(tooldir) as cm: + with ( + import_helper.force_eager_imports(), + import_helper.DirsOnSysPath(tooldir) as cm, + ): yield cm def import_tool(toolname): From 2d862b1d4f0b3fea1dd7c11a6992bcbc1629b5ef Mon Sep 17 00:00:00 2001 From: Sriraj <168966222+sricursion@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:46:04 +0530 Subject: [PATCH 104/229] gh-158978: Fix cloning of DOM Level 1 nodes in xml.dom.minidom (GH-158987) Node.cloneNode() and Document.importNode() raised NamespaceErr for nodes without a namespace whose name contains a colon, for example created with createElement("svg:rect"). Such nodes are now cloned with the DOM Level 1 factory methods, which keeps also their prefix. --- Lib/test/test_minidom.py | 72 ++++++++++++++++++++++++++++++++++++++++ Lib/xml/dom/minidom.py | 24 ++++++++++---- 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 0c418587afe8ed2..8262f8305f7abb0 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -6,6 +6,7 @@ from test import support import unittest +import xml.dom.expatbuilder import xml.dom.minidom from xml.dom.minidom import parse, Attr, Node, Document, Element, parseString @@ -1277,6 +1278,52 @@ def testCloneElementDeep(self): self.assertEqual(clone.toxml(), '') dom.unlink() + def create_dom1_element(self): + # DOM Level 1 nodes have no namespace, but their names can contain + # a colon. + doc = Document() + root = doc.appendChild(doc.createElement("root")) + root.setAttribute("xmlns", "http://xml.python.org/default") + root.setAttribute("xmlns:xsi", "http://xml.python.org/xsi") + root.setAttribute("xsi:type", "t") + root.setAttributeNS("http://xml.python.org/ns", "p:a", "v") + root.appendChild(doc.createElement("svg:rect")) + root.appendChild( + doc.createElementNS("http://xml.python.org/ns", "p:b")) + return doc, root + + def check_clone_names(self, clone, node): + for name in ("nodeName", "prefix", "localName", "namespaceURI"): + self.assertEqual(getattr(clone, name), getattr(node, name), name) + + def check_clone_element(self, clone, elem, doc): + self.assertIs(clone.ownerDocument, doc) + self.assertIsNone(clone.parentNode) + self.check_clone_names(clone, elem) + self.assertEqual(len(clone.attributes), len(elem.attributes)) + for attr in elem.attributes.values(): + with self.subTest(attr=attr.name): + a = clone.getAttributeNode(attr.name) + self.check_clone_names(a, attr) + self.assertEqual(a.value, attr.value) + self.assertIs(a.ownerElement, clone) + self.assertEqual(len(clone.childNodes), len(elem.childNodes)) + for c, child in zip(clone.childNodes, elem.childNodes): + self.check_clone_names(c, child) + + def testCloneElementDOM1Names(self): + doc, root = self.create_dom1_element() + self.check_clone_element(root.cloneNode(True), root, doc) + doc.unlink() + + def testCloneElementNoNamespaces(self): + doc = xml.dom.expatbuilder.parseString( + "", + namespaces=False) + root = doc.documentElement + self.check_clone_element(root.cloneNode(True), root, doc) + doc.unlink() + def testCloneDocumentShallow(self): doc = parseString("\n" "" @@ -1395,6 +1442,13 @@ def testImportDocumentTypeDeep(self): self.assertRaises(xml.dom.NotSupportedErr, target.importNode, src.doctype, 1) + def testImportElementDOM1Names(self): + doc1, root = self.create_dom1_element() + doc2 = Document() + self.check_clone_element(doc2.importNode(root, True), root, doc2) + doc1.unlink() + doc2.unlink() + # Testing attribute clones uses a helper, and should always be deep, # even if the argument to cloneNode is false. def check_clone_attribute(self, deep, testName): @@ -1417,6 +1471,24 @@ def testCloneAttributeShallow(self): def testCloneAttributeDeep(self): self.check_clone_attribute(1, "testCloneAttributeDeep") + def testCloneAttributeDOM1Names(self): + doc1 = Document() + attr = doc1.createAttribute("a:b") + attr.value = "v" + doc2 = Document() + for method, clone, owner in [ + ("cloneNode", attr.cloneNode(True), doc1), + ("importNode", doc2.importNode(attr, True), doc2), + ]: + with self.subTest(method=method): + self.check_clone_names(clone, attr) + self.assertEqual(clone.value, "v") + self.assertTrue(clone.specified) + self.assertIsNone(clone.ownerElement) + self.assertIs(clone.ownerDocument, owner) + doc1.unlink() + doc2.unlink() + def check_clone_pi(self, deep, testName): doc = parseString("") pi = doc.firstChild diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index f83cd2898daa3e4..3ccc5761115117c 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -2170,11 +2170,20 @@ def _clone_node(node, deep, newOwnerDocument): else: operation = xml.dom.UserDataHandler.NODE_IMPORTED if node.nodeType == Node.ELEMENT_NODE: - clone = newOwnerDocument.createElementNS(node.namespaceURI, - node.nodeName) + if node.namespaceURI is None: + clone = newOwnerDocument.createElement(node.nodeName) + else: + clone = newOwnerDocument.createElementNS(node.namespaceURI, + node.nodeName) for attr in node.attributes.values(): - clone.setAttributeNS(attr.namespaceURI, attr.nodeName, attr.value) - a = clone.getAttributeNodeNS(attr.namespaceURI, attr.localName) + if attr.namespaceURI is None: + clone.setAttribute(attr.nodeName, attr.value) + a = clone.getAttributeNode(attr.nodeName) + else: + clone.setAttributeNS(attr.namespaceURI, attr.nodeName, + attr.value) + a = clone.getAttributeNodeNS(attr.namespaceURI, + attr.localName) a.specified = attr.specified if deep: @@ -2201,8 +2210,11 @@ def _clone_node(node, deep, newOwnerDocument): elif node.nodeType == Node.ENTITY_REFERENCE_NODE: clone = newOwnerDocument.createEntityReference(node.nodeName) elif node.nodeType == Node.ATTRIBUTE_NODE: - clone = newOwnerDocument.createAttributeNS(node.namespaceURI, - node.nodeName) + if node.namespaceURI is None: + clone = newOwnerDocument.createAttribute(node.nodeName) + else: + clone = newOwnerDocument.createAttributeNS(node.namespaceURI, + node.nodeName) clone.specified = True clone.value = node.value elif node.nodeType == Node.DOCUMENT_TYPE_NODE: From f8714494a9d54910b5142e0b3fe6eb5875e8c09d Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 12 Nov 2026 04:07:57 -0700 Subject: [PATCH 105/229] gh-160397: Don't take the warnings lock when emitting a warning (#160401) --- Include/internal/pycore_interp_structs.h | 2 +- Lib/test/test_warnings/__init__.py | 24 ++ ...0-22-21-19-51.gh-issue-160397.E-RwgUgm.rst | 5 + Python/_warnings.c | 327 ++++++++++++------ 4 files changed, 247 insertions(+), 111 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-21-19-51.gh-issue-160397.E-RwgUgm.rst diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index a9ad61f458944ab..c9002081306eb83 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -654,7 +654,7 @@ struct _warnings_runtime_state { // Whether the thread calling fork() owns the warnings lock. bool lock_held_at_fork; #endif - long filters_version; + Py_ssize_t filters_version; PyObject *context; }; diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index bfb43025de11405..c069b81eb91c2ef 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -2020,6 +2020,7 @@ class ThreadTests(BaseTest): def setUp(self): super().setUp() + self.enterContext(self.module.catch_warnings()) self.module.resetwarnings() @unittest.skipIf(not ENABLE_THREAD_TESTS, @@ -2069,6 +2070,29 @@ def run_threads(): run_threads() + @threading_helper.requires_working_threading() + def test_lock_not_held_while_showing(self): + # showwarning() must not run with the warnings lock held, otherwise + # another thread that needs the lock would block. + import threading + + def other_thread(): + self.module.filterwarnings("always", category=TestWarning) + + def showwarning(*args, **kwargs): + t = threading.Thread(target=other_thread) + t.start() + t.join(timeout=support.SHORT_TIMEOUT) + alive.append(t.is_alive()) + + alive = [] + with self.module.catch_warnings(): + self.module.simplefilter("always") + self.module.showwarning = showwarning + self.module.warn("test", TestWarning) + self.module.warn_explicit("test", TestWarning, "file.py", 1) + self.assertEqual(alive, [False, False]) + class CThreadTests(ThreadTests, unittest.TestCase): module = c_warnings diff --git a/Misc/NEWS.d/next/Library/2026-10-22-21-19-51.gh-issue-160397.E-RwgUgm.rst b/Misc/NEWS.d/next/Library/2026-10-22-21-19-51.gh-issue-160397.E-RwgUgm.rst new file mode 100644 index 000000000000000..bd880cfe1eaff32 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-21-19-51.gh-issue-160397.E-RwgUgm.rst @@ -0,0 +1,5 @@ +The C implementation of :mod:`warnings` no longer takes the warnings lock +when emitting a warning, so user code such as a custom +:func:`~warnings.showwarning` no longer runs with the lock held. On the +:term:`free-threaded build`, the filter search and the registry updates use +per-object critical sections instead. diff --git a/Python/_warnings.c b/Python/_warnings.c index 86eae5deb859691..c6a2441ec937cfa 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1,9 +1,12 @@ #include "Python.h" +#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_frame.h" // PyFrameObject #include "pycore_genobject.h" // PyAsyncGenObject #include "pycore_import.h" // _PyImport_GetModules() #include "pycore_interpframe.h" // _PyFrame_GetCode() #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_object.h" // _Py_XGetRef() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_traceback.h" // _Py_DisplaySourceLine() @@ -142,6 +145,7 @@ _PyWarnings_InitState(PyInterpreterState *interp) if (st->filters == NULL) { return -1; } + PyUnstable_EnableTryIncRef(st->filters); } if (st->once_registry == NULL) { @@ -149,6 +153,7 @@ _PyWarnings_InitState(PyInterpreterState *interp) if (st->once_registry == NULL) { return -1; } + PyUnstable_EnableTryIncRef(st->once_registry); } if (st->default_action == NULL) { @@ -156,6 +161,7 @@ _PyWarnings_InitState(PyInterpreterState *interp) if (st->default_action == NULL) { return -1; } + PyUnstable_EnableTryIncRef(st->default_action); } if (st->context == NULL) { @@ -333,7 +339,30 @@ get_warnings_context_filters(PyInterpreterState *interp) return context_filters; } -// Returns a borrowed reference to the list. +/* Store 'value' in the warnings state field '*ptr'. */ +static void +update_state_ref(PyObject **ptr, PyObject *value) +{ + if (FT_ATOMIC_LOAD_PTR_ACQUIRE(*ptr) == value) { + return; + } + PyUnstable_EnableTryIncRef(value); + PyObject *old = _Py_atomic_exchange_ptr(ptr, Py_NewRef(value)); + Py_XDECREF(old); +} + +/* Return a new reference to the warnings state field '*ptr'. */ +static PyObject * +get_state_ref(PyObject **ptr) +{ +#ifdef Py_GIL_DISABLED + return _Py_XGetRef(ptr); +#else + return Py_XNewRef(*ptr); +#endif +} + +// Returns a new reference to the list. static PyObject * get_warnings_filters(PyInterpreterState *interp) { @@ -342,15 +371,17 @@ get_warnings_filters(PyInterpreterState *interp) if (warnings_filters == NULL) { if (PyErr_Occurred()) return NULL; + warnings_filters = get_state_ref(&st->filters); } else { - Py_SETREF(st->filters, warnings_filters); + update_state_ref(&st->filters, warnings_filters); } - PyObject *filters = st->filters; + PyObject *filters = warnings_filters; if (filters == NULL || !PyList_Check(filters)) { PyErr_SetString(PyExc_ValueError, MODULE_NAME ".filters must be a list"); + Py_XDECREF(filters); return NULL; } return filters; @@ -393,20 +424,20 @@ warnings_release_lock_impl(PyObject *module) Py_RETURN_NONE; } +// Returns a new reference. static PyObject * get_once_registry(PyInterpreterState *interp) { WarningsState *st = warnings_get_state(interp); assert(st != NULL); - assert(warnings_lock_held(st)); - PyObject *registry = GET_WARNINGS_ATTR(interp, onceregistry, 0); if (registry == NULL) { if (PyErr_Occurred()) return NULL; - assert(st->once_registry); - return st->once_registry; + registry = get_state_ref(&st->once_registry); + assert(registry != NULL); + return registry; } if (!PyDict_Check(registry)) { PyErr_Format(PyExc_TypeError, @@ -416,26 +447,26 @@ get_once_registry(PyInterpreterState *interp) Py_DECREF(registry); return NULL; } - Py_SETREF(st->once_registry, registry); + update_state_ref(&st->once_registry, registry); return registry; } +// Returns a new reference. static PyObject * get_default_action(PyInterpreterState *interp) { WarningsState *st = warnings_get_state(interp); assert(st != NULL); - assert(warnings_lock_held(st)); - PyObject *default_action = GET_WARNINGS_ATTR(interp, defaultaction, 0); if (default_action == NULL) { if (PyErr_Occurred()) { return NULL; } - assert(st->default_action); - return st->default_action; + default_action = get_state_ref(&st->default_action); + assert(default_action != NULL); + return default_action; } if (!PyUnicode_Check(default_action)) { PyErr_Format(PyExc_TypeError, @@ -445,7 +476,7 @@ get_default_action(PyInterpreterState *interp) Py_DECREF(default_action); return NULL; } - Py_SETREF(st->default_action, default_action); + update_state_ref(&st->default_action, default_action); return default_action; } @@ -531,18 +562,12 @@ filter_search(PyInterpreterState *interp, PyObject *category, return result; } -/* The item is a new reference. */ +/* The returned action and the item are new references. */ static PyObject* get_filter(PyInterpreterState *interp, PyObject *category, PyObject *text, Py_ssize_t lineno, PyObject *module, PyObject *filename, PyObject **item) { -#ifdef Py_DEBUG - WarningsState *st = warnings_get_state(interp); - assert(st != NULL); - assert(warnings_lock_held(st)); -#endif - /* check _warning_context _filters list */ PyObject *context_filters = get_warnings_context_filters(interp); if (context_filters == NULL) { @@ -560,7 +585,7 @@ get_filter(PyInterpreterState *interp, PyObject *category, } Py_DECREF(context_filters); if (context_action != NULL) { - return context_action; + return Py_NewRef(context_action); } } @@ -572,12 +597,14 @@ get_filter(PyInterpreterState *interp, PyObject *category, if (filters == NULL) { return NULL; } - if (!filter_search(interp, category, text, lineno, module, filename, "filters", - filters, item, &action)) { + bool ok = filter_search(interp, category, text, lineno, module, + filename, "filters", filters, item, &action); + Py_DECREF(filters); + if (!ok) { return NULL; } if (action != NULL) { - return action; + return Py_NewRef(action); } } @@ -591,72 +618,143 @@ get_filter(PyInterpreterState *interp, PyObject *category, } +/* Returns 1 if 'registry' is stamped with 'version', 0 if not, -1 on error. */ static int -already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, - int should_set) +registry_version_matches(PyObject *registry, Py_ssize_t version) { - PyObject *already_warned; + PyObject *version_obj; + if (PyDict_GetItemRef(registry, &_Py_ID(version), &version_obj) < 0) { + return -1; + } + int rc = 0; + if (version_obj != NULL && PyLong_CheckExact(version_obj)) { + int overflow; + long long v = PyLong_AsLongLongAndOverflow(version_obj, &overflow); + rc = (!overflow && v == version); + } + Py_XDECREF(version_obj); + return rc; +} - if (key == NULL) +static int +set_registry_version(PyObject *registry, Py_ssize_t version) +{ + PyObject *version_obj = PyLong_FromSsize_t(version); + if (version_obj == NULL) { return -1; + } + int rc = PyDict_SetItem(registry, &_Py_ID(version), version_obj); + Py_DECREF(version_obj); + return rc; +} +/* Make 'registry' usable for filters 'version', clearing it if it is stamped + with a different version. Returns 1 if it can be used, 0 if 'version' is + no longer current, -1 on error. */ +static int +check_registry_version(PyInterpreterState *interp, PyObject *registry, + Py_ssize_t version) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(registry); WarningsState *st = warnings_get_state(interp); - assert(st != NULL); - assert(warnings_lock_held(st)); - PyObject *version_obj; - if (PyDict_GetItemRef(registry, &_Py_ID(version), &version_obj) < 0) { + int rc = registry_version_matches(registry, version); + if (rc != 0) { + return rc; + } + if (version != FT_ATOMIC_LOAD_SSIZE_ACQUIRE(st->filters_version)) { + // The caller's filter search is stale, don't record its result. + return 0; + } + + PyDict_Clear(registry); + return set_registry_version(registry, version) < 0 ? -1 : 1; +} + +static int +registry_contains(PyObject *registry, PyObject *key) +{ + PyObject *value; + if (PyDict_GetItemRef(registry, key, &value) < 0) { return -1; } - bool should_update_version = ( - version_obj == NULL - || !PyLong_CheckExact(version_obj) - || PyLong_AsLong(version_obj) != st->filters_version - ); - Py_XDECREF(version_obj); - if (should_update_version) { - PyDict_Clear(registry); - version_obj = PyLong_FromLong(st->filters_version); - if (version_obj == NULL) - return -1; - if (PyDict_SetItem(registry, &_Py_ID(version), version_obj) < 0) { - Py_DECREF(version_obj); - return -1; - } - Py_DECREF(version_obj); + if (value == NULL) { + return 0; } - else { - if (PyDict_GetItemRef(registry, key, &already_warned) < 0) { - return -1; - } - if (already_warned != NULL) { - int rc = PyObject_IsTrue(already_warned); - Py_DECREF(already_warned); - if (rc != 0) - return rc; + int rc = PyObject_IsTrue(value); + Py_DECREF(value); + return rc; +} + +/* Returns 1 if 'key' is recorded in 'registry' for the current filters version, + 0 if not, -1 on error. Stores the version checked in '*pversion'. */ +static int +already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, + Py_ssize_t *pversion) +{ + int rc; + Py_BEGIN_CRITICAL_SECTION(registry); + WarningsState *st = warnings_get_state(interp); + Py_ssize_t version = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(st->filters_version); + *pversion = version; + rc = registry_version_matches(registry, version); + if (rc > 0) { + rc = registry_contains(registry, key); + } + else if (rc == 0) { + /* Stamped with an old version, reset it. 'key' can't be present. */ + PyDict_Clear(registry); + if (set_registry_version(registry, version) < 0) { + rc = -1; } } + Py_END_CRITICAL_SECTION(); + return rc; +} - /* This warning wasn't found in the registry, set it. */ - if (should_set) - return PyDict_SetItem(registry, key, Py_True); - return 0; +/* Returns 1 if 'key' was already in 'registry', 0 if it was added, -1 on + error. */ +static int +registry_add(PyObject *registry, PyObject *key) +{ + int rc = registry_contains(registry, key); + if (rc == 0 && PyDict_SetItem(registry, key, Py_True) < 0) { + rc = -1; + } + return rc; } +/* Record 'key' and 'altkey' (either may be NULL) in 'registry' for filters + 'version'. Returns 0 if the warning should be shown, 1 if it was already + recorded, -1 on error. */ static int -update_registry(PyInterpreterState *interp, PyObject *registry, PyObject *text, - PyObject *category, int add_zero) +record_warning(PyInterpreterState *interp, PyObject *registry, + PyObject *key, PyObject *altkey, Py_ssize_t version) { - PyObject *altkey; int rc; - - if (add_zero) - altkey = PyTuple_Pack(3, text, category, _PyLong_GetZero()); - else - altkey = _PyTuple_FromPair(text, category); - - rc = already_warned(interp, registry, altkey, 1); - Py_XDECREF(altkey); + Py_BEGIN_CRITICAL_SECTION(registry); + rc = check_registry_version(interp, registry, version); + if (rc > 0) { + rc = 0; + if (key != NULL) { + rc = registry_add(registry, key); + } + if (rc == 0 && altkey != NULL) { + rc = registry_add(registry, altkey); + } + if (rc >= 0) { + /* The critical section may have been suspended while the keys + were added, letting another thread clear and re-stamp the + registry. If so, invalidate it. */ + int match = registry_version_matches(registry, version); + if (match < 0 || + (match == 0 && set_registry_version(registry, -1) < 0)) + { + rc = -1; + } + } + } + Py_END_CRITICAL_SECTION(); return rc; } @@ -795,8 +893,8 @@ warn_explicit(PyThreadState *tstate, PyObject *category, PyObject *message, PyObject *source) { PyObject *key = NULL, *text = NULL, *result = NULL, *lineno_obj = NULL; - PyObject *item = NULL; - PyObject *action; + PyObject *item = NULL, *action = NULL; + PyObject *altkey = NULL, *once_registry = NULL; int rc; PyInterpreterState *interp = tstate->interp; @@ -840,14 +938,23 @@ warn_explicit(PyThreadState *tstate, PyObject *category, PyObject *message, if (key == NULL) goto cleanup; + /* The warnings lock is not held. 'version' ties the registry updates + to the filters searched; a result from filters that changed since is + never recorded. */ + Py_ssize_t version; if ((registry != NULL) && (registry != Py_None)) { - rc = already_warned(interp, registry, key, 0); + rc = already_warned(interp, registry, key, &version); if (rc == -1) goto cleanup; else if (rc == 1) goto return_none; /* Else this warning hasn't been generated before. */ } + else { + registry = NULL; + WarningsState *st = warnings_get_state(interp); + version = FT_ATOMIC_LOAD_SSIZE_ACQUIRE(st->filters_version); + } action = get_filter(interp, category, text, lineno, module, filename, &item); if (action == NULL) @@ -864,50 +971,58 @@ warn_explicit(PyThreadState *tstate, PyObject *category, PyObject *message, /* Store in the registry that we've been here, *except* when the action is "always" or "all". */ - rc = 0; if (!_PyUnicode_EqualToASCIIString(action, "always") && !_PyUnicode_EqualToASCIIString(action, "all")) { - if (registry != NULL && registry != Py_None && - PyDict_SetItem(registry, key, Py_True) < 0) + bool is_once = _PyUnicode_EqualToASCIIString(action, "once"); + bool is_module = _PyUnicode_EqualToASCIIString(action, "module"); + if (!is_once && !is_module && + !_PyUnicode_EqualToASCIIString(action, "default")) { + PyErr_Format(PyExc_RuntimeError, + "Unrecognized action (%R) in warnings.filters:\n %R", + action, item); goto cleanup; } - if (_PyUnicode_EqualToASCIIString(action, "once")) { - if (registry == NULL || registry == Py_None) { - registry = get_once_registry(interp); - if (registry == NULL) + if (is_once && registry == NULL) { + /* WarningsState.once_registry[(text, category)] = 1 */ + once_registry = get_once_registry(interp); + if (once_registry == NULL) + goto cleanup; + altkey = _PyTuple_FromPair(text, category); + if (altkey == NULL) + goto cleanup; + rc = record_warning(interp, once_registry, NULL, altkey, version); + } + else if (registry != NULL) { + /* registry[key] = 1, and for "once" and "module" also + registry[(text, category)] = 1 */ + if (is_once || is_module) { + altkey = _PyTuple_FromPair(text, category); + if (altkey == NULL) goto cleanup; } - /* WarningsState.once_registry[(text, category)] = 1 */ - rc = update_registry(interp, registry, text, category, 0); + rc = record_warning(interp, registry, key, altkey, version); } - else if (_PyUnicode_EqualToASCIIString(action, "module")) { - /* registry[(text, category, 0)] = 1 */ - if (registry != NULL && registry != Py_None) - rc = update_registry(interp, registry, text, category, 0); + else { + rc = 0; } - else if (!_PyUnicode_EqualToASCIIString(action, "default")) { - PyErr_Format(PyExc_RuntimeError, - "Unrecognized action (%R) in warnings.filters:\n %R", - action, item); + if (rc == -1) goto cleanup; - } + if (rc == 1) /* Already warned for this module. */ + goto return_none; } - if (rc == 1) /* Already warned for this module. */ - goto return_none; - if (rc == 0) { - if (call_show_warning(tstate, category, text, message, filename, - lineno, lineno_obj, sourceline, source, module) < 0) - goto cleanup; - } - else /* if (rc == -1) */ + if (call_show_warning(tstate, category, text, message, filename, + lineno, lineno_obj, sourceline, source, module) < 0) goto cleanup; return_none: result = Py_NewRef(Py_None); cleanup: + Py_XDECREF(once_registry); + Py_XDECREF(altkey); + Py_XDECREF(action); Py_XDECREF(item); Py_XDECREF(key); Py_XDECREF(text); @@ -1154,10 +1269,8 @@ do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level, &filename, &lineno, &module, ®istry)) return NULL; - warnings_lock(tstate->interp); res = warn_explicit(tstate, category, message, filename, lineno, module, registry, NULL, source); - warnings_unlock(tstate->interp); Py_DECREF(filename); Py_DECREF(registry); Py_DECREF(module); @@ -1335,10 +1448,8 @@ warnings_warn_explicit_impl(PyObject *module, PyObject *message, } } - warnings_lock(tstate->interp); returned = warn_explicit(tstate, category, message, filename, lineno, mod, registry, source_line, sourceobj); - warnings_unlock(tstate->interp); Py_XDECREF(source_line); return returned; } @@ -1366,7 +1477,7 @@ warnings_filters_mutated_lock_held_impl(PyObject *module) return NULL; } - st->filters_version++; + FT_ATOMIC_ADD_SSIZE(st->filters_version, 1); Py_RETURN_NONE; } @@ -1485,10 +1596,8 @@ PyErr_WarnExplicitObject(PyObject *category, PyObject *message, return -1; } - warnings_lock(tstate->interp); res = warn_explicit(tstate, category, message, filename, lineno, module, registry, NULL, NULL); - warnings_unlock(tstate->interp); if (res == NULL) return -1; Py_DECREF(res); @@ -1553,10 +1662,8 @@ PyErr_WarnExplicitFormat(PyObject *category, PyObject *res; PyThreadState *tstate = get_current_tstate(); if (tstate != NULL) { - warnings_lock(tstate->interp); res = warn_explicit(tstate, category, message, filename, lineno, module, registry, NULL, NULL); - warnings_unlock(tstate->interp); Py_DECREF(message); if (res != NULL) { Py_DECREF(res); From ca0cc5063301bb62a3a73ff1328144c91fa558bc Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Thu, 12 Nov 2026 15:07:37 +0100 Subject: [PATCH 106/229] gh-110941: Fix json.dump encoding dict subclasses as empty (continued) (#150631) * gh-110941: Fix `json.dump` encoding dict subclasses as empty. Co-authored-by: Pieter Eendebak Co-authored-by: Alexander Ljungberg --- Lib/test/test_json/test_default.py | 39 +++++++++++++++++++ ...-10-18-13-31-10.gh-issue-110941.UVrYGE.rst | 2 + Modules/_json.c | 5 ++- 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-18-13-31-10.gh-issue-110941.UVrYGE.rst diff --git a/Lib/test/test_json/test_default.py b/Lib/test/test_json/test_default.py index 811880a15c80208..7f3ef8b8cb9f96c 100644 --- a/Lib/test/test_json/test_default.py +++ b/Lib/test/test_json/test_default.py @@ -1,7 +1,29 @@ import collections +import collections.abc from test.test_json import PyTest, CTest +class CustomIndexDict(collections.abc.Mapping, dict): + # Using `Mapping` here to make this a complete dict subclass, but note the bug in #110941 + # is not specific to `Mapping` subclasses and the outcome would have been the same with a + # fully manually implemented dict subclass. + + def __init__(self, keys: tuple = ()): + self._keys = keys + + def __getitem__(self, k): + try: + return self._keys.index(k) + except ValueError: + raise KeyError(k) + + def __iter__(self): + return iter(self._keys) + + def __len__(self): + return len(self._keys) + + class TestDefault: def test_default(self): self.assertEqual( @@ -36,6 +58,23 @@ def test_ordereddict(self): self.dumps(od, sort_keys=True), '{"a": 1, "b": 2, "c": 3, "d": 4}') + # This should behave identically for PyTest and CTest: see #110941. + def test_custom_dict(self): + cd = CustomIndexDict(("b", "a")) + self.assertEqual( + self.dumps(cd), + '{"b": 0, "a": 1}') + self.assertEqual( + self.dumps(cd, sort_keys=True), + '{"a": 1, "b": 0}') + + def test_empty_custom_dict(self): + # Exercise the fast path when a dict subclass is empty. + cd = CustomIndexDict() + self.assertEqual( + self.dumps(cd), + '{}') + class TestPyDefault(TestDefault, PyTest): pass class TestCDefault(TestDefault, CTest): pass diff --git a/Misc/NEWS.d/next/Library/2023-10-18-13-31-10.gh-issue-110941.UVrYGE.rst b/Misc/NEWS.d/next/Library/2023-10-18-13-31-10.gh-issue-110941.UVrYGE.rst new file mode 100644 index 000000000000000..5824b593460e9be --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-18-13-31-10.gh-issue-110941.UVrYGE.rst @@ -0,0 +1,2 @@ +Fix ``json.dump`` and ``json.dumps`` encoding certain dict subclasses as +empty. diff --git a/Modules/_json.c b/Modules/_json.c index 3a724a3e72b185b..e79b697f06c994c 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1802,7 +1802,10 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, PyObject *ident = NULL; bool first = true; - if (PyDict_GET_SIZE(dct) == 0) { + // Only take the fast path for exact dicts: a subclass may keep its + // contents outside the dict storage, so PyDict_GET_SIZE could be 0 while + // the mapping is non-empty (gh-110941). + if (PyAnyDict_CheckExact(dct) && PyDict_GET_SIZE(dct) == 0) { /* Fast path */ return PyUnicodeWriter_WriteASCII(writer, "{}", 2); } From 020c5f3b3867b0d392fd8b2586435b4018ccdf8a Mon Sep 17 00:00:00 2001 From: Corencas <81588100+Corencas@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:15:48 -0600 Subject: [PATCH 107/229] gh-106338: Link the tokenizer docs from the re.Scanner source (GH-161595) --- Lib/re/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/re/__init__.py b/Lib/re/__init__.py index e6d29fd7a403b64..8ab77efc8ce48cc 100644 --- a/Lib/re/__init__.py +++ b/Lib/re/__init__.py @@ -391,6 +391,13 @@ def _pickle(p): # experimental stuff (see python-dev discussions for details) class Scanner: + """Experimental regex-based tokenizer, not part of the documented API. + + New code is better off starting from the tokenizer example in the + documentation: + https://docs.python.org/3/library/re.html#writing-a-tokenizer + """ + def __init__(self, lexicon, flags=0): from ._constants import BRANCH, SUBPATTERN if isinstance(flags, RegexFlag): From 27026842f6774c167295f4b59972589f12ec5987 Mon Sep 17 00:00:00 2001 From: Brett Smith Date: Thu, 12 Nov 2026 10:50:42 -0500 Subject: [PATCH 108/229] gh-161707: Refuse `;` in all cookie attributes except Comment Co-authored-by: Victor Stinner --- Doc/library/http.cookies.rst | 13 +++++++ Lib/http/cookies.py | 38 +++++++++++-------- Lib/test/test_http_cookies.py | 24 ++++++++++++ ...-11-09-17-19-29.gh-issue-161707.xWzwqQ.rst | 5 +++ 4 files changed, 64 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-11-09-17-19-29.gh-issue-161707.xWzwqQ.rst diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index 4a8d461213f4e57..5ac1a318be80f8a 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -188,6 +188,11 @@ Morsel Objects .. versionchanged:: 3.14 Added support for the :attr:`partitioned` attribute. + .. versionchanged:: next + :meth:`!__setitem__` and :meth:`!__ior__` now raise :exc:`CookieError` if + a value string for any attribute except :attr:`comment` contains the + reserved separator character ``;``. (If :attr:`comment` contains ``;`` + it will be escaped in the output.) .. attribute:: Morsel.value @@ -253,6 +258,10 @@ Morsel Objects .. versionchanged:: 3.5 an error is raised for invalid keys. + .. versionchanged:: next + :exc:`CookieError` is now raised if a value string for any attribute + except :attr:`comment` contains the reserved separator character ``;``. + (If :attr:`comment` contains ``;`` it will be escaped in the output.) .. method:: Morsel.copy(value) @@ -267,6 +276,10 @@ Morsel Objects Raise an error if key is not a valid :rfc:`2109` attribute, otherwise behave the same as :meth:`dict.setdefault`. + .. versionchanged:: next + :exc:`CookieError` is now raised if the value string for any attribute + except :attr:`comment` contains the reserved separator character ``;``. + (If :attr:`comment` contains ``;`` it will be escaped in the output.) .. _cookie-example: diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 9a6f01dfb5e69a6..57ee1bd12f6ff22 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -300,20 +300,31 @@ def value(self): def coded_value(self): return self._coded_value + def _check_attribute(self, key, val): + """Normalize and validate an attribute name and value + + Raise `CookieError` if the key is not a recognized attribute name or + value contains illegal characters. Otherwise return the canonicalized + key. + """ + key = key.lower() + val_str = str(val) + if key not in self._reserved: + raise CookieError(f"Invalid attribute {key!r}") + if _has_control_character(key, val_str): + raise CookieError(f"Control characters are not allowed in cookies {key!r} {val!r}") + # Comment is quoted like a cookie value so semicolons can be escaped. + # No other attribute expects `;` in the value, so refuse it. + if key != 'comment' and ';' in val_str: + raise CookieError(f"Reserved character ';' is not allowed in cookie attribute value {key!r} {val!r}") + return key + def __setitem__(self, K, V): - K = K.lower() - if not K in self._reserved: - raise CookieError("Invalid attribute %r" % (K,)) - if _has_control_character(K, V): - raise CookieError(f"Control characters are not allowed in cookies {K!r} {V!r}") + K = self._check_attribute(K, V) dict.__setitem__(self, K, V) def setdefault(self, key, val=None): - key = key.lower() - if key not in self._reserved: - raise CookieError("Invalid attribute %r" % (key,)) - if _has_control_character(key, val): - raise CookieError("Control characters are not allowed in cookies %r %r" % (key, val,)) + key = self._check_attribute(key, val) return dict.setdefault(self, key, val) def __eq__(self, morsel): @@ -335,12 +346,7 @@ def copy(self): def update(self, values): data = {} for key, val in dict(values).items(): - key = key.lower() - if key not in self._reserved: - raise CookieError("Invalid attribute %r" % (key,)) - if _has_control_character(key, val): - raise CookieError("Control characters are not allowed in " - f"cookies {key!r} {val!r}") + key = self._check_attribute(key, val) data[key] = val dict.update(self, data) diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index d1df2ec42f0d146..6632c39159865c8 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -628,6 +628,30 @@ def test_control_characters(self): with self.assertRaises(cookies.CookieError): morsel |= {c0: "val"} + def test_no_semicolons_in_attribute_values(self): + for attrkey, attrname in cookies.Morsel._reserved.items(): + if attrname == "Comment": + continue + with self.subTest(attrkey=attrkey, attrname=attrname): + morsel = cookies.Morsel() + with self.assertRaises(cookies.CookieError): + morsel[attrkey] = "foo; SameSite=none" + with self.assertRaises(cookies.CookieError): + morsel.setdefault(attrkey, "foo; SameSite=none") + with self.assertRaises(cookies.CookieError): + morsel.update({attrkey: "foo; SameSite=none"}) + with self.assertRaises(cookies.CookieError): + morsel |= {attrkey: "foo; SameSite=none"} + + def test_semicolon_escaped_in_comment(self): + morsel = cookies.Morsel() + morsel.set("key", "value", "coded") + morsel["comment"] = "foo;SameSite=none;Domain=example.com" + self.assertEqual( + morsel.OutputString(), + r'key=coded; Comment="foo\073SameSite=none\073Domain=example.com"', + ) + def test_control_characters_output(self): # Tests that even if the internals of Morsel are modified # that a call to .output() has control character safeguards. diff --git a/Misc/NEWS.d/next/Security/2026-11-09-17-19-29.gh-issue-161707.xWzwqQ.rst b/Misc/NEWS.d/next/Security/2026-11-09-17-19-29.gh-issue-161707.xWzwqQ.rst new file mode 100644 index 000000000000000..63731daafff41ef --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-11-09-17-19-29.gh-issue-161707.xWzwqQ.rst @@ -0,0 +1,5 @@ +.. cve: CVE-2026-16052 + +:mod:`http.cookies` now raises :exc:`~http.cookies.CookieError` on any attempt +to set a value string that contains the reserved separator character ``;`` to +any attribute except :attr:`~http.cookies.Morsel.comment`. From 755e3239cca1386c93dd55fb54c07c74dac1c85d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?tonghuaroot=20=28=E7=AB=A5=E8=AF=9D=29?= Date: Thu, 12 Nov 2026 23:58:35 +0800 Subject: [PATCH 109/229] gh-161861: preserve http continue timeout after rejected request (#161862) move the continue timeout assignment after the connection state check so a rejected putrequest call cannot modify the active request. --- Lib/http/client.py | 2 +- Lib/test/test_httplib.py | 14 ++++++++++++++ .../2026-11-12-17-53-13.gh-issue-161861.Kf7Qm2.rst | 2 ++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-12-17-53-13.gh-issue-161861.Kf7Qm2.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index 4bdb8f72f979b16..aa643e7f63ddd85 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -1268,7 +1268,6 @@ def putrequest(self, method, url, skip_host=False, """ if continue_timeout <= 0.0: raise ValueError("continue_timeout must be greater than 0") - self._continue_timeout = continue_timeout # if a prior response has been completed, then forget about it. if self.__response and self.__response.isclosed(): @@ -1294,6 +1293,7 @@ def putrequest(self, method, url, skip_host=False, # if self.__state == _CS_IDLE: self.__state = _CS_REQ_STARTED + self._continue_timeout = continue_timeout else: raise CannotSendRequest(self.__state) diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 849c44e17aab0a2..adcd11e22d33378 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2587,6 +2587,20 @@ def test_putrequest_continue_timeout(self): self.assertIn(b'Body content', conn.sock.data) self.assertEqual(conn.getresponse().code, 200) + def test_failed_putrequest_preserves_continue_timeout(self): + conn = FakeSocketHTTPConnection( + b'HTTP/1.1 100 Continue\r\n' + b'\r\n' + ) + conn._select = mock.Mock(wraps=conn.fake_select) + conn.putrequest('PUT', '/', continue_timeout=0.25) + with self.assertRaises(client.CannotSendRequest): + conn.putrequest('GET', '/', continue_timeout=10) + conn.putheader('Expect', '100-continue') + conn.putheader('Content-Length', '12') + conn.endheaders(b'Body content') + conn._select.assert_called_once_with([conn.sock], [], [], 0.25) + def test_invalid_continue_timeout(self): conn = FakeSocketHTTPConnection(b'') for timeout in (0, -1): diff --git a/Misc/NEWS.d/next/Library/2026-11-12-17-53-13.gh-issue-161861.Kf7Qm2.rst b/Misc/NEWS.d/next/Library/2026-11-12-17-53-13.gh-issue-161861.Kf7Qm2.rst new file mode 100644 index 000000000000000..febea87b27d5a1b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-12-17-53-13.gh-issue-161861.Kf7Qm2.rst @@ -0,0 +1,2 @@ +Preserve the active request's ``continue_timeout`` when a second +:meth:`~http.client.HTTPConnection.putrequest` call is rejected. From ab575e4e11912b3089b33c0a93cf1e31386dd84c Mon Sep 17 00:00:00 2001 From: Stian Halseth <33956011+shalseth@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:08:16 +0100 Subject: [PATCH 110/229] gh-160253: Fix test_c_stack_unwind manual unwinding on sparc64 (GH-160255) SPARC V9 has no frame record. The caller's %fp (biased by 2047) and return address are in slots 14 and 15 of the register window save area at fp, and are only there after the windows have been flushed to the stack. --- Modules/_testinternalcapi.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 3efb70f662698b9..586b687f233ce02 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -111,10 +111,20 @@ static const uintptr_t min_frame_pointer_addr = 0x1000; // address is stored at fp[-1]. # define FRAME_POINTER_NEXT_OFFSET -2 # define FRAME_POINTER_RETURN_OFFSET -1 +#elif defined(__sparc__) && defined(__arch64__) +// SPARC V9 has no frame record: fp points to the register window save area, +// where the caller's %fp is saved at fp[14] and its return address (%i7) at +// fp[15]. The saved %fp is biased by 2047. +# define FRAME_POINTER_NEXT_OFFSET 14 +# define FRAME_POINTER_RETURN_OFFSET 15 +# define FRAME_POINTER_BIAS 2047 #else # define FRAME_POINTER_NEXT_OFFSET 0 # define FRAME_POINTER_RETURN_OFFSET 1 #endif +#ifndef FRAME_POINTER_BIAS +# define FRAME_POINTER_BIAS 0 +#endif static PyObject * @@ -564,7 +574,8 @@ manual_unwind_from_fp(uintptr_t *frame_pointer) stack_min, stack_max)) { break; } - uintptr_t *next_fp = (uintptr_t *)frame_pointer[FRAME_POINTER_NEXT_OFFSET]; + uintptr_t *next_fp = (uintptr_t *)(frame_pointer[FRAME_POINTER_NEXT_OFFSET] + + FRAME_POINTER_BIAS); uintptr_t return_addr = frame_pointer[FRAME_POINTER_RETURN_OFFSET]; PyObject *addr_obj = PyLong_FromUnsignedLongLong(return_addr); @@ -636,6 +647,10 @@ gnu_backtrace_unwind(PyObject *self, PyObject *Py_UNUSED(args)) static PyObject * manual_frame_pointer_unwind(PyObject *self, PyObject *args) { +#if defined(__sparc__) && defined(__arch64__) + // The callers' register windows are only in their save areas once spilled. + __asm__ volatile ("flushw" ::: "memory"); +#endif uintptr_t *frame_pointer = (uintptr_t *)__builtin_frame_address(0); return manual_unwind_from_fp(frame_pointer); } From 7ac989f86503b6c7707c698ecb77eee93e861897 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:17:00 -0800 Subject: [PATCH 111/229] gh-161104: Check the http.client tunnel host after IDNA encoding (GH-161105) IDNA normalization maps U+00A0 and similar characters to an ASCII space, so check the encoded host that is put into the CONNECT request line. ANT-2026-PIZPMECM --- Lib/http/client.py | 8 ++++++-- Lib/test/test_httplib.py | 5 +++++ .../2026-10-22-05-23-04.gh-issue-161104.rvNPjJ.rst | 4 ++++ 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-05-23-04.gh-issue-161104.rvNPjJ.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index aa643e7f63ddd85..864ffc12b21e8a1 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -1028,11 +1028,15 @@ def _wrap_ipv6(self, ip): return ip def _tunnel(self): - if _contains_disallowed_url_pchar_re.search(self._tunnel_host): + # Check the host as it is sent: the IDNA codec normalizes the name + # first (NFKC), which maps some non-ASCII characters to ASCII ones, + # e.g. U+00A0 (NO-BREAK SPACE) to a space. + tunnel_host = self._tunnel_host.encode("idna") + if _contains_disallowed_url_pchar_re.search(tunnel_host.decode("ascii")): raise ValueError('Tunnel host can\'t contain control characters %r' % (self._tunnel_host,)) connect = b"CONNECT %s:%d %s\r\n" % ( - self._wrap_ipv6(self._tunnel_host.encode("idna")), + self._wrap_ipv6(tunnel_host), self._tunnel_port, self._http_vsn_str.encode("ascii")) headers = [connect] diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index adcd11e22d33378..04fcd0c883e638f 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -426,6 +426,11 @@ def test_invalid_tunnel_host(self): 'invalid.host\r\n', 'invalid.host\x00', 'invalid host', + # The IDNA codec normalizes these to an ASCII space + 'invalid\xa0host', + 'invalid\u3000host', + # ... and this one to a space and a combining character + 'invalid\xa8host', ) for tunnel_host in cases: with self.subTest(tunnel_host): diff --git a/Misc/NEWS.d/next/Library/2026-10-22-05-23-04.gh-issue-161104.rvNPjJ.rst b/Misc/NEWS.d/next/Library/2026-10-22-05-23-04.gh-issue-161104.rvNPjJ.rst new file mode 100644 index 000000000000000..acae526043a2979 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-05-23-04.gh-issue-161104.rvNPjJ.rst @@ -0,0 +1,4 @@ +Fix :meth:`http.client.HTTPConnection.set_tunnel` accepting a tunnel host +containing a non-ASCII character that IDNA encoding maps to a space, such as +U+00A0 NO-BREAK SPACE. Connecting now raises :exc:`ValueError`, as it already +does for an ASCII space. From 16b9ac289d655cf88843752af9f9883fe973e2f1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:23:20 +0200 Subject: [PATCH 112/229] Lock GitHub Actions workflows (#161811) --- .github/actionlint.yaml | 3 + .github/workflows/actions.lock | 161 ++++++++++++++++++ .github/workflows/build.yml | 50 +++--- .github/workflows/jit.yml | 18 +- .github/workflows/lint.yml | 19 ++- .github/workflows/mypy.yml | 4 +- .../workflows/new-bugs-announce-notifier.yml | 8 +- .github/workflows/require-pr-label.yml | 8 +- .../workflows/reusable-check-c-api-docs.yml | 4 +- .github/workflows/reusable-check-html-ids.yml | 6 +- .github/workflows/reusable-cifuzz.yml | 4 +- .github/workflows/reusable-context.yml | 4 +- .github/workflows/reusable-docs.yml | 16 +- .github/workflows/reusable-emscripten.yml | 8 +- .github/workflows/reusable-install.yml | 2 +- .github/workflows/reusable-macos.yml | 2 +- .github/workflows/reusable-san.yml | 6 +- .../reusable-test-lazy-imports-all.yml | 2 +- .github/workflows/reusable-ubuntu.yml | 4 +- .github/workflows/reusable-wasi.yml | 8 +- .github/workflows/reusable-windows.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/tail-call.yml | 4 +- .github/workflows/verify-ensurepip-wheels.yml | 4 +- .github/workflows/verify-expat.yml | 2 +- .pre-commit-config.yaml | 8 +- 26 files changed, 270 insertions(+), 89 deletions(-) create mode 100644 .github/workflows/actions.lock diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 61ee08db9919bb9..37d9121dadfc8f8 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -12,3 +12,6 @@ paths: ignore: - 1st argument of function call is not assignable - SC2(015|038|086|091|097|098|129|155) + # Pending release of actionlint > 1.7.12 for `$/` self-repository syntax: + # https://github.com/rhysd/actionlint/issues/711 + - reusable workflow call "\$/[^"]+" at "uses" is not following the format diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock new file mode 100644 index 000000000000000..9e4ec30ba9d2546 --- /dev/null +++ b/.github/workflows/actions.lock @@ -0,0 +1,161 @@ +# This file is machine-generated by `gh actions-lock`. +# Do not edit by hand; run `gh actions-lock` to update. +# Docs: https://gh.io/actions-lockfile +version: 'v0.0.3' +workflows: + '.github/workflows/build.yml': + - 'actions/cache@v6.1.0' + - 'actions/checkout@v7.0.1' + - 'actions/setup-python@v7.0.0' + - 'actions/upload-artifact@v7.0.1' + - 're-actors/alls-green@v1.3.0' + '.github/workflows/cache-cleanup-codeql.yml': [] + '.github/workflows/jit.yml': + - 'actions/checkout@v7.0.1' + - 'actions/setup-python@v7.0.0' + '.github/workflows/lint.yml': + - 'actions/checkout@v7.0.1' + - 'j178/prek-action@v3.0.0' + '.github/workflows/mypy.yml': + - 'actions/checkout@v7.0.1' + - 'astral-sh/setup-uv@v10.2.0' + '.github/workflows/new-bugs-announce-notifier.yml': + - 'actions/github-script@v9.0.0' + - 'actions/setup-node@v7.0.0' + '.github/workflows/require-pr-label.yml': + - 'mheap/github-action-required-labels@v5.6.0' + '.github/workflows/reusable-check-c-api-docs.yml': + - 'actions/checkout@v7.0.1' + - 'actions/setup-python@v7.0.0' + '.github/workflows/reusable-check-html-ids.yml': + - 'actions/checkout@v7.0.1' + - 'actions/download-artifact@v8.0.1' + - 'actions/setup-python@v7.0.0' + '.github/workflows/reusable-cifuzz.yml': + - 'actions/upload-artifact@v7.0.1' + - 'github/codeql-action@v4.38.2' + - 'google/oss-fuzz@ed23f8af80ff82b25ca67cd9b101e690b8897b3f' + '.github/workflows/reusable-context.yml': + - 'actions/checkout@v7.0.1' + - 'actions/setup-python@v7.0.0' + '.github/workflows/reusable-docs.yml': + - 'actions/cache@v6.1.0' + - 'actions/checkout@v7.0.1' + - 'actions/setup-python@v7.0.0' + - 'actions/upload-artifact@v7.0.1' + '.github/workflows/reusable-emscripten.yml': + - 'actions/cache@v6.1.0' + - 'actions/checkout@v7.0.1' + - 'actions/setup-node@v7.0.0' + - 'actions/setup-python@v7.0.0' + '.github/workflows/reusable-install.yml': + - 'actions/checkout@v7.0.1' + '.github/workflows/reusable-macos.yml': + - 'actions/checkout@v7.0.1' + '.github/workflows/reusable-san.yml': + - 'actions/cache@v6.1.0' + - 'actions/checkout@v7.0.1' + - 'actions/upload-artifact@v7.0.1' + '.github/workflows/reusable-test-lazy-imports-all.yml': + - 'actions/checkout@v7.0.1' + '.github/workflows/reusable-ubuntu.yml': + - 'actions/cache@v6.1.0' + - 'actions/checkout@v7.0.1' + '.github/workflows/reusable-wasi.yml': + - 'actions/checkout@v7.0.1' + - 'actions/setup-python@v7.0.0' + - 'bytecodealliance/actions@v1.1.3' + - 'bytecodealliance/setup-wasi-sdk-action@v1.0.0' + '.github/workflows/reusable-windows.yml': + - 'actions/checkout@v7.0.1' + '.github/workflows/stale.yml': + - 'actions/stale@v11.0.0' + '.github/workflows/tail-call.yml': + - 'actions/checkout@v7.0.1' + - 'actions/setup-python@v7.0.0' + '.github/workflows/verify-ensurepip-wheels.yml': + - 'actions/checkout@v7.0.1' + - 'actions/setup-python@v7.0.0' + '.github/workflows/verify-expat.yml': + - 'actions/checkout@v7.0.1' +dependencies: + 'actions/cache@v6.1.0': + ref: 'v6.1.0' + commit: 'sha1-55cc8345863c7cc4c66a329aec7e433d2d1c52a9' + owner_id: 44036562 + repo_id: 215566462 + 'actions/checkout@v7.0.1': + ref: 'v7.0.1' + commit: 'sha1-3d3c42e5aac5ba805825da76410c181273ba90b1' + owner_id: 44036562 + repo_id: 197814629 + 'actions/download-artifact@v8.0.1': + ref: 'v8.0.1' + commit: 'sha1-3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c' + owner_id: 44036562 + repo_id: 192626254 + 'actions/github-script@v9.0.0': + ref: 'v9.0.0' + commit: 'sha1-3a2844b7e9c422d3c10d287c895573f7108da1b3' + owner_id: 44036562 + repo_id: 205262760 + 'actions/setup-node@v7.0.0': + ref: 'v7.0.0' + commit: 'sha1-820762786026740c76f36085b0efc47a31fe5020' + owner_id: 44036562 + repo_id: 189476904 + 'actions/setup-python@v7.0.0': + ref: 'v7.0.0' + commit: 'sha1-5fda3b95a4ea91299a34e894583c3862153e4b97' + owner_id: 44036562 + repo_id: 192625525 + 'actions/stale@v11.0.0': + ref: 'v11.0.0' + commit: 'sha1-4391f3da665fdf50b6810c1a66712fb9ba21aa93' + owner_id: 44036562 + repo_id: 200437066 + 'actions/upload-artifact@v7.0.1': + ref: 'v7.0.1' + commit: 'sha1-043fb46d1a93c77aae656e7c1c64a875d1fc6a0a' + owner_id: 44036562 + repo_id: 192625955 + 'astral-sh/setup-uv@v10.2.0': + ref: 'v10.2.0' + commit: 'sha1-c18668ad3cf93ea998bef934396af7bb5c839dc7' + owner_id: 115962839 + repo_id: 846733598 + 'bytecodealliance/actions@v1.1.3': + ref: 'v1.1.3' + commit: 'sha1-9152e710e9f7182e4c29ad218e4f335a7b203613' + owner_id: 54038801 + repo_id: 697422232 + 'bytecodealliance/setup-wasi-sdk-action@v1.0.0': + ref: 'v1.0.0' + commit: 'sha1-77d09f3df1e8da344d8403cf8631af8977938b4c' + owner_id: 54038801 + repo_id: 1113975359 + 'github/codeql-action@v4.38.2': + ref: 'v4.38.2' + commit: 'sha1-2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2' + owner_id: 9919 + repo_id: 259445878 + 'google/oss-fuzz@ed23f8af80ff82b25ca67cd9b101e690b8897b3f': + ref: 'master' + commit: 'sha1-ed23f8af80ff82b25ca67cd9b101e690b8897b3f' + owner_id: 1342004 + repo_id: 63809205 + 'j178/prek-action@v3.0.0': + ref: 'v3.0.0' + commit: 'sha1-4e14d07f9231acabce116ccfca13b13dd9755ece' + owner_id: 10510431 + repo_id: 1052387486 + 'mheap/github-action-required-labels@v5.6.0': + ref: 'v5.6.0' + commit: 'sha1-23e10fde7e062233401931a0eece796cd9bf3177' + owner_id: 59130 + repo_id: 272521528 + 're-actors/alls-green@v1.3.0': + ref: 'v1.3.0' + commit: 'sha1-b5b5b37504aa4183270bd3d855c52a67f212be35' + owner_id: 46427759 + repo_id: 434447646 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 07087b836cb597e..50b0469867c1a4e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,13 +41,13 @@ jobs: # || 'falsy-branch' # }} # - uses: ./.github/workflows/reusable-context.yml + uses: $/.github/workflows/reusable-context.yml check-docs: name: Docs needs: build-context if: fromJSON(needs.build-context.outputs.run-docs) - uses: ./.github/workflows/reusable-docs.yml + uses: $/.github/workflows/reusable-docs.yml check-autoconf-regen: name: 'Check if Autoconf files are up to date' @@ -64,7 +64,7 @@ jobs: run: | apt update && apt install git -yq git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: fetch-depth: 1 persist-credentials: false @@ -101,10 +101,10 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@v7.0.0 with: python-version: '3.x' - name: Runner image version @@ -147,7 +147,7 @@ jobs: if: >- needs.build-context.outputs.run-tests == 'true' || needs.build-context.outputs.run-docs == 'true' - uses: ./.github/workflows/reusable-check-c-api-docs.yml + uses: $/.github/workflows/reusable-check-c-api-docs.yml build-windows: name: >- @@ -175,7 +175,7 @@ jobs: # and only supported on 3.15+. - { arch: x64, free-threading: false, interpreter: tail-call } - { arch: x64, free-threading: true, interpreter: tail-call } - uses: ./.github/workflows/reusable-windows.yml + uses: $/.github/workflows/reusable-windows.yml with: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} @@ -201,7 +201,7 @@ jobs: exclude: - os: macos-26-intel free-threading: true - uses: ./.github/workflows/reusable-macos.yml + uses: $/.github/workflows/reusable-macos.yml with: free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -238,7 +238,7 @@ jobs: bolt: false free-threading: false test-opts: '-u cpu' - uses: ./.github/workflows/reusable-ubuntu.yml + uses: $/.github/workflows/reusable-ubuntu.yml with: bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} @@ -249,7 +249,7 @@ jobs: name: 'Ubuntu (installed)' needs: build-context if: needs.build-context.outputs.run-ubuntu == 'true' - uses: ./.github/workflows/reusable-install.yml + uses: $/.github/workflows/reusable-install.yml build-ubuntu-ssltests: name: 'Ubuntu SSL tests' @@ -280,7 +280,7 @@ jobs: SSLLIB_DIR: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }}/lib steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Runner image version @@ -291,7 +291,7 @@ jobs: run: sudo ./.github/workflows/posix-deps-apt.sh - name: 'Restore SSL library build' id: cache-ssl-lib - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: actions/cache@v6.1.0 with: path: ./multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} key: ${{ matrix.os }}-multissl-${{ matrix.ssllib.name }}-${{ matrix.ssllib.version }} @@ -339,7 +339,7 @@ jobs: runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Build and test @@ -360,7 +360,7 @@ jobs: - false - true steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false @@ -373,13 +373,13 @@ jobs: name: 'Emscripten' needs: build-context if: needs.build-context.outputs.run-emscripten == 'true' - uses: ./.github/workflows/reusable-emscripten.yml + uses: $/.github/workflows/reusable-emscripten.yml build-wasi: name: 'WASI' needs: build-context if: needs.build-context.outputs.run-wasi == 'true' - uses: ./.github/workflows/reusable-wasi.yml + uses: $/.github/workflows/reusable-wasi.yml test-hypothesis: name: "Hypothesis tests on Ubuntu" @@ -391,7 +391,7 @@ jobs: OPENSSL_VER: 3.5.9 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Register gcc problem matcher @@ -405,7 +405,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: actions/cache@v6.1.0 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -452,7 +452,7 @@ jobs: ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: actions/cache@v6.1.0 with: path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} @@ -479,7 +479,7 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - uses: actions/upload-artifact@v7.0.1 if: always() with: name: hypothesis-example-db @@ -489,7 +489,7 @@ jobs: name: 'Lazy imports enabled' needs: build-context if: fromJSON(needs.build-context.outputs.run-tests) - uses: ./.github/workflows/reusable-test-lazy-imports-all.yml + uses: $/.github/workflows/reusable-test-lazy-imports-all.yml build-san: # ${{ '' } is a hack to nest jobs under the same sidebar category. @@ -516,7 +516,7 @@ jobs: - check-name: Memory sanitizer: MSan free-threading: false - uses: ./.github/workflows/reusable-san.yml + uses: $/.github/workflows/reusable-san.yml with: sanitizer: ${{ matrix.sanitizer }} free-threading: ${{ matrix.free-threading }} @@ -528,7 +528,7 @@ jobs: needs: build-context if: needs.build-context.outputs.run-ubuntu == 'true' steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Runner image version @@ -598,7 +598,7 @@ jobs: && 'no-exclude' || 'python3-libraries' }} - uses: ./.github/workflows/reusable-cifuzz.yml + uses: $/.github/workflows/reusable-cifuzz.yml with: oss-fuzz-project-name: ${{ matrix.oss-fuzz-project-name }} sanitizer: ${{ matrix.sanitizer }} @@ -630,7 +630,7 @@ jobs: steps: - name: Check whether the needed jobs succeeded or failed - uses: re-actors/alls-green@b5b5b37504aa4183270bd3d855c52a67f212be35 + uses: re-actors/alls-green@v1.3.0 with: allowed-failures: >- build-android, diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 381b00185b1ae87..21ecb2c958c12d3 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Install dependencies @@ -72,10 +72,10 @@ jobs: architecture: ARM64 runner: windows-11-arm steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@v7.0.0 with: python-version: '3.11' # PCbuild downloads LLVM automatically: @@ -98,10 +98,10 @@ jobs: - true - false steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@v7.0.0 with: python-version: '3.11' - name: Install LLVM @@ -141,10 +141,10 @@ jobs: - target: aarch64-unknown-linux-gnu/gcc runner: ubuntu-26.04-arm steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@v7.0.0 with: python-version: '3.11' - name: Install dependencies @@ -182,10 +182,10 @@ jobs: use_clang: true run_tests: false steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@v7.0.0 with: python-version: '3.11' - name: Install dependencies diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cd413cd4cfbaa0f..0b89ebe92d367d4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,7 +19,22 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 + - uses: j178/prek-action@v3.0.0 + + # Fails when .github/workflows/actions.lock is out of sync with the + # workflows or a locked pin no longer verifies upstream. + lockfile: + runs-on: ubuntu-slim + timeout-minutes: 10 + env: + GH_TOKEN: ${{ github.token }} + + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - run: gh extension install github/gh-actions-lock --pin v0.4.1 + - run: gh actions-lock --verify diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index a8c95627c2417de..6109870724ea59c 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -66,10 +66,10 @@ jobs: "Tools/peg_generator", ] steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + - uses: astral-sh/setup-uv@v10.2.0 with: python-version: "3.15" activate-environment: true diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 640474273a9de33..8a1c59f50d8d1de 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -15,12 +15,14 @@ jobs: issues: read timeout-minutes: 10 steps: - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + - uses: actions/setup-node@v7.0.0 with: node-version: 20 - - run: npm install mailgun.js form-data + # TODO: Replace with a committed lockfile and `npm ci`, or drop npm + # and call the Mailgun REST API from the standard library. + - run: npm install mailgun.js form-data # zizmor: ignore[adhoc-packages] - name: Send notification - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + uses: actions/github-script@v9.0.0 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 05c862c59181221..58d223bb34f203c 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Check there's no DO-NOT-MERGE - uses: mheap/github-action-required-labels@23e10fde7e062233401931a0eece796cd9bf3177 # v5.6.0 + uses: mheap/github-action-required-labels@v5.6.0 with: mode: exactly count: 0 @@ -36,7 +36,7 @@ jobs: steps: # Check that the PR is not awaiting changes from the author due to previous review. - name: Check there's no required changes - uses: mheap/github-action-required-labels@23e10fde7e062233401931a0eece796cd9bf3177 # v5.6.0 + uses: mheap/github-action-required-labels@v5.6.0 with: mode: exactly count: 0 @@ -45,7 +45,7 @@ jobs: awaiting change review - id: is-feature name: Check whether this PR is a feature (contains a "type-feature" label) - uses: mheap/github-action-required-labels@23e10fde7e062233401931a0eece796cd9bf3177 # v5.6.0 + uses: mheap/github-action-required-labels@v5.6.0 with: mode: exactly count: 1 @@ -56,7 +56,7 @@ jobs: - id: awaiting-merge if: steps.is-feature.outputs.status == 'success' name: Check for complete review - uses: mheap/github-action-required-labels@23e10fde7e062233401931a0eece796cd9bf3177 # v5.6.0 + uses: mheap/github-action-required-labels@v5.6.0 with: mode: exactly count: 1 diff --git a/.github/workflows/reusable-check-c-api-docs.yml b/.github/workflows/reusable-check-c-api-docs.yml index 636c960df4a086b..ed4b3f86df4e00b 100644 --- a/.github/workflows/reusable-check-c-api-docs.yml +++ b/.github/workflows/reusable-check-c-api-docs.yml @@ -15,10 +15,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@v7.0.0 with: python-version: '3.x' - name: Check for undocumented C APIs diff --git a/.github/workflows/reusable-check-html-ids.yml b/.github/workflows/reusable-check-html-ids.yml index d07659c96a20b03..bd258ecd971a7ad 100644 --- a/.github/workflows/reusable-check-html-ids.yml +++ b/.github/workflows/reusable-check-html-ids.yml @@ -16,7 +16,7 @@ jobs: timeout-minutes: 30 steps: - name: 'Check out PR head' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@v7.0.1 with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} @@ -43,7 +43,7 @@ jobs: MERGE_BASE: ${{ steps.merge-base.outputs.sha }} run: git worktree add /tmp/merge-base "$MERGE_BASE" --detach - name: 'Set up Python' - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@v7.0.0 with: python-version: '3' cache: 'pip' @@ -55,7 +55,7 @@ jobs: - name: 'Collect HTML IDs' run: python Doc/tools/check-html-ids.py collect /tmp/merge-base/Doc/build/html -o /tmp/html-ids-base.json.gz - name: 'Download PR head HTML IDs' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/download-artifact@v8.0.1 with: name: html-ids-head.json.gz path: /tmp diff --git a/.github/workflows/reusable-cifuzz.yml b/.github/workflows/reusable-cifuzz.yml index a70b75eebc4a989..e14c842d7a8dafe 100644 --- a/.github/workflows/reusable-cifuzz.yml +++ b/.github/workflows/reusable-cifuzz.yml @@ -37,13 +37,13 @@ jobs: sanitizer: ${{ inputs.sanitizer }} - name: Upload crash if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@v7.0.1 with: name: ${{ inputs.sanitizer }}-artifacts path: ./out/artifacts - name: Upload SARIF if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + uses: github/codeql-action/upload-sarif@v4.38.2 with: sarif_file: cifuzz-sarif/results.sarif checkout_path: cifuzz-sarif diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index dedfdb3e552f2f7..6c3b454d1a40997 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -73,14 +73,14 @@ jobs: run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} steps: - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@v7.0.0 with: python-version: "3" - run: >- echo '${{ github.event_name }}' - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false ref: >- diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index e13c43526bc1907..b60bdfd302db8fa 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -38,7 +38,7 @@ jobs: refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@v7.0.1 with: persist-credentials: false ref: >- @@ -63,7 +63,7 @@ jobs: git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@v7.0.0 with: python-version: '3' cache: 'pip' @@ -94,7 +94,7 @@ jobs: run: python Doc/tools/check-html-ids.py collect Doc/build/html -o Doc/build/html-ids-head.json.gz - name: 'Upload HTML IDs' if: github.event_name == 'pull_request' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@v7.0.1 with: name: html-ids-head path: Doc/build/html-ids-head.json.gz @@ -104,7 +104,7 @@ jobs: name: 'Check for removed HTML IDs' needs: build-doc if: github.event_name == 'pull_request' - uses: ./.github/workflows/reusable-check-html-ids.yml + uses: $/.github/workflows/reusable-check-html-ids.yml # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: @@ -112,10 +112,10 @@ jobs: runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - uses: actions/cache@v6.1.0 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} @@ -138,11 +138,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: 'Set up Python' - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@v7.0.0 with: python-version: '3' cache: 'pip' diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index d093c86eb2db967..6565e671420f09e 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-26.04 timeout-minutes: 40 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: "Read Emscripten config" @@ -41,18 +41,18 @@ jobs: with open(os.environ["GITHUB_ENV"], "a") as f: f.write(f"PY_EMSDK_CACHE={emsdk_cache}\n") - name: "Install Node.js" - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + uses: actions/setup-node@v7.0.0 with: node-version: ${{ steps.emscripten-config.outputs.node-version }} - name: "Cache Emscripten SDK" id: emsdk-cache - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: actions/cache@v6.1.0 with: path: ${{ env.PY_EMSDK_CACHE }} key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }}-${{ steps.emscripten-config.outputs.deps-hash }} restore-keys: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }} - name: "Install Python" - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@v7.0.0 with: python-version: '3.x' - name: "Runner image version" diff --git a/.github/workflows/reusable-install.yml b/.github/workflows/reusable-install.yml index b8acf1a0fb1771e..fa09a9a4f34c58d 100644 --- a/.github/workflows/reusable-install.yml +++ b/.github/workflows/reusable-install.yml @@ -17,7 +17,7 @@ jobs: env: PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Register gcc problem matcher diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 2883a20b612f852..ec9fd64b282aa1c 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -31,7 +31,7 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Runner image version diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 9c31241498a75e9..fb7d84e889e118f 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Runner image version @@ -100,7 +100,7 @@ jobs: - name: 'Restore OpenSSL build' id: cache-openssl if: inputs.sanitizer == 'TSan' || inputs.sanitizer == 'ASan' - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: actions/cache@v6.1.0 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: >- @@ -163,7 +163,7 @@ jobs: run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 - name: Archive logs if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@v7.0.1 with: name: >- ${{ inputs.sanitizer }}-logs-${{ diff --git a/.github/workflows/reusable-test-lazy-imports-all.yml b/.github/workflows/reusable-test-lazy-imports-all.yml index b30a776f1a98581..ea2615b880558b0 100644 --- a/.github/workflows/reusable-test-lazy-imports-all.yml +++ b/.github/workflows/reusable-test-lazy-imports-all.yml @@ -26,7 +26,7 @@ jobs: env: EXCLUDE_FILE: Lib/test/lazy_imports_all_exclude.txt steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Register gcc problem matcher diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 1b10078b8409f4f..f6e42e1861349c4 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -39,7 +39,7 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Register gcc problem matcher @@ -59,7 +59,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: actions/cache@v6.1.0 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 5b2fda1fe4f82dc..d4d51fbd1f3029c 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -17,12 +17,12 @@ jobs: env: WASMTIME_VERSION: 48.0.2 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false # No problem resolver registered as one doesn't currently exist for Clang. - name: "Install wasmtime" - uses: bytecodealliance/actions/wasmtime/setup@9152e710e9f7182e4c29ad218e4f335a7b203613 # v1.1.3 + uses: bytecodealliance/actions/wasmtime/setup@v1.1.3 with: version: ${{ env.WASMTIME_VERSION }} - name: "Read WASI SDK version" @@ -38,12 +38,12 @@ jobs: shell: python - name: "Install WASI SDK" id: install-wasi-sdk - uses: bytecodealliance/setup-wasi-sdk-action@77d09f3df1e8da344d8403cf8631af8977938b4c + uses: bytecodealliance/setup-wasi-sdk-action@v1.0.0 with: version: ${{ steps.wasi-sdk-version.outputs.version }} add-to-path: false - name: "Install Python" - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@v7.0.0 with: python-version: '3.x' - name: "Runner image version" diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 4be4bc05fa6d0d1..eb6c22f2c4cf437 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -31,7 +31,7 @@ jobs: env: ARCH: ${{ inputs.arch }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Register MSVC problem matcher diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c6094947cf98f67..4b528a6f9861778 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -18,7 +18,7 @@ jobs: steps: - name: "Check PRs" - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0 + uses: actions/stale@v11.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 90 days with no activity.' diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 1dc5ea1bc51356c..c302f543a8fd746 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -41,10 +41,10 @@ jobs: runner: ubuntu-26.04-arm configure_flags: --with-pydebug steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@v7.0.0 with: python-version: '3.11' - name: Build diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 4d0c5d21d5def23..89083b65d71bfa3 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -25,10 +25,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@v7.0.0 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/.github/workflows/verify-expat.yml b/.github/workflows/verify-expat.yml index 7a1046059e0ba89..ce1ae0ee8a77fca 100644 --- a/.github/workflows/verify-expat.yml +++ b/.github/workflows/verify-expat.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-slim timeout-minutes: 5 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7.0.1 with: persist-credentials: false - name: Download and verify bundled libexpat files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 45528d9f32d6c94..bda616bad11c54f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 3b3f7c3f57fe9925356faf5fe6230835138be230 # frozen: v0.15.17 + rev: c0f278f1254ccc8fb09676bcdb8d847e7db493e5 # frozen: v0.17.4 hooks: - id: ruff-check name: Run Ruff (lint) on Platforms/Apple/ @@ -70,7 +70,7 @@ repos: files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror - rev: ea488cebbfd88a5f50b8bd95d5c829d0bb76feb8 # frozen: 26.1.0 + rev: a5439e9af2ff9371297bb1a76a01b8ad17989b88 # frozen: 26.10.1 hooks: - id: black name: Run Black on Tools/jit/ @@ -104,7 +104,7 @@ repos: files: '^\.github/CODEOWNERS|\.(gram)$' - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 9f48a48aa91a6040d749ad68ec70907d907a5a7f # frozen: 0.37.0 + rev: a76b92851fc9eede9f9bc4073eb55202f34eb0df # frozen: 0.39.0 hooks: - id: check-dependabot - id: check-github-workflows @@ -116,7 +116,7 @@ repos: - id: actionlint - repo: https://github.com/zizmorcore/zizmor-pre-commit - rev: b546b77c44c466a54a42af5499dcc0dcc1a3193f # frozen: v1.22.0 + rev: 06a39750ec31860bb44f2acf67182c40016cd1e7 # frozen: v1.32.0 hooks: - id: zizmor From a969bcf0713caac7ad5366de1eb193cf4ba25a46 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Thu, 12 Nov 2026 10:35:18 -0600 Subject: [PATCH 113/229] gh-161735: Update zlib-ng to 2.2.5 on Windows (GH-161813) --- .../2026-11-11-10-44-20.gh-issue-161735.MZLRnD.rst | 1 + Misc/externals.spdx.json | 8 ++++---- PCbuild/get_externals.bat | 2 +- PCbuild/python.props | 2 +- PCbuild/readme.txt | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2026-11-11-10-44-20.gh-issue-161735.MZLRnD.rst diff --git a/Misc/NEWS.d/next/Windows/2026-11-11-10-44-20.gh-issue-161735.MZLRnD.rst b/Misc/NEWS.d/next/Windows/2026-11-11-10-44-20.gh-issue-161735.MZLRnD.rst new file mode 100644 index 000000000000000..a824533a8caa9d6 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-11-11-10-44-20.gh-issue-161735.MZLRnD.rst @@ -0,0 +1 @@ +Updated bundled version of ``zlib-ng`` to v2.2.5. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 0800347e34016db..59fbb2056c7be0c 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -175,21 +175,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004" + "checksumValue": "a9734c94080792af0b50d9c51c2cc8845c2f8ecfa4db87f6462a9b260af67f96" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.5.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:zlib-ng:zlib-ng:2.2.4:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:zlib-ng:zlib-ng:2.2.5:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "zlib-ng", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.2.4" + "versionInfo": "2.2.5" }, { "SPDXID": "SPDXRef-PACKAGE-zstd", diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index b5f070ca2d858d3..dca67e1e6c0a8ae 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -60,7 +60,7 @@ set libraries=%libraries% sqlite-3.53.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-9.0.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-9.0.4.1 set libraries=%libraries% xz-5.8.1.1 -set libraries=%libraries% zlib-ng-2.2.4 +set libraries=%libraries% zlib-ng-2.2.5 set libraries=%libraries% zstd-1.5.7 for %%e in (%libraries%) do ( diff --git a/PCbuild/python.props b/PCbuild/python.props index 4c49decd7490d1b..ef6f8a88d62d090 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -110,7 +110,7 @@ $(opensslOutDir)include $(ExternalsDir)\nasm-2.11.06\ $(ExternalsDir)\zlib-1.3.1\ - $(ExternalsDir)\zlib-ng-2.2.4\ + $(ExternalsDir)\zlib-ng-2.2.5\ $(ExternalsDir)\zstd-1.5.7\ diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 32ea8921c667c32..d232b4b61e87e26 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -267,7 +267,7 @@ _zstd zlib-ng Compiles zlib-ng as a static library, which is later included by pythoncore.vcxproj. This was generated using CMake against zlib-ng - version 2.2.4, and should be minimally updated as needed to adapt + version 2.2.5, and should be minimally updated as needed to adapt to changes in their source layout. The zbuild.h, zconf.h and zconf-ng.h file in the PC directory were likewise generated and vendored. From 9894dc7af5e1e095467b41648c0572d76280c001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Preng=C3=A8re?= <2138730+alexprengere@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:14:58 +0100 Subject: [PATCH 114/229] gh-158166: Make ElementTree ParseError messages strings (GH-158168) --- Lib/test/test_xml_etree.py | 5 +++++ Lib/xml/etree/ElementTree.py | 2 +- .../Library/2026-09-26-09-44-59.gh-issue-158166.lVGCsI.rst | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-26-09-44-59.gh-issue-158166.lVGCsI.rst diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index f0cf7dffab0e8a9..acb11615b35c0b3 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1324,6 +1324,11 @@ def test_parse_error_after_chunk_boundary(self): ET.fromstring('%s<' % ('x' * size)) self.assertEqual(cm.exception.position, (1, size + 4)) + def test_parse_error_msg_is_string(self): + with self.assertRaises(ET.ParseError) as cm: + ET.XML("<") + self.assertIsInstance(cm.exception.msg, str) + self.assertEqual(cm.exception.msg, "unclosed token: line 1, column 0") @support.subTests('sample,exception', [ (b' \xa1', UnicodeDecodeError), # crashed diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 0cd04ead801600e..3a50b1329bde3e9 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -1709,7 +1709,7 @@ def handler(pi_target, data, event=event_name, append=append, raise ValueError("unknown event %r" % event_name) def _raiseerror(self, value): - err = ParseError(value) + err = ParseError(str(value)) err.code = value.code err.position = value.lineno, value.offset raise err diff --git a/Misc/NEWS.d/next/Library/2026-09-26-09-44-59.gh-issue-158166.lVGCsI.rst b/Misc/NEWS.d/next/Library/2026-09-26-09-44-59.gh-issue-158166.lVGCsI.rst new file mode 100644 index 000000000000000..528160a6709369a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-26-09-44-59.gh-issue-158166.lVGCsI.rst @@ -0,0 +1 @@ +Fix the type of the ``msg`` attribute of :exc:`xml.etree.ElementTree.ParseError` in the pure Python implementation: it is now a string, as in the C implementation. From f322b3ec8ac9bce2ddaaa26a70a4e505bc18cba1 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Thu, 12 Nov 2026 09:19:03 -0800 Subject: [PATCH 115/229] gh-101472: Redocument `argparse.REMAINDER` and its limitations (#160119) --- Doc/library/argparse.rst | 52 +++++++++++++++++++++++++- Lib/test/test_argparse.py | 79 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 128 insertions(+), 3 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index ffcf2ff37be53bf..0c69c00950c5b5b 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -977,6 +977,53 @@ See also :ref:`specifying-ambiguous-arguments`. The supported values are: usage: PROG [-h] foo [foo ...] PROG: error: the following arguments are required: foo +.. _argparse.REMAINDER: + +* ``argparse.REMAINDER``. Remaining command-line arguments are gathered into a + list, including strings that look like options. This is useful for forwarding + arguments to another command. A common pattern is a final positional argument + after a command name; options for the outer parser must precede that command:: + + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo') + >>> parser.add_argument('command') + >>> parser.add_argument('args', nargs=argparse.REMAINDER) + >>> parser.parse_args('--foo B cmd --foo C'.split()) + Namespace(foo='B', command='cmd', args=['--foo', 'C']) + >>> parser.parse_args(['cmd']) + Namespace(foo=None, command='cmd', args=[]) + + When the remainder is the first positional argument, leading options are still + parsed normally. It starts with the first positional value, or with ``--``. + In a subparser, the subcommand name has already been consumed by the parent + parser and does not start the remainder. Any ``--`` consumed by the remainder + is retained:: + + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('args', nargs=argparse.REMAINDER) + >>> parser.parse_args(['--', '--foo', 'B']) + Namespace(args=['--', '--foo', 'B']) + + For an optional argument, supply the option separately from its values:: + + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--forward', nargs=argparse.REMAINDER) + >>> parser.parse_args('--forward --foo B --'.split()) + Namespace(forward=['--foo', 'B', '--']) + >>> parser.parse_args(['--forward']) + Namespace(forward=[]) + + An attached value such as ``--forward=value`` supplies only that value; + subsequent arguments are parsed normally. + + .. versionchanged:: 3.15 + Optional arguments with ``nargs=argparse.REMAINDER`` consume ``--`` and + subsequent arguments instead of stopping at ``--``. + + With the default ``'store'`` action, an empty positional remainder ignores + default_ and produces ``[]``. The choices_ restriction is not applied to the + remainder. + If the ``nargs`` keyword argument is not provided, the number of arguments consumed is determined by the action_. Generally this means a single command-line argument will be consumed and a single item (not a list) will be produced. @@ -2334,8 +2381,9 @@ Intermixed parsing These parsers do not support all the :mod:`!argparse` features, and will raise exceptions if unsupported features are used. In particular, subparsers, - and mutually exclusive groups that include both - optionals and positionals are not supported. + positional arguments with + :ref:`nargs=argparse.REMAINDER `, and mutually exclusive + groups that include both optionals and positionals are not supported. The following example shows the difference between :meth:`~ArgumentParser.parse_known_args` and diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 7aba0da52462ae5..bb6e5117a369c65 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -1839,6 +1839,75 @@ class TestNargsRemainder(ParserTestCase): ] +class TestNargsRemainderBoundaries(TestCase): + + def test_first_positional(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo') + parser.add_argument('args', nargs=argparse.REMAINDER) + + for args in (['--unknown'], ['--foo', 'B', '--unknown']): + with self.subTest(args=args): + with self.assertRaisesRegex( + argparse.ArgumentError, + 'unrecognized arguments: --unknown'): + parser.parse_args(args) + self.assertEqual(parser.parse_args(['--foo', 'B']), + NS(foo='B', args=[])) + self.assertEqual(parser.parse_args(['cmd', '--foo', 'C']), + NS(foo=None, args=['cmd', '--foo', 'C'])) + self.assertEqual(parser.parse_args(['--', '--foo', 'C']), + NS(foo=None, args=['--', '--foo', 'C'])) + + def test_first_positional_in_subparser(self): + parser = argparse.ArgumentParser(exit_on_error=False) + subparser = parser.add_subparsers().add_parser('run') + subparser.add_argument('--foo', action='store_true') + subparser.add_argument('args', nargs=argparse.REMAINDER) + + with self.assertRaisesRegex(argparse.ArgumentError, + 'unrecognized arguments: --unknown'): + parser.parse_args(['run', '--unknown']) + self.assertEqual(parser.parse_args(['run', '--foo']), + NS(foo=True, args=[])) + self.assertEqual(parser.parse_args(['run', 'cmd', '--foo']), + NS(foo=False, args=['cmd', '--foo'])) + self.assertEqual(parser.parse_args(['run', '--', '--foo']), + NS(foo=False, args=['--', '--foo'])) + + def test_optional_captures_options(self): + parser = argparse.ArgumentParser() + parser.add_argument('--foo', action='store_true') + parser.add_argument('--forward', nargs=argparse.REMAINDER, + default=['fallback']) + + self.assertEqual(parser.parse_args(['--forward']), + NS(foo=False, forward=[])) + self.assertEqual(parser.parse_args(['--forward', '--foo']), + NS(foo=False, forward=['--foo'])) + + def test_optional_attached_value(self): + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--forward', nargs=argparse.REMAINDER) + + for option in ('--forward=a', '-fa'): + with self.subTest(option=option): + self.assertEqual(parser.parse_known_args([option, 'b']), + (NS(forward=['a']), ['b'])) + + def test_empty_positional_ignores_default(self): + parser = argparse.ArgumentParser() + parser.add_argument('args', nargs=argparse.REMAINDER, + default=['fallback']) + self.assertEqual(parser.parse_args([]), NS(args=[])) + + def test_type_conversion_without_choices_validation(self): + parser = argparse.ArgumentParser() + parser.add_argument('args', nargs=argparse.REMAINDER, + type=int, choices=[1]) + self.assertEqual(parser.parse_args(['2']), NS(args=[2])) + + class TestOptionLike(ParserTestCase): """Tests options that may or may not be arguments""" @@ -6857,7 +6926,7 @@ def test_basic(self): self.assertEqual(bar.required, True) def test_remainder(self): - # Intermixed and remainder are incompatible + # Intermixed and positional remainder are incompatible parser = ErrorRaisingArgumentParser(prog='PROG') parser.add_argument('-z') parser.add_argument('x') @@ -6869,6 +6938,14 @@ def test_remainder(self): parser.parse_intermixed_args(argv) self.assertRegex(str(cm.exception), r'\.\.\.') + def test_optional_remainder(self): + parser = argparse.ArgumentParser() + parser.add_argument('command') + parser.add_argument('--forward', nargs=argparse.REMAINDER) + self.assertEqual( + parser.parse_intermixed_args(['cmd', '--forward', '--foo']), + NS(command='cmd', forward=['--foo'])) + def test_required_exclusive(self): # required mutually exclusive group; intermixed works fine parser = argparse.ArgumentParser(prog='PROG', exit_on_error=False) From 8b85375e9e72f6de9f1cad498a99658c3a2ad91c Mon Sep 17 00:00:00 2001 From: John Date: Thu, 12 Nov 2026 17:32:01 +0000 Subject: [PATCH 116/229] gh-112564: Fix UAF for gc debug print (gh-160662) Fix a crash in the garbage collector when gc.DEBUG_COLLECTABLE or gc.DEBUG_UNCOLLECTABLE is set and writing the debug output to sys.stderr frees one of the objects being reported. --- ...-10-25-12-30-00.gh-issue-112564.gcDbgW.rst | 5 ++ Python/gc.c | 66 +++++++++++++++---- 2 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-25-12-30-00.gh-issue-112564.gcDbgW.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-25-12-30-00.gh-issue-112564.gcDbgW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-25-12-30-00.gh-issue-112564.gcDbgW.rst new file mode 100644 index 000000000000000..5ff3cb147e71c0d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-25-12-30-00.gh-issue-112564.gcDbgW.rst @@ -0,0 +1,5 @@ +Fix a crash in the garbage collector when :data:`gc.DEBUG_COLLECTABLE` or +:data:`gc.DEBUG_UNCOLLECTABLE` is set and writing the debug output to +:data:`sys.stderr` frees one of the objects being reported. Object addresses +in this output now use the same ``0x``-prefixed lowercase format on all +platforms, including Windows. diff --git a/Python/gc.c b/Python/gc.c index 5dc79e20191ebc7..e363fa77f724f88 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -996,11 +996,55 @@ clear_weakrefs(PyGC_Head *unreachable) } } -static void +/* "gc: " + " <" + " 0x" + 16 hex digits + ">\n" + NUL, excluding msg and + * tp_name */ +#define DEBUG_LINE_FIXED_LEN 32 + +static char * debug_cycle(const char *msg, PyObject *op) { - PySys_FormatStderr("gc: %s <%s %p>\n", - msg, Py_TYPE(op)->tp_name, op); + const char *name = Py_TYPE(op)->tp_name; + size_t size = strlen(msg) + strlen(name) + DEBUG_LINE_FIXED_LEN; + char *line = PyMem_RawMalloc(size); + if (line == NULL) { + return NULL; + } + /* Not %p: its output is platform-specific (no "0x" prefix on Windows). */ + PyOS_snprintf(line, size, "gc: %s <%s 0x%" PRIxPTR ">\n", + msg, name, (uintptr_t)op); + return line; +} + +/* Format a line for each object in `list` before printing any of them, + * since sys.stderr.write() can run Python code that frees objects in the + * list. + */ +static Py_ssize_t +debug_cycles(const char *msg, PyGC_Head *list) +{ + Py_ssize_t n = gc_list_size(list); + if (n == 0) { + return n; + } + + char **lines = PyMem_RawCalloc(n, sizeof(char *)); + if (lines == NULL) { + return n; + } + + Py_ssize_t i = 0; + for (PyGC_Head *gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc), i++) { + lines[i] = debug_cycle(msg, FROM_GC(gc)); + } + + for (i = 0; i < n; i++) { + if (lines[i] != NULL) { + PySys_FormatStderr("%s", lines[i]); + } + PyMem_RawFree(lines[i]); + } + PyMem_RawFree(lines); + return n; } /* Handle uncollectable garbage (cycles with tp_del slots, and stuff reachable @@ -1435,7 +1479,6 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) PyGC_Head *old; /* next older generation */ PyGC_Head unreachable; /* non-problematic unreachable trash */ PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ - PyGC_Head *gc; GCState *gcstate = &tstate->interp->gc; // gc_collect_main() must not be called before _PyGC_Init @@ -1558,9 +1601,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) { /* sys.stderr.write() can resurrect objects. */ needs_resurrection_check = true; - for (gc = GC_NEXT(&unreachable); gc != &unreachable; gc = GC_NEXT(gc)) { - debug_cycle("collectable", FROM_GC(gc)); - } + debug_cycles("collectable", &unreachable); } /* Clear weakrefs and invoke callbacks as necessary. */ @@ -1603,13 +1644,12 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) /* Collect statistics on uncollectable objects found and print * debugging information. */ - Py_ssize_t n = 0; - for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { - n++; - if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) - debug_cycle("uncollectable", FROM_GC(gc)); + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) { + stats.uncollectable = debug_cycles("uncollectable", &finalizers); + } + else { + stats.uncollectable = gc_list_size(&finalizers); } - stats.uncollectable = n; (void)PyTime_PerfCounterRaw(&stats.ts_stop); stats.duration = PyTime_AsSecondsDouble(stats.ts_stop - stats.ts_start); if (gcstate->debug & _PyGC_DEBUG_STATS) { From 46d437dfedfd7cc0995927085b368b0ffe895b4d Mon Sep 17 00:00:00 2001 From: Yusuke Hayashi Date: Fri, 13 Nov 2026 03:14:41 +0900 Subject: [PATCH 117/229] gh-140774: Fix clearing the read-only file attribute on Windows (GH-154852) SetFileInformationByHandle() interprets a FILE_BASIC_INFO with FileAttributes == 0 as "leave the attributes unchanged", so clearing the read-only flag of a file with no other attribute set was silently ignored by os.chmod(follow_symlinks=True), os.fchmod() and pathlib.Path.chmod(). Substitute FILE_ATTRIBUTE_NORMAL in that case. Co-Authored-By: Claude Fable 5 --- Lib/test/test_os/test_windows.py | 61 +++++++++++++++++++ ...-07-29-13-05-00.gh-issue-140774.wA3ttr.rst | 3 + Modules/posixmodule.c | 6 ++ 3 files changed, 70 insertions(+) create mode 100644 Misc/NEWS.d/next/Windows/2026-07-29-13-05-00.gh-issue-140774.wA3ttr.rst diff --git a/Lib/test/test_os/test_windows.py b/Lib/test/test_os/test_windows.py index e79d2c2c94d25b2..b9659267af54c4b 100644 --- a/Lib/test/test_os/test_windows.py +++ b/Lib/test/test_os/test_windows.py @@ -454,6 +454,67 @@ def test_unlink_removes_junction(self): self.assertFalse(os.path.exists(self.junction)) +class Win32ChmodTests(unittest.TestCase): + # gh-140774: os.chmod(follow_symlinks=True) and os.fchmod() go through + # SetFileInformationByHandle(), which interprets FileAttributes == 0 as + # "leave the attributes unchanged". Clearing the read-only flag of a + # file with no other attribute set must substitute FILE_ATTRIBUTE_NORMAL + # for 0 to take effect. + + def setUp(self): + self.fname = os.path.abspath(os_helper.TESTFN) + self.addCleanup(os_helper.unlink, self.fname) + self.addCleanup(self._make_writable) + create_file(self.fname) + + def _make_writable(self): + try: + os.chmod(self.fname, stat.S_IWRITE | stat.S_IREAD, + follow_symlinks=False) + except OSError: + pass + + def _set_readonly_only_attribute(self): + # Make FILE_ATTRIBUTE_READONLY the only attribute of the file + # (in particular, clear FILE_ATTRIBUTE_ARCHIVE). + ctypes = import_helper.import_module('ctypes') + wintypes = import_helper.import_module('ctypes.wintypes') + kernel32 = ctypes.WinDLL('Kernel32.dll', use_last_error=True) + SetFileAttributesW = kernel32.SetFileAttributesW + SetFileAttributesW.argtypes = (wintypes.LPCWSTR, wintypes.DWORD) + SetFileAttributesW.restype = wintypes.BOOL + + if not SetFileAttributesW(self.fname, stat.FILE_ATTRIBUTE_READONLY): + raise ctypes.WinError(ctypes.get_last_error()) + attributes = os.stat(self.fname).st_file_attributes + if attributes != stat.FILE_ATTRIBUTE_READONLY: + self.skipTest('file system reports extra file attributes ' + f'{attributes:#x}') + + def _assert_writable(self): + self.assertEqual(os.stat(self.fname).st_file_attributes + & stat.FILE_ATTRIBUTE_READONLY, 0) + + def test_chmod_clears_readonly_without_other_attributes(self): + self._set_readonly_only_attribute() + os.chmod(self.fname, stat.S_IWRITE | stat.S_IREAD, + follow_symlinks=True) + self._assert_writable() + + def test_fchmod_clears_readonly_without_other_attributes(self): + fd = os.open(self.fname, os.O_RDWR) + try: + self._set_readonly_only_attribute() + os.fchmod(fd, stat.S_IWRITE | stat.S_IREAD) + self._assert_writable() + + self._set_readonly_only_attribute() + os.chmod(fd, stat.S_IWRITE | stat.S_IREAD) + self._assert_writable() + finally: + os.close(fd) + + class Win32NtTests(unittest.TestCase): def test_getfinalpathname_handles(self): nt = import_helper.import_module('nt') diff --git a/Misc/NEWS.d/next/Windows/2026-07-29-13-05-00.gh-issue-140774.wA3ttr.rst b/Misc/NEWS.d/next/Windows/2026-07-29-13-05-00.gh-issue-140774.wA3ttr.rst new file mode 100644 index 000000000000000..aaf120efb992e79 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-07-29-13-05-00.gh-issue-140774.wA3ttr.rst @@ -0,0 +1,3 @@ +Fix :func:`os.chmod` with ``follow_symlinks=True``, :func:`os.fchmod` and +:meth:`pathlib.Path.chmod` on Windows: clearing the read-only attribute of a +file which has no other attribute set is no longer silently ignored. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 67461fdb3730860..c6f89fc2454e88a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4021,6 +4021,12 @@ win32_hchmod(HANDLE hfile, int mode) } if (mode & _S_IWRITE) { info.FileAttributes &= ~FILE_ATTRIBUTE_READONLY; + if (info.FileAttributes == 0) { + /* SetFileInformationByHandle() interprets FileAttributes == 0 + as "leave the attributes unchanged"; FILE_ATTRIBUTE_NORMAL + is the documented way to clear all attributes. */ + info.FileAttributes = FILE_ATTRIBUTE_NORMAL; + } } else { info.FileAttributes |= FILE_ATTRIBUTE_READONLY; From 20876589722925438ed2988c3f2a9d8869b20e9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=87=E5=BD=AC=E5=AE=87?= <3431359639@qq.com> Date: Fri, 13 Nov 2026 02:21:55 +0800 Subject: [PATCH 118/229] gh-126253: Ignore unknown Windows file attributes in test_os (#156305) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Victor Stinner --- Lib/test/test_os/test_os.py | 24 +++++++++++++++++-- ...-09-16-14-37-28.gh-issue-126253.9FOKPp.rst | 1 + 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-09-16-14-37-28.gh-issue-126253.9FOKPp.rst diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 6e7201d0b056c21..8b6b7deb65b88bd 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -84,6 +84,19 @@ HAVE_WHEEL_GROUP = sys.platform.startswith('freebsd') and os.getgid() == 0 +def _get_file_attribute_mask(): + mask = 0 + for name in dir(stat): + if name.startswith("FILE_ATTRIBUTE_"): + mask |= getattr(stat, name) + return mask + + +# gh-126253: Ignore unknown attributes, such as 0x1_000_0000 +# (Directory flag of DOS file attribute in NTFS) +FILE_ATTRIBUTE_MASK = _get_file_attribute_mask() + + def requires_os_func(name): return unittest.skipUnless(hasattr(os, name), 'requires os.%s' % name) @@ -5045,8 +5058,15 @@ def assert_stat_equal(self, stat1, stat2, skip_fields): if attr in ("st_dev", "st_ino", "st_nlink", "st_ctime", "st_ctime_ns"): continue - self.assertEqual(getattr(stat1, attr), - getattr(stat2, attr), + value1 = getattr(stat1, attr) + value2 = getattr(stat2, attr) + if attr == "st_file_attributes": + # Filesystems may report attribute bits that aren't + # exposed by the stat module. + value1 &= FILE_ATTRIBUTE_MASK + value2 &= FILE_ATTRIBUTE_MASK + self.assertEqual(value1, + value2, (stat1, stat2, attr)) else: self.assertEqual(stat1, stat2) diff --git a/Misc/NEWS.d/next/Tests/2026-09-16-14-37-28.gh-issue-126253.9FOKPp.rst b/Misc/NEWS.d/next/Tests/2026-09-16-14-37-28.gh-issue-126253.9FOKPp.rst new file mode 100644 index 000000000000000..370d822f883aa9a --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-09-16-14-37-28.gh-issue-126253.9FOKPp.rst @@ -0,0 +1 @@ +Fix ``test_os.test_attributes()`` on Windows NTFS: ignore unknown file attributes. From a6018c587e4e8c154581b70d43149606ffce4783 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Thu, 12 Nov 2026 19:54:57 +0100 Subject: [PATCH 119/229] gh-151126: set `MemoryError` on alloc failure in `_PyXIData_InitWithSize` (#156301) Co-authored-by: Cody Maloney --- .../2026-08-23-16-31-00.gh-issue-151126.phAWIG.rst | 1 + Python/crossinterp.c | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-23-16-31-00.gh-issue-151126.phAWIG.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-23-16-31-00.gh-issue-151126.phAWIG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-23-16-31-00.gh-issue-151126.phAWIG.rst new file mode 100644 index 000000000000000..b1459de1c1596d8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-23-16-31-00.gh-issue-151126.phAWIG.rst @@ -0,0 +1 @@ +Fix a crash when allocation failure occurs during :ref:`interp-object-sharing`. diff --git a/Python/crossinterp.c b/Python/crossinterp.c index b8dd5f11c2a44c7..d189d7bcefe313e 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -388,6 +388,8 @@ _PyXIData_InitWithSize(_PyXIData_t *xidata, _PyXIData_Init(xidata, interp, NULL, obj, new_object); xidata->data = PyMem_RawCalloc(1, size); if (xidata->data == NULL) { + Py_CLEAR(xidata->obj); + PyErr_NoMemory(); return -1; } xidata->free = PyMem_RawFree; From eb7cd6e636d7a01891b11b1e3b22e9fb4ea45412 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 12 Nov 2026 12:23:43 -0700 Subject: [PATCH 120/229] gh-161821: Fall back to lld for Clang LTO builds (gh-161822) Clang LTO with a GNU linker needs the LLVMgold.so plugin, which some packages (e.g. LLVM for Debian 13) don't ship. When using Clang with --with-lto on non-macOS platforms, configure now test-links an LTO program and, if that fails, retries with -fuse-ld=lld. --- Doc/using/configure.rst | 9 +- ...1-11-11-36-01.gh-issue-161821.9wjElMQP.rst | 4 + configure | 86 +++++++++++++++++++ configure.ac | 26 ++++++ 4 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2026-11-11-11-36-01.gh-issue-161821.9wjElMQP.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index ea608394b8eb101..4f2e4dfc88d3863 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -739,7 +739,14 @@ also be used to improve performance. Enable Link Time Optimization (LTO) in any build (disabled by default). The C compiler Clang requires ``llvm-ar`` for LTO (``ar`` on macOS), as well - as an LTO-aware linker (``ld.gold`` or ``lld``). + as an LTO-aware linker (``ld.gold`` or ``lld``). GNU linkers require the + ``LLVMgold.so`` plugin for Clang LTO; ``lld`` does not. On platforms other + than macOS, configure tries ``-fuse-ld=lld`` if the configured linker cannot + link a Clang LTO program. + + .. versionchanged:: next + Fall back to ``lld`` when the configured linker cannot link Clang LTO + programs. .. versionadded:: 3.6 diff --git a/Misc/NEWS.d/next/Build/2026-11-11-11-36-01.gh-issue-161821.9wjElMQP.rst b/Misc/NEWS.d/next/Build/2026-11-11-11-36-01.gh-issue-161821.9wjElMQP.rst new file mode 100644 index 000000000000000..2cfece1209c6e94 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-11-11-11-36-01.gh-issue-161821.9wjElMQP.rst @@ -0,0 +1,4 @@ +When building with ``--with-lto`` and Clang, :program:`configure` now checks +that the linker can link LTO objects and falls back to ``lld`` +(``-fuse-ld=lld``) if it cannot. This fixes LTO builds on systems where the +default GNU linker lacks the ``LLVMgold.so`` plugin, such as Debian 13. diff --git a/configure b/configure index 65d58fd1e810e23..e033b0217aefad5 100755 --- a/configure +++ b/configure @@ -9314,6 +9314,92 @@ fi else LTOFLAGS="-flto=${Py_LTO_POLICY}" fi + LTOCFLAGS="$LTOFLAGS" + # GNU ld needs LLVMgold.so for Clang LTO. If the configured linker + # cannot link LTO objects, try lld, which does not need that plugin. + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + LDFLAGS="$LDFLAGS $LDFLAGS_NODIST $LTOFLAGS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether Clang LTO works with the configured linker" >&5 +printf %s "checking whether Clang LTO works with the configured linker... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + clang_lto_link=yes +else case e in #( + e) clang_lto_link=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $clang_lto_link" >&5 +printf "%s\n" "$clang_lto_link" >&6; } + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + if test "$clang_lto_link" = no; then + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + LDFLAGS="$LDFLAGS -fuse-ld=lld" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether Clang LTO works with lld" >&5 +printf %s "checking whether Clang LTO works with lld... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + clang_lto_link=yes +else case e in #( + e) clang_lto_link=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $clang_lto_link" >&5 +printf "%s\n" "$clang_lto_link" >&6; } + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + if test "$clang_lto_link" = no; then + as_fn_error $? "Clang LTO requires a working LTO-aware linker. Install lld or the LLVMgold.so plugin, select a linker with LDFLAGS, or disable LTO with --without-lto." "$LINENO" 5 + fi + LTOFLAGS="$LTOFLAGS -fuse-ld=lld" + LDFLAGS_NOLTO="$LDFLAGS_NOLTO -fuse-ld=lld" + fi ;; esac ;; diff --git a/configure.ac b/configure.ac index a041f0115bd7ab3..d654f680643041c 100644 --- a/configure.ac +++ b/configure.ac @@ -2220,6 +2220,32 @@ if test "$Py_LTO" = 'true' ; then else LTOFLAGS="-flto=${Py_LTO_POLICY}" fi + LTOCFLAGS="$LTOFLAGS" + # GNU ld needs LLVMgold.so for Clang LTO. If the configured linker + # cannot link LTO objects, try lld, which does not need that plugin. + WITH_SAVE_ENV([ + LDFLAGS="$LDFLAGS $LDFLAGS_NODIST $LTOFLAGS" + AC_MSG_CHECKING([whether Clang LTO works with the configured linker]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([], [])], [clang_lto_link=yes], [clang_lto_link=no]) + AC_MSG_RESULT([$clang_lto_link]) + ]) + if test "$clang_lto_link" = no; then + WITH_SAVE_ENV([ + LDFLAGS="$LDFLAGS -fuse-ld=lld" + AC_MSG_CHECKING([whether Clang LTO works with lld]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([], [])], [clang_lto_link=yes], [clang_lto_link=no]) + AC_MSG_RESULT([$clang_lto_link]) + ]) + if test "$clang_lto_link" = no; then + AC_MSG_ERROR([m4_normalize([ + Clang LTO requires a working LTO-aware linker. + Install lld or the LLVMgold.so plugin, select a linker + with LDFLAGS, or disable LTO with --without-lto. + ])]) + fi + LTOFLAGS="$LTOFLAGS -fuse-ld=lld" + LDFLAGS_NOLTO="$LDFLAGS_NOLTO -fuse-ld=lld" + fi ;; esac ;; From 78a70f862e3b7f28c62bd9352af83920678f399a Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 12 Nov 2026 18:30:04 -0300 Subject: [PATCH 121/229] gh-158032: Keep the element tag alive during Element.iter comparison (GH-158044) --- Lib/test/test_xml_etree_c.py | 29 +++++++++++++++++++ ...-09-23-21-15-00.gh-issue-158032.k4nT9p.rst | 2 ++ Modules/_elementtree.c | 4 ++- 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-23-21-15-00.gh-issue-158032.k4nT9p.rst diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index deb2bd8d572581a..cf616f300670a3c 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -161,6 +161,35 @@ def test_xmlpullparser_leaks(self): del parser support.gc_collect() + def test_iter_tag_reentrant_compare(self): + destroyed = [] + compared = [] + + class Tag: + def __del__(self): + destroyed.append(True) + + def __eq__(self, other): + child.tag = "replaced" + return NotImplemented + + __hash__ = object.__hash__ + + class Sought: + def __eq__(self, other): + if isinstance(other, Tag): + compared.append(not destroyed) + return NotImplemented + + __hash__ = object.__hash__ + + root = cET.Element("root") + child = cET.SubElement(root, "x") + child.tag = Tag() + self.assertEqual(list(root.iter(Sought())), []) + self.assertEqual(compared, [True]) + self.assertEqual(child.tag, "replaced") + def test_dict_disappearing_during_get_item(self): # test fix for seg fault reported in issue 27946 class X: diff --git a/Misc/NEWS.d/next/Library/2026-09-23-21-15-00.gh-issue-158032.k4nT9p.rst b/Misc/NEWS.d/next/Library/2026-09-23-21-15-00.gh-issue-158032.k4nT9p.rst new file mode 100644 index 000000000000000..0e0a565519fa8e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-23-21-15-00.gh-issue-158032.k4nT9p.rst @@ -0,0 +1,2 @@ +Fix a crash in :meth:`xml.etree.ElementTree.Element.iter` when a tag's +equality comparison replaces that tag. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 983a00593b85d89..2195010261678f7 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2391,7 +2391,9 @@ elementiter_next(PyObject *op) if (it->sought_tag == Py_None) return (PyObject *)elem; - rc = PyObject_RichCompareBool(elem->tag, it->sought_tag, Py_EQ); + PyObject *tag = Py_NewRef(elem->tag); + rc = PyObject_RichCompareBool(tag, it->sought_tag, Py_EQ); + Py_DECREF(tag); if (rc > 0) return (PyObject *)elem; From ecefd40671f340d543c214b80744d4301d249f37 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 13 Nov 2026 01:50:45 +0200 Subject: [PATCH 122/229] gh-148467: Move BufferedIOBase close to concrete implementations (#150838) This reverts commit b6c7beb722deedc5125c81747af5c1f1204e2619. Co-authored-by: Cody Maloney --- Doc/library/io.rst | 149 +++++++++++++++++++++++---------------------- 1 file changed, 75 insertions(+), 74 deletions(-) diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 495505ec63f8d13..7a74f3c8b2c9d7c 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -526,6 +526,81 @@ I/O base classes write calls until all unwritten bytes are written. High-level I/O objects like :ref:`binary-io` and :ref:`text-io` implement retry behavior. + +Raw file I/O +^^^^^^^^^^^^ + +.. class:: FileIO(name, mode='r', closefd=True, opener=None) + + A raw binary stream representing an OS-level file containing bytes data. It + inherits from :class:`RawIOBase` and implements its low-level access design. + This means :meth:`~RawIOBase.write` does not guarantee all bytes are written + and :meth:`~RawIOBase.read` may read less bytes than requested even when more + bytes may be present in the underlying file. To get "write all" and + "read at least" behavior, use :ref:`binary-io`. + + The *name* can be one of two things: + + * a character string or :class:`bytes` object representing the path to the + file which will be opened. In this case closefd must be ``True`` (the default) + otherwise an error will be raised. + * an integer representing the number of an existing OS-level file descriptor + to which the resulting :class:`FileIO` object will give access. When the + FileIO object is closed this fd will be closed as well, unless *closefd* + is set to ``False``. + + The *mode* can be ``'r'``, ``'w'``, ``'x'`` or ``'a'`` for reading + (default), writing, exclusive creation or appending. The file will be + created if it doesn't exist when opened for writing or appending; it will be + truncated when opened for writing. :exc:`FileExistsError` will be raised if + it already exists when opened for creating. Opening a file for creating + implies writing, so this mode behaves in a similar way to ``'w'``. Add a + ``'+'`` to the mode to allow simultaneous reading and writing. + + A custom opener can be used by passing a callable as *opener*. The underlying + file descriptor for the file object is then obtained by calling *opener* with + (*name*, *flags*). *opener* must return an open file descriptor (passing + :mod:`os.open` as *opener* results in functionality similar to passing + ``None``). + + The newly created file is :ref:`non-inheritable `. + + See the :func:`open` built-in function for examples on using the *opener* + parameter. + + .. warning:: + :class:`FileIO` is a low-level I/O object and members, such as + :meth:`~RawIOBase.read` and :meth:`~RawIOBase.write`, need to have their + return values checked explicitly in a retry loop to implement "write all" + and "read at least" behavior. High-level I/O objects :ref:`binary-io` and + :ref:`text-io` implement retry behavior. + + .. versionchanged:: 3.3 + The *opener* parameter was added. + The ``'x'`` mode was added. + + .. versionchanged:: 3.4 + The file is now non-inheritable. + + :class:`FileIO` provides these data attributes in addition to those from + :class:`RawIOBase` and :class:`IOBase`: + + .. attribute:: mode + + The mode as given in the constructor. + + .. attribute:: name + + The file name. This is the file descriptor of the file when no name is + given in the constructor. + + +Buffered streams +^^^^^^^^^^^^^^^^ + +Buffered I/O streams provide a higher-level interface to an I/O device +than raw I/O does. + .. class:: BufferedIOBase Base class for binary streams that support some kind of buffering. @@ -647,80 +722,6 @@ I/O base classes so the implementation should only access *b* during the method call. -Raw file I/O -^^^^^^^^^^^^ - -.. class:: FileIO(name, mode='r', closefd=True, opener=None) - - A raw binary stream representing an OS-level file containing bytes data. It - inherits from :class:`RawIOBase` and implements its low-level access design. - This means :meth:`~RawIOBase.write` does not guarantee all bytes are written - and :meth:`~RawIOBase.read` may read less bytes than requested even when more - bytes may be present in the underlying file. To get "write all" and - "read at least" behavior, use :ref:`binary-io`. - - The *name* can be one of two things: - - * a character string or :class:`bytes` object representing the path to the - file which will be opened. In this case closefd must be ``True`` (the default) - otherwise an error will be raised. - * an integer representing the number of an existing OS-level file descriptor - to which the resulting :class:`FileIO` object will give access. When the - FileIO object is closed this fd will be closed as well, unless *closefd* - is set to ``False``. - - The *mode* can be ``'r'``, ``'w'``, ``'x'`` or ``'a'`` for reading - (default), writing, exclusive creation or appending. The file will be - created if it doesn't exist when opened for writing or appending; it will be - truncated when opened for writing. :exc:`FileExistsError` will be raised if - it already exists when opened for creating. Opening a file for creating - implies writing, so this mode behaves in a similar way to ``'w'``. Add a - ``'+'`` to the mode to allow simultaneous reading and writing. - - A custom opener can be used by passing a callable as *opener*. The underlying - file descriptor for the file object is then obtained by calling *opener* with - (*name*, *flags*). *opener* must return an open file descriptor (passing - :mod:`os.open` as *opener* results in functionality similar to passing - ``None``). - - The newly created file is :ref:`non-inheritable `. - - See the :func:`open` built-in function for examples on using the *opener* - parameter. - - .. warning:: - :class:`FileIO` is a low-level I/O object and members, such as - :meth:`~RawIOBase.read` and :meth:`~RawIOBase.write`, need to have their - return values checked explicitly in a retry loop to implement "write all" - and "read at least" behavior. High-level I/O objects :ref:`binary-io` and - :ref:`text-io` implement retry behavior. - - .. versionchanged:: 3.3 - The *opener* parameter was added. - The ``'x'`` mode was added. - - .. versionchanged:: 3.4 - The file is now non-inheritable. - - :class:`FileIO` provides these data attributes in addition to those from - :class:`RawIOBase` and :class:`IOBase`: - - .. attribute:: mode - - The mode as given in the constructor. - - .. attribute:: name - - The file name. This is the file descriptor of the file when no name is - given in the constructor. - - -Buffered streams -^^^^^^^^^^^^^^^^ - -Buffered I/O streams provide a higher-level interface to an I/O device -than raw I/O does. - .. class:: BytesIO(initial_bytes=b'') A binary stream using an in-memory bytes buffer. It inherits from From ab48ba8edf267d11397255e5a28e37f2ed1caeac Mon Sep 17 00:00:00 2001 From: rajorin <167426833+rajorin@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:00:16 +0530 Subject: [PATCH 123/229] gh-158907: Do not send proxy credentials to a direct origin on 407 (#158956) * gh-158907: Do not send proxy credentials to a direct origin on 407 ProxyBasicAuthHandler and ProxyDigestAuthHandler looked up passwords with the scheme-less origin host, so an HTTP 407 could match credentials stored for HTTPS and retry in cleartext. --------- Co-authored-by: Senthil Kumaran --- Lib/test/test_urllib2.py | 31 +++++++++++++++++++ Lib/urllib/request.py | 6 ++++ ...-10-07-13-30-00.gh-issue-158907.k4Qm2p.rst | 5 +++ 3 files changed, 42 insertions(+) create mode 100644 Misc/NEWS.d/next/Security/2026-10-07-13-30-00.gh-issue-158907.k4Qm2p.rst diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index a198a9de68b64f3..be1da0735b98fd2 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -2067,6 +2067,37 @@ def test_digest_auth_empty_path(self): self.assertIn('uri="/"', auth) self.assertIn('response="%s"' % respdig, auth) + def test_proxy_auth_ignores_direct_407(self): + # gh-158907: a direct HTTP origin that returns 407 must not receive + # credentials registered for the same authority over HTTPS. + realm = "test-realm" + cases = [ + (urllib.request.ProxyBasicAuthHandler, + 'Basic realm="%s"' % realm), + (urllib.request.ProxyDigestAuthHandler, + 'Digest realm="%s", nonce="abc", qop="auth"' % realm), + ] + for handler_class, challenge in cases: + with self.subTest(handler=handler_class.__name__): + opener = OpenerDirector() + opener.add_handler(urllib.request.ProxyHandler({})) + password_manager = urllib.request.HTTPPasswordMgr() + auth_handler = handler_class(password_manager) + http_handler = MockHTTPHandlerRedirect( + 407, 'Proxy-Authenticate: %s\r\n\r\n' % challenge) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) + + password_manager.add_password( + realm, "https://example.com/", + "victim-user", "victim-secret") + opener.open("http://example.com/resource") + + self.assertEqual(len(http_handler.requests), 1) + self.assertFalse( + http_handler.requests[0].has_header( + "Proxy-authorization")) + def test_basic_and_digest_auth_handlers(self): # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* # response (https://bugs.python.org/issue1479302), where it should instead diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index a5166caed586d74..26c27fd5b2c3ab6 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1051,6 +1051,9 @@ class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler): auth_header = 'Proxy-authorization' def http_error_407(self, req, fp, code, msg, headers): + # gh-158907: a 407 from a direct origin is not a proxy challenge + if not req.has_proxy() and not req._tunnel_host: + return None # http_error_auth_reqed requires that there is no userinfo component in # authority. Assume there isn't one, since urllib.request does not (and # should not, RFC 3986 s. 3.2.1) support requests for URLs containing @@ -1319,6 +1322,9 @@ class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler): handler_order = 490 # before Basic auth def http_error_407(self, req, fp, code, msg, headers): + # gh-158907: a 407 from a direct origin is not a proxy challenge + if not req.has_proxy() and not req._tunnel_host: + return None host = req.host retry = self.http_error_auth_reqed('proxy-authenticate', host, req, headers) diff --git a/Misc/NEWS.d/next/Security/2026-10-07-13-30-00.gh-issue-158907.k4Qm2p.rst b/Misc/NEWS.d/next/Security/2026-10-07-13-30-00.gh-issue-158907.k4Qm2p.rst new file mode 100644 index 000000000000000..67bbd3da1cbbe8d --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-10-07-13-30-00.gh-issue-158907.k4Qm2p.rst @@ -0,0 +1,5 @@ +:class:`~urllib.request.ProxyBasicAuthHandler` and +:class:`~urllib.request.ProxyDigestAuthHandler` no longer send credentials +in response to ``407`` from a direct origin. A scheme-less lookup against +the origin host could match passwords stored for an HTTPS URL and retry +the cleartext request with ``Proxy-Authorization``. From f345a1e24fb55a79954999b3baadecf072c0d049 Mon Sep 17 00:00:00 2001 From: Vyron Vasileiadis Date: Fri, 13 Nov 2026 12:56:50 +0200 Subject: [PATCH 124/229] gh-159637: Fix renaming an attribute to a prefixed name in minidom (GH-159638) Document.renameNode() set the prefix before the namespace URI, so the prefix was validated against the old namespace URI of the attribute and a valid rename was rejected with NamespaceErr. The attribute had already been removed from its element by then, so a rejected rename also lost it. Now the namespace URI is set first and the prefix is checked before the attribute is removed. --- Lib/test/test_minidom.py | 34 +++++++++++++++++++ Lib/xml/dom/minidom.py | 4 ++- ...-11-03-17-54-07.gh-issue-159637.0p15UL.rst | 4 +++ 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-03-17-54-07.gh-issue-159637.0p15UL.rst diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 8262f8305f7abb0..f50deaa2075d652 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -1952,6 +1952,28 @@ def testRenameAttribute(self): self.checkRenameNodeSharedConstraints(doc, attr) doc.unlink() + def testRenameAttributeToPrefixedName(self): + doc = parseString("") + elem = doc.documentElement + attr = elem.getAttributeNode("a") + attr = doc.renameNode(attr, "http://xml.python.org/ns", "p:c") + self.assertEqual(attr.name, "p:c") + self.assertEqual(attr.prefix, "p") + self.assertEqual(attr.localName, "c") + self.assertEqual(attr.namespaceURI, "http://xml.python.org/ns") + self.assertIsNone(elem.getAttributeNode("a")) + self.assertIs(elem.getAttributeNodeNS("http://xml.python.org/ns", "c"), + attr) + + # A rejected rename leaves the attribute on its element. + attr = elem.getAttributeNode("b") + self.assertRaises(xml.dom.NamespaceErr, doc.renameNode, attr, + "http://xml.python.org/ns", "xml:lang") + self.assertEqual(attr.name, "b") + self.assertIsNone(attr.namespaceURI) + self.assertIs(elem.getAttributeNode("b"), attr) + doc.unlink() + def testRenameElement(self): doc = parseString("") elem = doc.documentElement @@ -1995,6 +2017,18 @@ def testRenameElement(self): self.checkRenameNodeSharedConstraints(doc, elem) doc.unlink() + def testRenameElementToPrefixedName(self): + doc = parseString("") + elem = doc.documentElement + elem = doc.renameNode(elem, "http://xml.python.org/ns", "p:c") + self.assertEqual(elem.tagName, "p:c") + self.assertEqual(elem.nodeName, "p:c") + self.assertEqual(elem.prefix, "p") + self.assertEqual(elem.localName, "c") + self.assertEqual(elem.namespaceURI, "http://xml.python.org/ns") + self.assertIs(doc.documentElement, elem) + doc.unlink() + def testRenameOther(self): # We have to create a comment node explicitly since not all DOM # builders used with minidom add comments to the DOM. diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index 3ccc5761115117c..2bc5f8d3b71360e 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -2130,15 +2130,17 @@ def renameNode(self, n, namespaceURI, name): prefix = None localName = None if n.nodeType == Node.ATTRIBUTE_NODE: + if prefix is not None: + _check_prefix(prefix, namespaceURI, True) element = n.ownerElement if element is not None: is_id = n._is_id element.removeAttributeNode(n) else: element = None + n.namespaceURI = namespaceURI n.prefix = prefix n._localName = localName - n.namespaceURI = namespaceURI n.nodeName = name if n.nodeType == Node.ELEMENT_NODE: n.tagName = name diff --git a/Misc/NEWS.d/next/Library/2026-11-03-17-54-07.gh-issue-159637.0p15UL.rst b/Misc/NEWS.d/next/Library/2026-11-03-17-54-07.gh-issue-159637.0p15UL.rst new file mode 100644 index 000000000000000..cd820a2868f52ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-03-17-54-07.gh-issue-159637.0p15UL.rst @@ -0,0 +1,4 @@ +Fix :meth:`Document.renameNode() ` in +:mod:`xml.dom.minidom` rejecting a valid rename of an attribute to a +prefixed name, and removing the attribute from its element when a rename +is rejected. From d8d560b950489995c407c7488caabc6fa03cc84f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 13 Nov 2026 14:37:38 +0100 Subject: [PATCH 125/229] gh-145667: Update dis docs for GET_ITER, FOR_ITER and SEND changes (#161867) * gh-145667: Update dis docs for GET_ITER, FOR_ITER and SEND changes Document the (iterator, index or NULL) pair pushed by GET_ITER and consumed by FOR_ITER, SEND, END_SEND and POP_ITER, and the yield_from argument of GET_ITER that replaced GET_YIELD_FROM_ITER in 3.15. Co-Authored-By: Claude Opus 5.5 * Exclude removed GET_YIELD_FROM_ITER anchor from the HTML ID check Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- Doc/library/dis.rst | 51 ++++++++++++++++++++++++++++++--------- Doc/tools/removed-ids.txt | 1 + 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 5a93a328d487289..16c2dab5b8adba1 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -621,10 +621,14 @@ operations on it as if it was a Python list. The top of the stack corresponds to .. opcode:: POP_ITER - Removes the iterator from the top of the stack. + Removes the iterator and the value pushed with it by :opcode:`GET_ITER` + from the top of the stack. .. versionadded:: 3.14 + .. versionchanged:: 3.15 + Two values are now removed. + .. opcode:: POP_TOP @@ -644,11 +648,15 @@ operations on it as if it was a Python list. The top of the stack corresponds to .. opcode:: END_SEND - Implements ``del STACK[-2]``. + Implements ``del STACK[-3:-1]``. Used to clean up when a generator exits. .. versionadded:: 3.12 + .. versionchanged:: 3.15 + The receiver and the value pushed with it by :opcode:`GET_ITER` are + removed, instead of only the receiver. + .. opcode:: COPY (i) @@ -712,17 +720,23 @@ result back on the stack. Implements ``STACK[-1] = ~STACK[-1]``. -.. opcode:: GET_ITER - - Implements ``STACK[-1] = iter(STACK[-1])``. +.. opcode:: GET_ITER (yield_from) + Replaces ``STACK[-1]`` with an iterator for it and pushes a second value, + which is used by :opcode:`FOR_ITER`, :opcode:`SEND` and :opcode:`POP_ITER`. -.. opcode:: GET_YIELD_FROM_ITER + For some built-in sequence types, such as :class:`list` and :class:`tuple`, + the object itself is left on the stack as a "virtual iterator" and the + index to start iterating from is pushed. Otherwise, implements + ``STACK[-1] = iter(STACK[-1])`` and pushes ``NULL``. - If ``STACK[-1]`` is a :term:`generator iterator` or :term:`coroutine` object - it is left as is. Otherwise, implements ``STACK[-1] = iter(STACK[-1])``. + If *yield_from* is non-zero, the instruction is used for ``yield from``: + a :term:`coroutine` object is left as is, unless the current code is + a regular generator, in which case :exc:`TypeError` is raised. - .. versionadded:: 3.5 + .. versionchanged:: 3.15 + A second value is now pushed after the iterator. The *yield_from* + argument was added, replacing the ``GET_YIELD_FROM_ITER`` instruction. .. opcode:: TO_BOOL @@ -1425,7 +1439,10 @@ iterations of the loop. .. opcode:: FOR_ITER (delta) - ``STACK[-1]`` is an :term:`iterator`. Call its :meth:`~iterator.__next__` method. + ``STACK[-2]`` is an :term:`iterator` (or a virtual iterator, see + :opcode:`GET_ITER`) and ``STACK[-1]`` is the value pushed with it by + :opcode:`GET_ITER`. Get the next item, calling the iterator's + :meth:`~iterator.__next__` method if needed. If this yields a new value, push it on the stack (leaving the iterator below it). If the iterator indicates it is :term:`exhausted` then the byte code counter is incremented by *delta*. @@ -1433,6 +1450,10 @@ iterations of the loop. .. versionchanged:: 3.12 Up until 3.11 the iterator was popped when it was exhausted. + .. versionchanged:: 3.15 + The iterator is now followed on the stack by the value pushed with it + by :opcode:`GET_ITER`. + .. opcode:: LOAD_GLOBAL (namei) Loads the global named ``co_names[namei>>1]`` onto the stack. @@ -1810,8 +1831,10 @@ iterations of the loop. .. opcode:: SEND (delta) - Equivalent to ``STACK[-1] = STACK[-2].send(STACK[-1])``. Used in ``yield from`` - and ``await`` statements. + Equivalent to ``STACK[-1] = STACK[-3].send(STACK[-1])``. Used in ``yield from`` + and ``await`` statements. ``STACK[-2]`` is the value pushed with the + receiver by :opcode:`GET_ITER`; if the receiver is a virtual iterator, + it is advanced as by :opcode:`FOR_ITER`. If the call raises :exc:`StopIteration`, pop the top value from the stack, push the exception's ``value`` attribute, and increment the bytecode counter @@ -1819,6 +1842,10 @@ iterations of the loop. .. versionadded:: 3.11 + .. versionchanged:: 3.15 + The receiver is now at ``STACK[-3]``, followed by the value pushed with + it by :opcode:`GET_ITER`. + .. opcode:: HAVE_ARGUMENT diff --git a/Doc/tools/removed-ids.txt b/Doc/tools/removed-ids.txt index f846e4b8dc15249..bc9546eb0336b16 100644 --- a/Doc/tools/removed-ids.txt +++ b/Doc/tools/removed-ids.txt @@ -34,6 +34,7 @@ library/asyncio-task.html: terminating-a-task-group library/dis.html: opcode-DELETE_ATTR library/dis.html: opcode-DELETE_GLOBAL library/dis.html: opcode-DELETE_NAME +library/dis.html: opcode-GET_YIELD_FROM_ITER library/asyncio-llapi-index.html: event-loop-policies deprecations/index.html: pending-removal-in-python-3-15 From 537ce192e4d121b0581bd7c421b69f6ae29f4f71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 13 Nov 2026 15:09:56 +0100 Subject: [PATCH 126/229] GH-161739: Fix test_hashlib pbkdf2 tests on FIPS (#161748) To compensate the tests skipped on FIPS, add some tests with longer salt and more iterations. Co-authored-by: Victor Stinner --- Lib/test/test_hashlib.py | 69 ++++++++++++++++--- ...-11-10-14-49-01.gh-issue-161739.NZTbYC.rst | 2 + 2 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-11-10-14-49-01.gh-issue-161739.NZTbYC.rst diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index a718be415b99055..eff95fd8aa84f62 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -1236,6 +1236,11 @@ class KDFTests(unittest.TestCase): (b'passwordPASSWORDpassword', b'saltSALTsaltSALTsaltSALTsaltSALTsalt', 4096, -1), (b'pass\0word', b'sa\0lt', 4096, 16), + # more vectors that don't trip on self.fips_incompatible(...) + (b'password', b'salt'*4, 1024, None), + (b'password', b'salt'*4, 2048, None), + (b'password', b'salt'*4, 4096, None), + (b'pass\0word', b'sa\0lt'*4, 4096, 16), ] pbkdf2_results = { @@ -1247,7 +1252,14 @@ class KDFTests(unittest.TestCase): #(bytes.fromhex('eefe3d61cd4da4e4e9945b3d6ba2158c2634e984'), None), (bytes.fromhex('3d2eec4fe41c849b80c8d83662c0e44a8b291a964c' 'f2f07038'), 25), - (bytes.fromhex('56fa6aa75548099dcc37d7f03425e0c3'), None),], + (bytes.fromhex('56fa6aa75548099dcc37d7f03425e0c3'), None), + # more test vectors (not from RFC 6070) + (bytes.fromhex('1a39e29b04aa0488865b1e80fb125a8063e30b92'), None), + (bytes.fromhex('f19c46e6f912bc53a7b315ad4d3b717abe708c07'), None), + (bytes.fromhex('b7845a91aea7946bc875fcd9d19fd515381ddf94'), None), + (bytes.fromhex('bcc0d390b6671f99995e4c64223fa7c1fe12122460' + '94abe408'), 25), + ], "sha256": [ (bytes.fromhex('120fb6cffcf8b32c43e7225256c4f837' 'a86548c92ccc35480805987cb70be17b'), None), @@ -1259,7 +1271,16 @@ class KDFTests(unittest.TestCase): # 'f7f179e89b3b0bcb17ad10e3ac6eba46'), None), (bytes.fromhex('348c89dbcbd32b2f32d814b8116e84cf2b17' '347ebc1800181c4e2a1fb8dd53e1c635518c7dac47e9'), 40), - (bytes.fromhex('89b69d0516f829893c696226650a8687'), None),], + (bytes.fromhex('89b69d0516f829893c696226650a8687'), None), + (bytes.fromhex('72b62d16d04f2d4ce237e499167fa395' + '07191d6f4579bac8952aa7c74d24ff49'), None), + (bytes.fromhex('bd9d964fc9a92bb860829d795063ea08' + 'a5deccc6f4bdf295c86f8ad834ce5479'), None), + (bytes.fromhex('16e35b12b41f465ca64d069578d75005' + '9775ccad5df3ad53d6b204acb122d1ee'), None), + (bytes.fromhex('b6d737069a6b4aeed21574c73e38b342fa0f' + '4449112470e1e80f075611d8d0ef6a82669d662e15e2'), 40), + ], "sha512": [ (bytes.fromhex('867f70cf1ade02cff3752599a3a53dc4af34c7a669815ae5' 'd513554e1c8cf252c02d470a285a0501bad999bfe943c08f' @@ -1273,7 +1294,20 @@ class KDFTests(unittest.TestCase): (bytes.fromhex('8c0511f4c6e597c6ac6315d8f0362e225f3c501495ba23b8' '68c005174dc4ee71115b59f9e60cd9532fa33e0f75aefe30' '225c583a186cd82bd4daea9724a3d3b8'), 64), - (bytes.fromhex('9d9e9c4cd21fe4be24d5b8244c759665'), None),], + (bytes.fromhex('9d9e9c4cd21fe4be24d5b8244c759665'), None), + (bytes.fromhex('38284e5354120cfe567a1f6d2fbed512a613b3db36ba695e' + 'cf876a9d178ef55462986f2df5eb71f1b90ae9fbc4211b29' + '6405f882b910062d854d28953f90bde6'), None), + (bytes.fromhex('e41733921704084e8d31f823846c6d98ad3597a5035ce141' + 'f51963c08630a33cc2a1e263ed7a33b1e63e83adaf3ad912' + '6c0f5b5496cfb2d29e7cd35ef9aa39c5'), None), + (bytes.fromhex('7dc69993c22ab444a13574b9d71b2cafc54e68914d9ddfd0' + '0ed5c23545b854d5e0e68b7f5614eef5df9087573bab8360' + 'b20267791ed7a77669005fd94b5731e2'), None), + (bytes.fromhex('a7555be88503f21c2314259add27d302db6bb03b6ea52b36' + '777f1af2e91f802c72b2b02b76745d7e0c3239e84cc8451c' + '5188d526a56c740f5f0f7c85c44717e7'), 64), + ], } def _test_pbkdf2_hmac(self, pbkdf2, supported): @@ -1282,21 +1316,23 @@ def _test_pbkdf2_hmac(self, pbkdf2, supported): continue for i, vector in enumerate(self.pbkdf2_test_vectors): password, salt, rounds, dklen = vector + if self.fips_incompatible(password, salt, rounds): + continue expected, overwrite_dklen = results[i] if overwrite_dklen: dklen = overwrite_dklen out = pbkdf2(digest_name, password, salt, rounds, dklen) - self.assertEqual(out, expected, + self.assertEqual(out.hex(), expected.hex(), (digest_name, password, salt, rounds, dklen)) out = pbkdf2(digest_name, memoryview(password), memoryview(salt), rounds, dklen) - self.assertEqual(out, expected) + self.assertEqual(out.hex(), expected.hex()) out = pbkdf2(digest_name, bytearray(password), bytearray(salt), rounds, dklen) - self.assertEqual(out, expected) + self.assertEqual(out.hex(), expected.hex()) if dklen is None: out = pbkdf2(digest_name, password, salt, rounds) - self.assertEqual(out, expected, + self.assertEqual(out.hex(), expected.hex(), (digest_name, password, salt, rounds)) with self.assertRaisesRegex(ValueError, '.*unsupported.*'): @@ -1321,9 +1357,22 @@ def _test_pbkdf2_hmac(self, pbkdf2, supported): self.assertRaises( ValueError, pbkdf2, 'sha1', b'pass', b'salt', 1, -1 ) - out = pbkdf2(hash_name='sha1', password=b'password', salt=b'salt', - iterations=1, dklen=None) - self.assertEqual(out, self.pbkdf2_results['sha1'][0][0]) + # Pick a single FIPS compatible vector + index = 5 + password, salt, rounds, dklen = self.pbkdf2_test_vectors[index] + self.assertFalse(self.fips_incompatible(password, salt, rounds)) + out = pbkdf2(hash_name='sha1', password=password, salt=salt, + iterations=rounds, dklen=dklen) + expected = self.pbkdf2_results['sha1'][index][0] + self.assertEqual(out.hex(), expected.hex()) + + # FIPS (NIST SP 800-132) needs: + # - password of at least 64 bits (8 bytes) + # - salt of at least 128 bits (16 bytes) + # - rounds of least 1000 + def fips_incompatible(self, password, salt, rounds): + return (get_fips_mode() + and (len(password) < 8 or len(salt) < 16 or rounds < 1000)) @unittest.skipIf(openssl_hashlib is None, "requires OpenSSL bindings") def test_pbkdf2_hmac_c(self): diff --git a/Misc/NEWS.d/next/Tests/2026-11-10-14-49-01.gh-issue-161739.NZTbYC.rst b/Misc/NEWS.d/next/Tests/2026-11-10-14-49-01.gh-issue-161739.NZTbYC.rst new file mode 100644 index 000000000000000..a480b9788f7574e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-11-10-14-49-01.gh-issue-161739.NZTbYC.rst @@ -0,0 +1,2 @@ +``test_hashlibtest_pbkdf2_hmac_c`` from ``test_haslib`` no longer fails with +``ValueError: [Provider routines] invalid salt length`` with FIPS enabled. From 11eed559c09ecfaddd87dbf9c10fbe742f0f57de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Delfino?= Date: Fri, 13 Nov 2026 13:16:22 -0300 Subject: [PATCH 127/229] gh-161452: Fix `python -m random` with a non-positive integer input (#161453) Fix the random command-line interface printing a random float for a zero or negative integer input instead of failing like --integer does. Co-authored-by: Claude Opus 5.5 Co-authored-by: Victor Stinner --- Lib/random.py | 20 ++++++++---- Lib/test/test_random.py | 32 ++++++++++++------- ...-11-05-22-26-44.gh-issue-161452.YfFfjq.rst | 3 ++ 3 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-05-22-26-44.gh-issue-161452.YfFfjq.rst diff --git a/Lib/random.py b/Lib/random.py index 7db761034509d37..a6a5784e8b51a40 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -1041,18 +1041,24 @@ def _parse_args(arg_list: list[str] | None): integer: same as `--integer` float: same as `--float`""") args = parser.parse_args(arg_list) - return args, parser.format_help() + return args, parser + + +def _randint_cli(parser, n: int) -> int: + if n < 1: + parser.error(f"integer must be 1 or greater, got {n}") + return randint(1, n) def main(arg_list: list[str] | None = None) -> int | str: - args, help_text = _parse_args(arg_list) + args, parser = _parse_args(arg_list) # Explicit arguments if args.choice: return choice(args.choice) if args.integer is not None: - return randint(1, args.integer) + return _randint_cli(parser, args.integer) if args.float is not None: return uniform(0, args.float) @@ -1067,20 +1073,22 @@ def main(arg_list: list[str] | None = None) -> int | str: try: # Is it an integer? val = int(val) - return randint(1, val) except ValueError: try: # Is it a float? val = float(val) - return uniform(0, val) except ValueError: # Split in case of space-separated string: "a b c" return choice(val.split()) + else: + return uniform(0, val) + else: + return _randint_cli(parser, val) if len(args.input) >= 2: return choice(args.input) - return help_text + return parser.format_help() if __name__ == '__main__': diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 8d093ab1b7014a4..80af93d6e45ba20 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -1443,29 +1443,23 @@ def test_after_fork(self): class CommandLineTest(unittest.TestCase): @support.force_not_colorized def test_parse_args(self): - args, help_text = random._parse_args(shlex.split("--choice a b c")) + args, parser = random._parse_args(shlex.split("--choice a b c")) self.assertEqual(args.choice, ["a", "b", "c"]) - self.assertStartsWith(help_text, "usage: ") - args, help_text = random._parse_args(shlex.split("--integer 5")) + args, parser = random._parse_args(shlex.split("--integer 5")) self.assertEqual(args.integer, 5) - self.assertStartsWith(help_text, "usage: ") - args, help_text = random._parse_args(shlex.split("--float 2.5")) + args, parser = random._parse_args(shlex.split("--float 2.5")) self.assertEqual(args.float, 2.5) - self.assertStartsWith(help_text, "usage: ") - args, help_text = random._parse_args(shlex.split("a b c")) + args, parser = random._parse_args(shlex.split("a b c")) self.assertEqual(args.input, ["a", "b", "c"]) - self.assertStartsWith(help_text, "usage: ") - args, help_text = random._parse_args(shlex.split("5")) + args, parser = random._parse_args(shlex.split("5")) self.assertEqual(args.input, ["5"]) - self.assertStartsWith(help_text, "usage: ") - args, help_text = random._parse_args(shlex.split("2.5")) + args, parser = random._parse_args(shlex.split("2.5")) self.assertEqual(args.input, ["2.5"]) - self.assertStartsWith(help_text, "usage: ") def test_main(self): for command, expected in [ @@ -1482,6 +1476,20 @@ def test_main(self): random.seed(0) self.assertEqual(random.main(shlex.split(command)), expected) + @support.force_not_colorized + def test_empty_integer_range_error(self): + # An integer input is the same as --integer, even when it is + # not positive. + for value in ["0", "-5"]: + err_msg = f"error: integer must be 1 or greater, got {value}\n" + for args in [[value], ["--integer", value]]: + with self.subTest(args=args): + with (support.captured_stderr() as stderr, + self.assertRaises(SystemExit) as cm): + random.main(args) + self.assertEqual(cm.exception.code, 2) + self.assertEndsWith(stderr.getvalue(), err_msg) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-11-05-22-26-44.gh-issue-161452.YfFfjq.rst b/Misc/NEWS.d/next/Library/2026-11-05-22-26-44.gh-issue-161452.YfFfjq.rst new file mode 100644 index 000000000000000..7ac5f49096c5401 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-05-22-26-44.gh-issue-161452.YfFfjq.rst @@ -0,0 +1,3 @@ +Fix the :mod:`random` :ref:`command-line interface ` +printing a random float for a zero or negative integer input instead of +failing like ``--integer`` does. From 727b56b4db5734864953a6586ea03633ef27bad5 Mon Sep 17 00:00:00 2001 From: Neko Asakura Date: Fri, 13 Nov 2026 11:27:18 -0500 Subject: [PATCH 128/229] gh-160436: Skip `PyStatus.exitcode` deprecation on clang++ 21 and older (#161341) --- Include/cpython/initconfig.h | 5 +++++ .../C_API/2026-11-04-14-24-36.gh-issue-160436.WmZEYT.rst | 4 ++++ 2 files changed, 9 insertions(+) create mode 100644 Misc/NEWS.d/next/C_API/2026-11-04-14-24-36.gh-issue-160436.WmZEYT.rst diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 846e05fb26a4d83..e1bdc6174d9de06 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -15,7 +15,12 @@ typedef struct { } _type; const char *func; const char *err_msg; +#if defined(__cplusplus) && defined(__clang__) && __clang_major__ < 22 + // gh-160436: clang < 22 warns in the implicit copy/move members + int exitcode; +#else Py_DEPRECATED(3.16) int exitcode; +#endif } PyStatus; PyAPI_FUNC(PyStatus) PyStatus_Ok(void); diff --git a/Misc/NEWS.d/next/C_API/2026-11-04-14-24-36.gh-issue-160436.WmZEYT.rst b/Misc/NEWS.d/next/C_API/2026-11-04-14-24-36.gh-issue-160436.WmZEYT.rst new file mode 100644 index 000000000000000..4cef63faaa4cdf2 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-11-04-14-24-36.gh-issue-160436.WmZEYT.rst @@ -0,0 +1,4 @@ +Fix a spurious ``-Wdeprecated-declarations`` warning when a +:c:type:`PyStatus` is copied in C++ with clang 21 and older: +:c:member:`PyStatus.exitcode` is no longer marked as deprecated for C++ on +those compilers. From 2c2d8018a20da33c749f42cc6a4ae55664a88321 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Fri, 13 Nov 2026 20:04:20 +0300 Subject: [PATCH 129/229] gh-142662: Fix interpreter startup crash when `codecs` are not initialized yet (#161860) --- Python/codecs.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Python/codecs.c b/Python/codecs.c index ba6f618f77e6f6c..6eb728e7f9f202a 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -139,7 +139,10 @@ PyObject *_PyCodec_Lookup(const char *encoding) } PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(interp->codecs.initialized); + if (!interp->codecs.initialized) { + PyErr_SetString(PyExc_SystemError, "Codecs are not initialized"); + return NULL; + } /* Convert the encoding to a normalized Python string: all ASCII letters are converted to lower case, spaces are From 5151e7f2a3dabf74941f2c3857e17886d87cb5dd Mon Sep 17 00:00:00 2001 From: Kasra Ghodsi Date: Fri, 13 Nov 2026 15:49:58 -0800 Subject: [PATCH 130/229] gh-152845: Preserve ZIP encoding flags when appending (gh-161950) --- Lib/test/test_zipfile/test_core.py | 60 +++++++++++++++++++ Lib/zipfile/__init__.py | 8 ++- ...1-13-09-49-13.gh-issue-152845.ZipFlags.rst | 3 + 3 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-13-09-49-13.gh-issue-152845.ZipFlags.rst diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 5e098720df77010..69203142f61589f 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -4539,6 +4539,66 @@ def test_empty_zipfile(self): self.fail("Unable to create empty ZIP file in 'a' mode") zipf.close() + def test_append_preserves_ascii_utf8_flag(self): + comments = (b'', 'caf\N{LATIN SMALL LETTER E WITH ACUTE}'.encode('utf-8')) + updates = ( + ('append', lambda zf: zf.writestr('added.txt', b'added')), + ('comment', lambda zf: setattr(zf, 'comment', b'archive comment')), + ) + for comment, (operation, update) in itertools.product(comments, updates): + with self.subTest(comment=comment, operation=operation): + stream = io.BytesIO() + with zipfile.ZipFile(stream, 'w') as archive: + entry = zipfile.ZipInfo('file.txt') + entry.comment = comment + archive.writestr(entry, b'payload') + original = bytearray(stream.getvalue()) + central_offset = original.index(b'PK\x01\x02') + struct.pack_into(' Date: Sat, 14 Nov 2026 00:59:53 +0100 Subject: [PATCH 131/229] gh-159739: Add a test on _PyTuple_Resize() error (#161957) Check that no reference is leaked when _PyTuple_Resize() fails. --- Lib/test/test_capi/test_tuple.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index 3f2318f303d57ae..1f8f5da0e51e909 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -1,6 +1,7 @@ import gc import sys import unittest +from test import support from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') @@ -364,6 +365,21 @@ def resize(tup, new_size, new=True, compute_hash=False): with self.assertRaises(SystemError): resize(a, 3, True, compute_hash=True) + # Test _PyTuple_Resize() failure: inject MemoryError on + # PyObject_GC_Resize(). Check that the test doesn't leak references + # if the test is run with -R 3:3. + a = tuple(object() for _ in range(10)) + got_memory_error = False + # Test different start values since _testcapi _tuple_resize() also + # allocates memory (to copy the tuple). + for start in range(5): + try: + with support.inject_memory_error_cm(start=start): + resize(a, 30) + except MemoryError: + got_memory_error = True + self.assertTrue(got_memory_error) + a = () self.assertRaises(MemoryError, resize, a, PY_SSIZE_T_MAX) self.assertRaises(SystemError, resize, a, -1) From af106b8f558e020fa4bbdd27577b4417354e41f1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 14 Nov 2026 01:24:32 +0100 Subject: [PATCH 132/229] gh-161932: Skip test_ssl ECH test with FIPS enabled (#161939) ECH hasn't been included in FIPS mode in OpenSSL so far. --- Lib/test/test_ssl.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 19a9f98c316ab3a..837445dc06d67b9 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -40,6 +40,10 @@ import ctypes except ImportError: ctypes = None +try: + from _hashlib import get_fips_mode +except ImportError: + def get_fips_mode(): return False ssl = import_helper.import_module("ssl") @@ -2478,6 +2482,9 @@ def test_outer_alpn_does_not_replace_inner(self): self.assertEqual(server.selected_alpn_protocol(), 'h2') @unittest.skipUnless(ssl.HAS_ECH, 'requires ECH') + # At 2026-11-13 (OpenSSL 4.0.3 on Fedora 45), the test is incompatible + # with FIPS mode: https://github.com/openssl/openssl/issues/33724 + @unittest.skipIf(get_fips_mode(), 'gh-161932: test incompatible with FIPS') def test_outer_alpn_without_ech_config(self): for options in (0, ssl.OP_ECH_GREASE): client_ctx, server_ctx, hostname = testing_context() @@ -2491,6 +2498,9 @@ def test_outer_alpn_without_ech_config(self): self.assertEqual(server.selected_alpn_protocol(), 'h2') @unittest.skipUnless(ssl.HAS_ECH, 'requires ECH') + # At 2026-11-13 (OpenSSL 4.0.3 on Fedora 45), the test is incompatible + # with FIPS mode: https://github.com/openssl/openssl/issues/33724 + @unittest.skipIf(get_fips_mode(), 'gh-161932: test incompatible with FIPS') def test_grease(self): client_ctx, server_ctx, hostname = testing_context() client_ctx.options |= ssl.OP_ECH_GREASE From f4fbacf64a094e4ae8a8052b28331a3eee178035 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 14 Nov 2026 01:31:19 +0100 Subject: [PATCH 133/229] Enhance encodings in regrtest header (#161959) If the UTF-8 mode is disabled and/or if PYTHONCOERCECLOCALE environment variable is set to "0": mention it in the "== encodings" regrtest header. If the locale encoding and the filesystem encoding are equal, log "locale=FS=" to have a shorter output. On Windows, log ANSI and OEM code pages. --- Lib/test/libregrtest/utils.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 41a798c71089f9e..28e6d904fa28aab 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -665,8 +665,32 @@ def display_header(use_resources: dict[str, str | None], if process_cpu_count and process_cpu_count != cpu_count: cpu_count = f"{process_cpu_count} (process) / {cpu_count} (system)" print("== CPU count:", cpu_count) - print("== encodings: locale=%s FS=%s" - % (locale.getencoding(), sys.getfilesystemencoding())) + + encodings = [] + if _winapi is not None: + # Log ANSI and OEM code pages + ansi = f'cp{_winapi.GetACP()}' + oem = f'cp{_winapi.GetOEMCP()}' + if ansi == oem: + encodings.append(f'ansi=oem={ansi}') + else: + encodings.append(f'ansi={ansi}') + encodings.append(f'oem={oem}') + encodings.append(f'FS={sys.getfilesystemencoding()}') + else: + locale_encoding = locale.getencoding() # nl_langinfo(CODESET) + fs_encoding = sys.getfilesystemencoding() # normalized name + if locale_encoding.lower() == fs_encoding.lower(): + encodings.append(f'locale=FS={fs_encoding}') + else: + encodings.append(f'locale={locale_encoding}') + encodings.append(f'FS={fs_encoding}') + if not sys.flags.utf8_mode: + encodings.append('utf8=0') + if (not sys.flags.ignore_environment + and os.environ.get('PYTHONCOERCECLOCALE') == '0'): + encodings.append('coerce=0') + print(f"== encodings: {' '.join(encodings)}") if use_resources: text = format_resources(use_resources) From ba935775f31b841f4eaa39095051f1a25aaad959 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 14 Nov 2026 01:49:07 +0100 Subject: [PATCH 134/229] Log Py_LIMITED_API and Py_TARGET_ABI3T in test_cext (#161958) --- Lib/test/test_cext/extension.c | 12 ++++++++++++ Lib/test/test_cext/runtests.py | 13 +++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 65dd25d71f494be..b462a949bd98991 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -483,6 +483,18 @@ module_exec(PyObject *module) return -1; } #endif + +#ifdef Py_LIMITED_API + if (PyModule_AddIntMacro(module, Py_LIMITED_API) < 0) { + return -1; + } +#endif +#ifdef Py_TARGET_ABI3T + if (PyModule_AddIntMacro(module, Py_TARGET_ABI3T) < 0) { + return -1; + } +#endif + // Ignore "unused argument" warning when none of these macros is defined (void)module; return 0; diff --git a/Lib/test/test_cext/runtests.py b/Lib/test/test_cext/runtests.py index 6700b410b7d7a78..cb2725209de7dc8 100644 --- a/Lib/test/test_cext/runtests.py +++ b/Lib/test/test_cext/runtests.py @@ -63,14 +63,19 @@ def main(): '__STRICT_ANSI__', '__cplusplus', '_MSVC_LANG', + 'Py_LIMITED_API', + 'Py_TARGET_ABI3T', ): try: value = getattr(testmod, name) except AttributeError: - pass - else: - print(f'{name}: {value}') - newline = True + continue + + if name in ('Py_LIMITED_API', 'Py_TARGET_ABI3T'): + value = f'0x{value:08x}' + + print(f'{name}: {value}') + newline = True if newline: print() From 9b633f55e1124697372caaec319a3b7583531a74 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 13 Nov 2026 23:24:38 -0500 Subject: [PATCH 135/229] gh-135893: Lazily initialize the libffi allocator in ctypes (GH-161844) --- Lib/test/test_ctypes/test_callbacks.py | 14 ++++++++++ ...-11-11-12-00-00.gh-issue-135893.cTyp3s.rst | 3 ++ Modules/_ctypes/_ctypes.c | 12 -------- Modules/_ctypes/callbacks.c | 28 +++++++++++++++++++ Tools/c-analyzer/cpython/ignored.tsv | 1 + 5 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-11-12-00-00.gh-issue-135893.cTyp3s.rst diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index c7f287d9fb6293d..7d746749130db25 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -15,6 +15,7 @@ from ctypes.util import find_library, struct, wrap_dll_function from test import support from test.support import import_helper +from test.support import script_helper _ctypes_test = import_helper.import_module("_ctypes_test") @@ -559,6 +560,19 @@ def cb(): self.assertEqual(_testfunc_callback_int_to_longlong(cb), -1) + def test_import_does_not_open_fds(self): + # gh-135893: Importing ctypes must not allocate a libffi closure, + # which can leave a file descriptor open for the rest of the + # process' lifetime. + code = """if 1: + from test.support import os_helper + before = os_helper.fd_count() + import ctypes + after = os_helper.fd_count() + assert before == after + """ + script_helper.assert_python_ok("-c", code) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-11-11-12-00-00.gh-issue-135893.cTyp3s.rst b/Misc/NEWS.d/next/Library/2026-11-11-12-00-00.gh-issue-135893.cTyp3s.rst new file mode 100644 index 000000000000000..bddd3d817ab1cbf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-11-12-00-00.gh-issue-135893.cTyp3s.rst @@ -0,0 +1,3 @@ +Fix a regression where importing :mod:`ctypes` could leave an extra file +descriptor open on some Linux systems, or hang in x86-64 macOS applications +signed with the hardened runtime. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 56d39e29f140dd0..217b5cfedf2adb0 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -6553,18 +6553,6 @@ _ctypes_add_version_constants(PyObject *mod) static int _ctypes_mod_exec(PyObject *mod) { - // See https://github.com/python/cpython/issues/128485 - // This allocates some memory and then frees it to ensure that the - // the dlmalloc allocator initializes itself to avoid data races - // in free-threading. - void *codeloc = NULL; - void *ptr = Py_ffi_closure_alloc(sizeof(void *), &codeloc); - if (ptr == NULL) { - PyErr_NoMemory(); - return -1; - } - Py_ffi_closure_free(ptr); - _ctypes_init_fielddesc(); ctypes_state *st = get_module_state(mod); diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 8395e9eaf3505ec..e54111b8a4fab44 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -9,6 +9,7 @@ #endif #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_lock.h" // _PyOnceFlag_CallOnce() #include "pycore_runtime.h" // _Py_ID() #include @@ -412,6 +413,27 @@ static CThunkObject* CThunkObject_new(ctypes_state *st, Py_ssize_t nargs) return p; } +static _PyOnceFlag closure_alloc_init_once = {0}; + +// This is done lazily with a once flag rather than in the module exec function, +// because allocating a closure can have some side effects that affect the whole +// process (gh-135893). +static int +closure_alloc_init(void *Py_UNUSED(arg)) +{ + // See https://github.com/python/cpython/issues/128485 + // This allocates some memory and then frees it to ensure that libffi's + // dlmalloc allocator initializes itself exactly once, avoiding data races + // between threads that don't share a GIL. + void *codeloc = NULL; + void *ptr = Py_ffi_closure_alloc(sizeof(void *), &codeloc); + if (ptr == NULL) { + return -1; + } + Py_ffi_closure_free(ptr); + return 0; +} + CThunkObject *_ctypes_alloc_callback(ctypes_state *st, PyObject *callable, PyObject *converters, @@ -431,6 +453,12 @@ CThunkObject *_ctypes_alloc_callback(ctypes_state *st, assert(CThunk_CheckExact(st, (PyObject *)p)); + if (_PyOnceFlag_CallOnce(&closure_alloc_init_once, + closure_alloc_init, NULL) < 0) { + PyErr_NoMemory(); + goto error; + } + p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); if (p->pcl_write == NULL) { PyErr_NoMemory(); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 3863872fd542624..bda6ce14e8bcd30 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -26,6 +26,7 @@ Modules/posixmodule.c os_pipe_impl pipe2_works_atomic - ## guards around resource init Python/thread_pthread.h PyThread__init_thread lib_initialized - Modules/_struct.c - endian_tables_init_once - +Modules/_ctypes/callbacks.c - closure_alloc_init_once - ##----------------------- ## other values (not Python-specific) From 713c16ec40960b800a473b5be5483c7b007bea24 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sat, 14 Nov 2026 08:36:06 +0300 Subject: [PATCH 136/229] gh-101703: Use `PyOS_snprintf` in `_freeze_module` (#161871) --- Programs/_freeze_module.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Programs/_freeze_module.c b/Programs/_freeze_module.c index 27a60171f3eca86..ae134f6caef1d59 100644 --- a/Programs/_freeze_module.c +++ b/Programs/_freeze_module.c @@ -122,11 +122,12 @@ read_text(const char *inpath) static PyObject * compile_and_marshal(const char *name, const char *text) { - char *filename = (char *) malloc(strlen(name) + 10); + int text_size = strlen(name) + 10; + char *filename = (char *) malloc(text_size); if (filename == NULL) { return PyErr_NoMemory(); } - sprintf(filename, "", name); + PyOS_snprintf(filename, text_size, "", name); PyObject *code = Py_CompileStringExFlags(text, filename, Py_file_input, NULL, 0); free(filename); From 797087c386dfaeaafe7fdd741194f10100f3ac13 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sat, 14 Nov 2026 08:42:42 +0300 Subject: [PATCH 137/229] gh-160669: Clarify `PyDict_Update` docs (#161314) Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner --- Doc/c-api/dict.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 6db8dfca8a80db7..e986c8b6456b749 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -472,11 +472,14 @@ Dictionary objects .. c:function:: int PyDict_Update(PyObject *a, PyObject *b) - This is the same as ``PyDict_Merge(a, b, 1)`` in C, and is similar to - ``a.update(b)`` in Python except that :c:func:`PyDict_Update` doesn't fall - back to the iterating over a sequence of key value pairs if the second - argument has no "keys" attribute. Return ``0`` on success or ``-1`` if an - exception was raised. + This is the same as ``PyDict_Merge(a, b, 1)`` in C. + + Similar to ``a.update(b)`` in Python, except that this function + does not fall back to the iterating over a sequence of key-value pairs + if *b* is not a mapping. + Use :c:func:`PyDict_MergeFromSeq2` for this case. + + Return ``0`` on success or ``-1`` if an exception was raised. .. note:: From 91569eb2398d494c71fd0abb48fe7e5f2cfed1ea Mon Sep 17 00:00:00 2001 From: Shamil Date: Sat, 14 Nov 2026 10:34:11 +0300 Subject: [PATCH 138/229] gh-160247: Reduce padding in _PyInstructionSequence (#160271) --- Include/internal/pycore_instruction_sequence.h | 2 +- .../2026-10-22-20-18-44.gh-issue-160247.WC8hRU.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-20-18-44.gh-issue-160247.WC8hRU.rst diff --git a/Include/internal/pycore_instruction_sequence.h b/Include/internal/pycore_instruction_sequence.h index 6257b771fd9539c..1643776c47ee697 100644 --- a/Include/internal/pycore_instruction_sequence.h +++ b/Include/internal/pycore_instruction_sequence.h @@ -40,8 +40,8 @@ typedef struct instruction_sequence { /* Map of a label id to instruction offset (index into s_instrs). * If s_labelmap is NULL, then each label id is the offset itself. */ - int *s_labelmap; int s_labelmap_size; + int *s_labelmap; /* PyList of instruction sequences of nested functions */ PyObject *s_nested; diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-20-18-44.gh-issue-160247.WC8hRU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-20-18-44.gh-issue-160247.WC8hRU.rst new file mode 100644 index 000000000000000..97755e25aac7c26 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-20-18-44.gh-issue-160247.WC8hRU.rst @@ -0,0 +1,2 @@ +Reduce padding in the internal instruction sequence structure, saving 8 bytes +per instance on 64-bit platforms. Patch by Shamil Abdulaev. From 1d6419572978146dd52dbf9fd9521a084ba1cd6c Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sat, 14 Nov 2026 12:55:25 +0300 Subject: [PATCH 139/229] gh-101703: Use `PyOS_snprintf` in `_datetimemodule.c` (#161982) --- Modules/_datetimemodule.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index b9939d5cc71a319..091da059afd34b4 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1870,12 +1870,17 @@ static PyObject * make_freplacement(PyObject *object) { char freplacement[64]; - if (PyTime_Check(object)) - sprintf(freplacement, "%06d", TIME_GET_MICROSECOND(object)); - else if (PyDateTime_Check(object)) - sprintf(freplacement, "%06d", DATE_GET_MICROSECOND(object)); - else - sprintf(freplacement, "%06d", 0); + if (PyTime_Check(object)) { + PyOS_snprintf(freplacement, sizeof(freplacement), + "%06d", TIME_GET_MICROSECOND(object)); + } + else if (PyDateTime_Check(object)) { + PyOS_snprintf(freplacement, sizeof(freplacement), + "%06d", DATE_GET_MICROSECOND(object)); + } + else { + PyOS_snprintf(freplacement, sizeof(freplacement), "%06d", 0); + } return PyUnicode_FromString(freplacement); } From fb08c03d5c49d0cd43ee0afac302609c8ab0b361 Mon Sep 17 00:00:00 2001 From: David Andrews Date: Sat, 14 Nov 2026 06:50:15 -0500 Subject: [PATCH 140/229] gh-161968: Fall back to the right operand's `sq_repeat` in `PyNumber_InPlaceMultiply` (#161969) Co-authored-by: Stan Ulbrych --- Lib/test/test_augassign.py | 19 +++++++++++++++++++ ...-11-13-00-00-00.gh-issue-161968.ytuf5z.rst | 2 ++ Objects/abstract.c | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-13-00-00-00.gh-issue-161968.ytuf5z.rst diff --git a/Lib/test/test_augassign.py b/Lib/test/test_augassign.py index 5930d9e7a29fa2a..8cef5a970a52138 100644 --- a/Lib/test/test_augassign.py +++ b/Lib/test/test_augassign.py @@ -64,6 +64,25 @@ def testSequences(self): self.assertEqual(x, [1, 2, 1, 2, 3]) self.assertTrue(x is y) + def testSequenceRepeatRightOperand(self): + # gh-161968 + class MyInt(int): + pass + class Index: + def __index__(self): + return 2 + + seq = [1] + x = MyInt(2) + x *= seq + self.assertEqual(x, [1, 1]) + # The right operand is not modified in place. + self.assertEqual(seq, [1]) + + x = Index() + x *= 'a' + self.assertEqual(x, 'aa') + def testCustomMethods1(self): class aug_test: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-13-00-00-00.gh-issue-161968.ytuf5z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-13-00-00-00.gh-issue-161968.ytuf5z.rst new file mode 100644 index 000000000000000..12d61b2eeee74e8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-13-00-00-00.gh-issue-161968.ytuf5z.rst @@ -0,0 +1,2 @@ +Fix ``x *= seq`` raising :exc:`TypeError` when ``x`` is an instance of an +:class:`int` subclass or of a class defining :meth:`~object.__index__`. diff --git a/Objects/abstract.c b/Objects/abstract.c index da2a0b0c285b52b..6bb1ac572b0c6e8 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1344,7 +1344,7 @@ PyNumber_InPlaceMultiply(PyObject *v, PyObject *w) if (f != NULL) return sequence_repeat(f, v, w); } - else if (mw != NULL) { + if (mw != NULL) { /* Note that the right hand operand should not be * mutated in this case so sq_inplace_repeat is not * used. */ From 69ac138c06645085c83279042b3b186eac824fd3 Mon Sep 17 00:00:00 2001 From: An Long Date: Sat, 14 Nov 2026 21:13:00 +0900 Subject: [PATCH 141/229] gh-155181: Untrack frozensets created from set literals when possible (gh-155182) --- Lib/test/test_set.py | 11 +++++++++++ .../2026-08-05-01-10-20.gh-issue-155181.jj8jhk.rst | 3 +++ Objects/setobject.c | 1 + 3 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-01-10-20.gh-issue-155181.jj8jhk.rst diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 9bfd4bc7d636699..54d229b71def9a7 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -762,6 +762,17 @@ def test_constructor_identity(self): t = self.thetype(s) self.assertEqual(id(s), id(t)) + def test_gc_tracking(self): + # gh-140232: frozensets whose elements can never be tracked by the + # GC are not tracked, including the compiled frozenset({...}) form + s = {1, 2} + self.assertFalse(gc.is_tracked(frozenset(s))) + self.assertFalse(gc.is_tracked(frozenset({1, 2}))) + self.assertFalse(gc.is_tracked(frozenset({x for x in range(2)}))) + class C: + pass + self.assertTrue(gc.is_tracked(frozenset({C()}))) + def test_hash(self): self.assertEqual(hash(self.thetype('abcdeb')), hash(self.thetype('ebecda'))) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-01-10-20.gh-issue-155181.jj8jhk.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-01-10-20.gh-issue-155181.jj8jhk.rst new file mode 100644 index 000000000000000..1acf8fda95e4652 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-01-10-20.gh-issue-155181.jj8jhk.rst @@ -0,0 +1,3 @@ +:class:`frozenset` objects created from a set literal or a set comprehension +are now untracked by the garbage collector when none of their elements can +be tracked. diff --git a/Objects/setobject.c b/Objects/setobject.c index 8c5f244d4894d8a..c6ceebeb09957c6 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1572,6 +1572,7 @@ _PySet_Freeze(PyObject *set) assert(PySet_CheckExact(set)); assert(_PyObject_IsUniquelyReferenced(set)); set->ob_type = &PyFrozenSet_Type; + _PyFrozenSet_MaybeUntrack(set); return Py_NewRef(set); } From 646673c05fe879b1dd2417a9845db33cad1deb90 Mon Sep 17 00:00:00 2001 From: "Michiel W. Beijen" Date: Sat, 14 Nov 2026 14:42:07 +0100 Subject: [PATCH 142/229] gh-153309: Add HTTP QUERY method (RFC 10008) to http library (#155786) Add HTTP QUERY method (RFC 10008) to http library Closes #153309 --- Doc/library/http.client.rst | 5 +++- Doc/library/http.rst | 2 ++ Doc/whatsnew/3.16.rst | 7 +++++ Lib/http/__init__.py | 2 ++ Lib/http/client.py | 2 +- Lib/test/test_httplib.py | 30 ++++++++++++++++++- Lib/wsgiref/validate.py | 2 +- ...-07-08-09-05-01.gh-issue-153309.g94LSO.rst | 1 + 8 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-08-09-05-01.gh-issue-153309.g94LSO.rst diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 20fd357c56ac3ef..50a11ff18ab9bb2 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -312,7 +312,7 @@ HTTPConnection Objects but there is a request body, one of those header fields will be added automatically. If *body* is ``None``, the Content-Length header is set to ``0`` for - methods that expect a body (``PUT``, ``POST``, and ``PATCH``). If + methods that expect a body (``PUT``, ``POST``, ``PATCH``, and ``QUERY``). If *body* is a string or a bytes-like object that is not also a :term:`file `, the Content-Length header is set to its length. Any other type of *body* (files @@ -369,6 +369,9 @@ HTTPConnection Objects *continue_timeout* value must be greater than zero, otherwise :exc:`ValueError` is raised. + .. versionchanged:: next + ``QUERY`` was added to the methods that expect a body. + .. method:: HTTPConnection.getresponse(support_continue=False) Should be called after a request is sent to get the response from the server. diff --git a/Doc/library/http.rst b/Doc/library/http.rst index 43a801416e24f99..2520c9718aed6c6 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -194,6 +194,7 @@ Property Indicates that Details , , , + , ] .. _http-methods: @@ -217,4 +218,5 @@ Method Enum Name Details ``OPTIONS`` ``OPTIONS`` HTTP Semantics :rfc:`9110`, Section 9.3.7 ``TRACE`` ``TRACE`` HTTP Semantics :rfc:`9110`, Section 9.3.8 ``PATCH`` ``PATCH`` HTTP/1.1 :rfc:`5789` +``QUERY`` ``QUERY`` The HTTP QUERY Method :rfc:`10008` =========== =================================== ================================================================== diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index f86a6ec69172d8a..a7972d23e435afd 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -508,6 +508,13 @@ gzip (Contributed by Marin Misur in :gh:`91372`.) +http +---- + +* Add the ``QUERY`` HTTP method (:rfc:`10008`) to :class:`~http.HTTPMethod`. + (Contributed by Michiel W. Beijen in :gh:`153309`.) + + http.client ----------- diff --git a/Lib/http/__init__.py b/Lib/http/__init__.py index 691b4a9a367bd0f..219b3d1355b7294 100644 --- a/Lib/http/__init__.py +++ b/Lib/http/__init__.py @@ -192,6 +192,7 @@ class HTTPMethod: * RFC 9110: HTTP Semantics, obsoletes 7231, which obsoleted 2616 * RFC 5789: PATCH Method for HTTP + * RFC 10008: The HTTP QUERY Method """ def __new__(cls, value, description): obj = str.__new__(cls, value) @@ -210,4 +211,5 @@ def __repr__(self): PATCH = 'PATCH', 'Apply partial modifications to a target.' POST = 'POST', 'Perform target-specific processing with the request payload.' PUT = 'PUT', 'Replace the target with the request payload.' + QUERY = 'QUERY', 'Request that the target process the request payload in a safe and idempotent manner.' TRACE = 'TRACE', 'Perform a message loop-back test along the path to the target.' diff --git a/Lib/http/client.py b/Lib/http/client.py index 864ffc12b21e8a1..c05d7d933028430 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -183,7 +183,7 @@ # We always set the Content-Length header for these methods because some # servers will otherwise respond with a 411 -_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} +_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT', 'QUERY'} def _encode(data, name='data'): diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 04fcd0c883e638f..eb604bd2c4421be 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -200,7 +200,7 @@ def append(self, item): # Here, we're testing that methods expecting a body get a # content-length set to zero if the body is empty (either None or '') bodies = (None, '') - methods_with_body = ('PUT', 'POST', 'PATCH') + methods_with_body = ('PUT', 'POST', 'PATCH', 'QUERY') for method, body in itertools.product(methods_with_body, bodies): conn = client.HTTPConnection('example.com') conn.sock = FakeSocket(None) @@ -554,6 +554,34 @@ def test_invalid_method_names(self): conn.sock = FakeSocket(None) conn.request(method=method, url="/") + def test_query_request(self): + # QUERY (RFC 10008) is sent with a body, like PUT/POST/PATCH. + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn.request('QUERY', '/contacts', body='name=python', + headers={'Content-Type': 'application/x-www-form-urlencoded'}) + self.assertEqual(conn.sock.data, + b'QUERY /contacts HTTP/1.1\r\n' + b'Host: example.com\r\n' + b'Accept-Encoding: identity\r\n' + b'Content-Length: 11\r\n' + b'Content-Type: application/x-www-form-urlencoded\r\n' + b'\r\n' + b'name=python') + + def test_query_request_no_body(self): + # A QUERY without a body still gets Content-Length: 0, since some + # servers respond with 411 otherwise. + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn.request('QUERY', '/contacts') + self.assertEqual(conn.sock.data, + b'QUERY /contacts HTTP/1.1\r\n' + b'Host: example.com\r\n' + b'Accept-Encoding: identity\r\n' + b'Content-Length: 0\r\n' + b'\r\n') + class TransferEncodingTest(TestCase): expected_body = b"It's just a flesh wound" diff --git a/Lib/wsgiref/validate.py b/Lib/wsgiref/validate.py index 1a1853cd63a0d2a..cc572e8a179d1eb 100644 --- a/Lib/wsgiref/validate.py +++ b/Lib/wsgiref/validate.py @@ -334,7 +334,7 @@ def check_environ(environ): # @@: these need filling out: if environ['REQUEST_METHOD'] not in ( - 'GET', 'HEAD', 'POST', 'OPTIONS', 'PATCH', 'PUT', 'DELETE', 'TRACE'): + 'GET', 'HEAD', 'POST', 'OPTIONS', 'PATCH', 'PUT', 'DELETE', 'QUERY', 'TRACE'): warnings.warn( "Unknown REQUEST_METHOD: %r" % environ['REQUEST_METHOD'], WSGIWarning) diff --git a/Misc/NEWS.d/next/Library/2026-07-08-09-05-01.gh-issue-153309.g94LSO.rst b/Misc/NEWS.d/next/Library/2026-07-08-09-05-01.gh-issue-153309.g94LSO.rst new file mode 100644 index 000000000000000..9245c1442b06f87 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-08-09-05-01.gh-issue-153309.g94LSO.rst @@ -0,0 +1 @@ +:mod:`http`: Add the HTTP QUERY method (:rfc:`10008`) to :class:`~http.HTTPMethod`. From 082c4bbfc2bfdac920a254fcb1b07ca12515f5e9 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sat, 14 Nov 2026 09:00:18 -0800 Subject: [PATCH 143/229] gh-88037: Update whatsnew section for the parsing difference in urlparse with non numeric port. (#162011) gh-158956: Update whatsnew section for the parsing difference in urlparse with non numeric port. --- Doc/whatsnew/3.16.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index a7972d23e435afd..ea9b27c27d78ed6 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1415,6 +1415,28 @@ that may require changes to your code. :exc:`TypeError`. (Contributed by Serhiy Storchaka in :gh:`152587`.) +* :func:`urllib.parse.urlsplit` and :func:`urllib.parse.urlparse` now raise + :exc:`ValueError` when the URL is parsed if the port is not a decimal + number or is outside the range 0--65535. Previously, the error was only + raised when the :attr:`!port` attribute was read. This also affects + :func:`~urllib.parse.urljoin`, and :func:`~urllib.parse.urldefrag` for + URLs with a fragment. Code which never read :attr:`!port` may now fail + on URLs such as: + + * SCP-style addresses given a scheme, such as + ``git+ssh://git@github.com:org/repo.git``; + * multi-host URLs with ports, such as + ``mongodb://h1:27017,h2:27017/db`` or + ``postgresql://h1:5432,h2:5433/db``; + * URL templates with a placeholder port, such as + ``http://localhost:{port}/``. + + These are not valid URLs under :rfc:`3986`. To process them, split off + the netloc yourself before calling :func:`~urllib.parse.urlsplit`, for + example by replacing it with a placeholder host, or fill in templates + before parsing them. + (Contributed by Miguel Brito and Senthil Kumaran in :gh:`88037`.) + * :class:`uuid.UUID` now consistently raises :exc:`TypeError` when the *bytes* or *bytes_le* argument is not a :class:`bytes` object. Previously, some invalid inputs raised :exc:`AssertionError` or From b968417104f7a7feb8419de2042676e31e610a19 Mon Sep 17 00:00:00 2001 From: Olayinka Vaughan <114308865+Builder106@users.noreply.github.com> Date: Sat, 14 Nov 2026 12:39:51 -0500 Subject: [PATCH 144/229] gh-161788: Handle path-like commands in shutil.which() (#161789) Resolve command objects through the path protocol before selecting text or bytes handling. This preserves the documented bytes return type for path-like objects backed by bytes. Ref #161788 --- Lib/shutil.py | 1 + Lib/test/test_shutil.py | 4 ++++ .../Library/2026-11-10-23-51-25.gh-issue-161788.J8vQmK.rst | 2 ++ 3 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-11-10-23-51-25.gh-issue-161788.J8vQmK.rst diff --git a/Lib/shutil.py b/Lib/shutil.py index c24b786fb639242..f6bc8e16619e765 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -1608,6 +1608,7 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): path. """ + cmd = os.fspath(cmd) use_bytes = isinstance(cmd, bytes) # If we're given a path with a directory part, look it up directly rather diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index c0b48ea306b71ed..1104552215a44f1 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2614,6 +2614,10 @@ def test_basic(self): rv = shutil.which(self.file, path=self.dir) self.assertEqual(rv, self.filepath) + def test_pathlike_cmd(self): + rv = shutil.which(FakePath(self.file), path=self.dir) + self.assertEqual(rv, self.filepath) + def test_absolute_cmd(self): # When given the fully qualified path to an executable that exists, # it should be returned. diff --git a/Misc/NEWS.d/next/Library/2026-11-10-23-51-25.gh-issue-161788.J8vQmK.rst b/Misc/NEWS.d/next/Library/2026-11-10-23-51-25.gh-issue-161788.J8vQmK.rst new file mode 100644 index 000000000000000..21565e386cd3cc6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-10-23-51-25.gh-issue-161788.J8vQmK.rst @@ -0,0 +1,2 @@ +Fix :func:`shutil.which` for path-like command objects whose +:meth:`~os.PathLike.__fspath__` method returns :class:`bytes`. From 3403de5283fd44bba842638414e07999e0449723 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sat, 14 Nov 2026 23:30:23 +0300 Subject: [PATCH 145/229] gh-161863: Fix possible `NULL` deref in `_PyErr_Display` (#161979) --- Python/pythonrun.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 5e026a93d5c1806..38003e60e985ca2 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1157,6 +1157,13 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) } } + // gh-161863: Interpreter can be not fully initialized + // when calling this function. May be not safe to call the import. + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->imports.importlib == NULL) { + goto fallback; + } + // Try first with the stdlib traceback module PyObject *print_exception_fn = PyImport_ImportModuleAttrString( "traceback", From 5131d723d84b7ebca4bcf266cb891f33e51555ef Mon Sep 17 00:00:00 2001 From: 0z5a Date: Sat, 14 Nov 2026 21:47:12 +0100 Subject: [PATCH 146/229] gh-161849: Fix crash compiling a class that binds `__classdict__` (#161933) When a class body binds __classdict__ (assignment, for target, global, del), analyze_name() treated the name as global in the annotation and type parameter scopes that can see the class, so they no longer got the __classdict__ cell the class creates for them. Compiling such a class with a generic class inside crashed, and with an annotated method or a type alias raised SystemError. Skip that rule for __classdict__ itself. --- Lib/test/test_type_annotations.py | 26 +++++++++++++++++++ ...-11-13-08-43-30.gh-issue-161849.m5BmNR.rst | 4 +++ Python/symtable.c | 4 ++- 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-13-08-43-30.gh-issue-161849.m5BmNR.rst diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index cb164ccd147127b..879a571d06b4a03 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -505,6 +505,32 @@ class C: """) self.assertEqual(ns["C"].__annotations__, {"__classdict__": int}) + def test_class_binds_dunder_classdict(self): + # The class body binding __classdict__ must not hide the cell that + # annotation and type parameter scopes use to see the class namespace. + for binding in ( + "__classdict__ = 1", + "__classdict__: str = 1", + "for __classdict__ in (1,): pass", + "global __classdict__", + ): + with self.subTest(binding=binding): + ns = run_code(f""" + class C: + {binding} + y = int + def f(x: y): pass + class D[T: y]: pass + class E[T]: pass + type A[T: y] = y + """) + C = ns["C"] + self.assertEqual(C.f.__annotations__, {"x": int}) + self.assertIs(C.D.__type_params__[0].__bound__, int) + self.assertEqual(C.E.__type_params__[0].__name__, "T") + self.assertIs(C.A.__type_params__[0].__bound__, int) + self.assertIs(C.A.__value__, int) + def test_future_annotations(self): code = """ from __future__ import annotations diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-13-08-43-30.gh-issue-161849.m5BmNR.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-13-08-43-30.gh-issue-161849.m5BmNR.rst new file mode 100644 index 000000000000000..744a5abed698f45 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-13-08-43-30.gh-issue-161849.m5BmNR.rst @@ -0,0 +1,4 @@ +Fix a crash and a :exc:`SystemError` when compiling a class whose body binds +``__classdict__`` (for example ``__classdict__ = 1`` or ``global +__classdict__``) and also contains an annotated function, a generic class or +function, or a generic type alias. diff --git a/Python/symtable.c b/Python/symtable.c index 7f945dad1984ed9..429c204dba29b94 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -753,7 +753,9 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, // namespace and the globals (not the namespace providing the bound). // Similarly, if the name is explicitly global in the class namespace (through the // global statement), we want to also treat it as a global in this scope. - if (class_entry != NULL) { + // __classdict__ is the exception: these scopes load it from the cell that + // the class creates for them, whatever the class body does with that name. + if (class_entry != NULL && !_PyUnicode_Equal(name, &_Py_ID(__classdict__))) { long class_flags = _PyST_GetSymbol(class_entry, name); if (class_flags < 0) { return 0; From 1ca9ef9a11c0e0f59f550ee9b664dd94447bdfc6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 14 Nov 2026 23:01:01 +0100 Subject: [PATCH 147/229] gh-143869: Add PyLongWriter_Create() to What's New in Python 3.15 (#162022) Co-authored-by: Sergey B Kirpichev --- Doc/whatsnew/3.15.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 19fc5801cbe8a91..b07c94287bdee6b 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -2526,6 +2526,17 @@ New features (Contributed by Victor Stinner in :gh:`141510`.) +* Add the :pep:`757` export/import functions to the limited C API: + + * :c:func:`PyLong_GetNativeLayout` + * :c:func:`PyLong_Export` + * :c:func:`PyLong_FreeExport` + * :c:func:`PyLongWriter_Create` + * :c:func:`PyLongWriter_Finish` + * :c:func:`PyLongWriter_Discard` + + (Contributed by Sergey B Kirpichev in :gh:`143869`.) + * Add :c:func:`PyObject_CallFinalizerFromDealloc` function to the limited C API. (Contributed by Victor Stinner in :gh:`146063`.) From cc987451e8e902ddc0db77485faa34973b4e3ca8 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 14 Nov 2026 14:13:57 -0800 Subject: [PATCH 148/229] gh-161262: Fix test_touch_common flake on Emscripten (#161464) Under Node, NODEFS takes a new file's mtime from the host kernel but utime(path, None) uses Date.now() written back via fs.utimesSync, which can land a millisecond before the creation time. Take the baseline from utime(path, None) too, so both timestamps come from the same source. Co-authored-by: Claude Opus 5.5 --- Lib/test/test_pathlib/test_pathlib.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 788547c49e43e33..10e3ad6d92daec8 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -2332,6 +2332,9 @@ def test_touch_common(self): self.assertFalse(p.exists()) p.touch() self.assertTrue(p.exists()) + # Take the baseline from utime(None) like touch() below; creation can + # use a different clock (Emscripten NODEFS). + os.utime(p, None) st = p.stat() old_mtime = st.st_mtime old_mtime_ns = st.st_mtime_ns From edd6f7692ad1bb30ba48dc74e0488cef074e402c Mon Sep 17 00:00:00 2001 From: Olayinka Vaughan <114308865+Builder106@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:53:42 -0500 Subject: [PATCH 149/229] gh-161788: Document path-like support in shutil.which() (#162013) --- Doc/library/shutil.rst | 3 +++ .../Library/2026-11-10-23-51-25.gh-issue-161788.J8vQmK.rst | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 6104ad3d3a801b1..db484beaf4b2abe 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -532,6 +532,9 @@ Directory and files operations or ends with an extension that is in ``PATHEXT``; and filenames that have no extension can now be found. + .. versionchanged:: next + Accepts a :term:`path-like object` for *cmd*. + .. exception:: Error Subclass of :exc:`OSError` collecting exceptions raised during a multi-file diff --git a/Misc/NEWS.d/next/Library/2026-11-10-23-51-25.gh-issue-161788.J8vQmK.rst b/Misc/NEWS.d/next/Library/2026-11-10-23-51-25.gh-issue-161788.J8vQmK.rst index 21565e386cd3cc6..34d3cff85123c79 100644 --- a/Misc/NEWS.d/next/Library/2026-11-10-23-51-25.gh-issue-161788.J8vQmK.rst +++ b/Misc/NEWS.d/next/Library/2026-11-10-23-51-25.gh-issue-161788.J8vQmK.rst @@ -1,2 +1 @@ -Fix :func:`shutil.which` for path-like command objects whose -:meth:`~os.PathLike.__fspath__` method returns :class:`bytes`. +:func:`shutil.which` now accepts path-like objects for the command. From 60eca2df18fcd47d71ab136e460efabbc2f4a044 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 14 Nov 2026 16:31:55 -0800 Subject: [PATCH 150/229] gh-161145: Test more bytes simultaneously in find_first_nonascii (#161147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Long ASCII runs now OR eight aligned words and test the high bits once, one branch per 64 bytes (32 on 32-bit) instead of one per word. When a block contains a non-ASCII byte, the existing one-word loop finds it. It is plain C: GCC and Clang compile it to SSE2/NEON loads, other compilers to ORed word loads with the same single branch. Multiple compiler and architecture codegen sample: https://godbolt.org/z/8bM83vx3a pyperf, PGO+LTO, 2 interleaved rounds: | Benchmark | main | this | |---|---:|---:| | (b'a' * 4096).decode() | 165 ns | 108 ns: 1.53x faster | | (b'a' * 200).decode() | 28.9 ns | 26.5 ns: 1.09x faster | | b'a' * 200 + 'é' + ..., .decode() | 60.9 ns | 56.0 ns: 1.09x faster | | b'a' * 136 + 'é' + ..., .decode() | 53.8 ns | 51.3 ns: 1.05x faster | | 8-100 byte decodes, 'é' within 100 | | within 1.2 ns | ```python import pyperf runner = pyperf.Runner() tail = 'é'.encode() + b'a' * 50 for n in (8, 100, 200, 4096): runner.timeit(f"(b'a' * {n}).decode()", "s.decode()", globals={"s": b'a' * n}) for n in (16, 100, 136, 200): runner.timeit(f"(b'a' * {n} + 'é'.encode() + b'a' * 50).decode()", "s.decode()", globals={"s": b'a' * n + tail}) ``` Co-authored-by: Claude Opus 5.5 Co-authored-by: Victor Stinner --- .../2026-11-01-16-02-42.gh-issue-161145.5vLdrm.rst | 2 ++ Objects/unicodeobject.c | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-01-16-02-42.gh-issue-161145.5vLdrm.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-01-16-02-42.gh-issue-161145.5vLdrm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-01-16-02-42.gh-issue-161145.5vLdrm.rst new file mode 100644 index 000000000000000..5d809624e25f420 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-01-16-02-42.gh-issue-161145.5vLdrm.rst @@ -0,0 +1,2 @@ +Speed up UTF-8 decoding of long ASCII runs, such as in :meth:`bytes.decode`, +by up to 1.5x. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index da1ba44dba81290..0fe765c36848f25 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5063,6 +5063,16 @@ find_first_nonascii(const unsigned char *start, const unsigned char *end) } #endif + // Long ASCII runs: test eight words per branch. + for (; end - p >= 8 * SIZEOF_SIZE_T; p += 8 * SIZEOF_SIZE_T) { + const size_t *w = (const size_t *)p; + if ((w[0] | w[1] | w[2] | w[3] | + w[4] | w[5] | w[6] | w[7]) & ASCII_CHAR_MASK) + { + break; + } + } + const unsigned char *e = end - SIZEOF_SIZE_T; while (p <= e) { size_t u = (*(const size_t *)p) & ASCII_CHAR_MASK; From 8750d3827ac038d3f597bc3bb3bd36b3bb9b9fe4 Mon Sep 17 00:00:00 2001 From: 777GE90 Date: Sun, 15 Nov 2026 04:43:43 +0000 Subject: [PATCH 151/229] gh-87027: Add prev_network function to ipaddress library (follow-up to GH-24180) (#154041) bpo-42861: Added prev_network method, required refactor of next_network to prevent duplication. --- Doc/library/ipaddress.rst | 33 +++++++- Doc/whatsnew/3.16.rst | 4 + Lib/ipaddress.py | 76 ++++++++++++++----- Lib/test/test_ipaddress.py | 69 +++++++++++++++-- ...6-10-20-12-00-00.gh-issue-87027.PrEvNw.rst | 2 + 5 files changed, 154 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-20-12-00-00.gh-issue-87027.PrEvNw.rst diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 2c7deb40e9044c0..92c2bbd5f97b9c7 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -739,7 +739,7 @@ dictionaries. Traceback (most recent call last): File "", line 1, in raise ValueError( - ValueError: next prefix must be between 1 and 32 + ValueError: prefix must be between 1 and 32 >>> IPv4Network('255.255.255.0/24').next_network() Traceback (most recent call last): File "", line 1, in @@ -748,6 +748,36 @@ dictionaries. .. versionadded:: 3.16 + .. method:: prev_network(prev_prefix=None) + + Finds the previous closest network of prefix size *prev_prefix*. If + *prev_prefix=None*, then the current network prefix will be used. + Returns a single network object. + + Raises :exc:`ValueError` if *prev_prefix* is out of range or no further + network of the requested size exists. + + >>> IPv4Network('192.0.2.0/24').prev_network() + IPv4Network('192.0.1.0/24') + >>> IPv4Network('192.0.2.0/24').prev_network(prev_prefix=25) + IPv4Network('192.0.1.128/25') + >>> IPv4Network('192.0.2.0/24').prev_network(prev_prefix=23) + IPv4Network('192.0.0.0/23') + >>> IPv4Network('192.0.80.0/22').prev_network(prev_prefix=18) + IPv4Network('192.0.0.0/18') + >>> IPv4Network('192.0.80.0/22').prev_network(prev_prefix=50) + Traceback (most recent call last): + File "", line 1, in + raise ValueError( + ValueError: prefix must be between 1 and 32 + >>> IPv4Network('0.0.0.0/24').prev_network() + Traceback (most recent call last): + File "", line 1, in + raise ValueError( + ValueError: out of address space, cannot make another /24 network + + .. versionadded:: 3.16 + .. class:: IPv6Network(address, strict=True) Construct an IPv6 network definition. *address* can be one of the following: @@ -820,6 +850,7 @@ dictionaries. .. method:: subnet_of(other) .. method:: supernet_of(other) .. method:: next_network(next_prefix=None) + .. method:: prev_network(prev_prefix=None) .. method:: compare_networks(other) Refer to the corresponding attribute documentation in diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index ea9b27c27d78ed6..33328b2ee005219 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -601,6 +601,10 @@ ipaddress and :meth:`IPv6Network.next_network() ` methods to find the next nearest network with a specific prefix size. (Contributed by Faisal Mahmood in :gh:`87027`.) +* Add :meth:`IPv4Network.prev_network() ` + and :meth:`IPv6Network.prev_network() ` + methods to find the previous nearest network with a specific prefix size. + (Contributed by Faisal Mahmood in :gh:`87027`.) logging diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index b11999a0f44bccc..3421c9f40ce843f 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1119,6 +1119,46 @@ def is_loopback(self): return (self.network_address.is_loopback and self.broadcast_address.is_loopback) + def _network_by_offset(self, prefix, offset): + """Get the network offset from this network. + + Args: + prefix: Target prefix length. + offset: Number of networks to move (+1 or -1). + + Returns: + An IPv(4|6) Network object. + + """ + if prefix < 1 or prefix > self.max_prefixlen: + raise ValueError( + f"prefix must be between 1 and {self.max_prefixlen}" + ) + new_netmask, _ = self._make_netmask(prefix) + error = f"out of address space, cannot make another /{prefix} network" + + if prefix > self.prefixlen: + if offset < 0: + new_ip = self.network_address._ip - 1 + else: + new_ip = self.broadcast_address._ip + 1 + if new_ip < 0 or new_ip > self._ALL_ONES: + raise ValueError(error) + + new_ip &= new_netmask._ip + else: + bit_shift = self.max_prefixlen - prefix + new_ip = ( + ((new_netmask._ip & self.network_address._ip) >> bit_shift) + + offset + ) << bit_shift + if not 0 <= new_ip <= self._ALL_ONES: + raise ValueError(error) + + return self.__class__( + f"{self._string_from_ip_int(new_ip)}/{prefix}" + ) + def next_network(self, next_prefix=None): """Get the next closest network with a specific prefix. @@ -1136,31 +1176,25 @@ def next_network(self, next_prefix=None): """ if next_prefix is None: next_prefix = self.prefixlen - new_netmask = self.netmask - else: - if next_prefix < 1 or next_prefix > self.max_prefixlen: - raise ValueError( - f"next prefix must be between 1 and {self.max_prefixlen}" - ) - new_netmask, _ = self._make_netmask(next_prefix) - bit_shift = ( - self.max_prefixlen - next_prefix - if next_prefix <= self.prefixlen - else self.max_prefixlen - self.prefixlen - ) + return self._network_by_offset(next_prefix, 1) - next_ip = ( - ((new_netmask._ip & self.network_address._ip) >> bit_shift) + 1 - ) << bit_shift - if next_ip > self._ALL_ONES: - raise ValueError( - f"out of address space, cannot make another /{next_prefix} " - "network" - ) + def prev_network(self, prev_prefix=None): + """Get the previous closest network with a specific prefix. + + Args: + prev_prefix: The desired previous prefix length, if not specified the + same self.prefixlen will be used + + Returns: + An IPv(4|6) Network object of the previous closest network. + + """ + if prev_prefix is None: + prev_prefix = self.prefixlen - return self.__class__((next_ip, next_prefix)) + return self._network_by_offset(prev_prefix, -1) class _BaseConstants: diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 9110124d9423cff..4b398c1f41ef602 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -1596,16 +1596,69 @@ def testNextNetworkWithBadPrefix(self): self.assertRaises(ValueError, self.ipv6_network.next_network, 150) def testNextNetworkOutOfAddressSpace(self): + error = r"out of address space, cannot make another /{} network" ipv4 = ipaddress.IPv4Network('255.255.255.0/24') - self.assertRaisesRegex( - ValueError, - 'out of address space, cannot make another /24 network', - ipv4.next_network) + with self.assertRaisesRegex(ValueError, error.format(24)): + ipv4.next_network() + with self.assertRaisesRegex(ValueError, error.format(25)): + ipv4.next_network(25) ipv6 = ipaddress.IPv6Network('ffff:ffff:ffff:ffff:ffff:ffff:ffff:0/112') - self.assertRaisesRegex( - ValueError, - 'out of address space, cannot make another /112 network', - ipv6.next_network) + with self.assertRaisesRegex(ValueError, error.format(112)): + ipv6.next_network() + with self.assertRaisesRegex(ValueError, error.format(120)): + ipv6.next_network(120) + + def testPrevNetwork(self): + ipv4 = ipaddress.IPv4Network('1.2.3.0/24') + self.assertEqual( + ipv4.prev_network(), + ipaddress.IPv4Network('1.2.2.0/24'), + ) + self.assertEqual( + ipv4.prev_network(prev_prefix=16), + ipaddress.IPv4Network('1.1.0.0/16'), + ) + self.assertEqual( + ipv4.prev_network(prev_prefix=25), + ipaddress.IPv4Network('1.2.2.128/25'), + ) + self.assertEqual( + ipv4.prev_network(prev_prefix=23), + ipaddress.IPv4Network('1.2.0.0/23'), + ) + + ipv6 = ipaddress.IPv6Network('2001:dbb8:aaaa:aaaa::/64') + self.assertEqual( + ipv6.prev_network(), + ipaddress.IPv6Network('2001:dbb8:aaaa:aaa9::/64'), + ) + self.assertEqual( + ipv6.prev_network(prev_prefix=48), + ipaddress.IPv6Network('2001:dbb8:aaa9::/48'), + ) + self.assertEqual( + ipv6.prev_network(prev_prefix=88), + ipaddress.IPv6Network('2001:dbb8:aaaa:aaa9:ffff:ff00::/88'), + ) + + def testPrevNetworkWithBadPrefix(self): + self.assertRaises(ValueError, self.ipv4_network.prev_network, 0) + self.assertRaises(ValueError, self.ipv4_network.prev_network, 35) + self.assertRaises(ValueError, self.ipv6_network.prev_network, 0) + self.assertRaises(ValueError, self.ipv6_network.prev_network, 150) + + def testPrevNetworkOutOfAddressSpace(self): + error = r"out of address space, cannot make another /{} network" + ipv4 = ipaddress.IPv4Network('0.0.0.0/24') + with self.assertRaisesRegex(ValueError, error.format(24)): + ipv4.prev_network() + with self.assertRaisesRegex(ValueError, error.format(25)): + ipv4.prev_network(25) + ipv6 = ipaddress.IPv6Network('::/112') + with self.assertRaisesRegex(ValueError, error.format(112)): + ipv6.prev_network() + with self.assertRaisesRegex(ValueError, error.format(120)): + ipv6.prev_network(120) def testFancySubnetting(self): self.assertEqual(sorted(self.ipv4_network.subnets(prefixlen_diff=3)), diff --git a/Misc/NEWS.d/next/Library/2026-10-20-12-00-00.gh-issue-87027.PrEvNw.rst b/Misc/NEWS.d/next/Library/2026-10-20-12-00-00.gh-issue-87027.PrEvNw.rst new file mode 100644 index 000000000000000..504cd054ca464c7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-20-12-00-00.gh-issue-87027.PrEvNw.rst @@ -0,0 +1,2 @@ +Add :meth:`~ipaddress.IPv4Network.prev_network` and +:meth:`~ipaddress.IPv6Network.prev_network`. Patch by Faisal Mahmood. From 369d8bc4f6fe1f751de2512523a1acb95491a5eb Mon Sep 17 00:00:00 2001 From: sobolevn Date: Sun, 15 Nov 2026 15:43:13 +0300 Subject: [PATCH 152/229] gh-152254: Fix a crash in `PyObject_ClearManagedDict` under memory pressure (#152255) Co-authored-by: Victor Stinner --- Lib/test/test_capi/test_object.py | 77 +++++++++++++++++++ ...-06-26-11-34-50.gh-issue-152254.yK6TGp.rst | 2 + Modules/_testcapi/object.c | 8 ++ Objects/dictobject.c | 1 + 4 files changed, 88 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-11-34-50.gh-issue-152254.yK6TGp.rst diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index 338fc61ea1eb5f5..ece673dbbec089d 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -354,6 +354,83 @@ def test_pyobject_dump(self): output = self.pyobject_dump(NULL) self.assertRegex(output, r'') + def test_object_clear_managed_dict(self): + # PyObject_ClearManagedDict + clear_managed_dict = _testcapi.pyobject_clear_managed_dict + ManagedDictType = _testcapi.ManagedDictType + + obj = ManagedDictType() + obj.value = 'first' + # asserting `__dict__` is a must to materialize the value before clean: + self.assertEqual(obj.__dict__, {'value': 'first'}) + self.assertIs(type(obj.__dict__), dict) + clear_managed_dict(obj) + self.assertEqual(obj.__dict__, {}) + self.assertIs(type(obj.__dict__), dict) + + # big object: + obj = ManagedDictType() + existing_dict = {f'attr_{x}': x for x in range(100)} + obj.__dict__ = existing_dict + self.assertIs(obj.__dict__, existing_dict) + clear_managed_dict(obj) + self.assertEqual(obj.__dict__, {}) + self.assertIs(type(obj.__dict__), dict) + + # empty dict: + obj = ManagedDictType() + self.assertEqual(obj.__dict__, {}) + self.assertIs(type(obj.__dict__), dict) + clear_managed_dict(obj) + self.assertEqual(obj.__dict__, {}) + self.assertIs(type(obj.__dict__), dict) + + # double clear: + obj = ManagedDictType() + obj.value = 'first' + self.assertEqual(obj.__dict__, {'value': 'first'}) + clear_managed_dict(obj) + clear_managed_dict(obj) + self.assertEqual(obj.__dict__, {}) + self.assertIs(type(obj.__dict__), dict) + + # unmaterialized: + obj = ManagedDictType() + obj.value = 'first' + clear_managed_dict(obj) + self.assertEqual(obj.__dict__, {}) + self.assertIs(type(obj.__dict__), dict) + + # CRASHES: clear_managed_dict(NULL) + + @threading_helper.requires_working_threading() + @support.nomemtest + def test_object_clear_managed_dict_ft_memory_error(self): + # gh-152254: Test PyObject_ClearManagedDict() error. + # The setup must contain: object owned by another thread + # and the attempt to clear the managed dict with no memory left. + clear_managed_dict = _testcapi.pyobject_clear_managed_dict + ManagedDictType = _testcapi.ManagedDictType + + obj = ManagedDictType() + obj.value = 'first' + self.assertEqual(len(obj.__dict__), 1) + + def worker(): + # This needs to be in a separate thread for the crash to happen: + with support.catch_unraisable_exception() as ex: + # `PyObject_ClearManagedDict()` used to crash: + _testcapi.call_with_nomemory(0, 1, clear_managed_dict, obj) + self.assertIsNotNone(ex.unraisable) + self.assertIs(ex.unraisable.exc_type, MemoryError) + self.assertIn('managed dict', ex.unraisable.err_msg) + self.assertEqual(obj.__dict__, {}) + self.assertIs(type(obj.__dict__), dict) + + threading_helper.run_concurrently([worker]) + self.assertEqual(obj.__dict__, {}) + self.assertIs(type(obj.__dict__), dict) + class RefTracerTest(unittest.TestCase): @support.requires_resource('cpu') diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-11-34-50.gh-issue-152254.yK6TGp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-11-34-50.gh-issue-152254.yK6TGp.rst new file mode 100644 index 000000000000000..206b88a2affb2b2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-11-34-50.gh-issue-152254.yK6TGp.rst @@ -0,0 +1,2 @@ +Fix a crash during the clean up of managed dict objects under a memory +pressure in :c:func:`PyObject_ClearManagedDict`. diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 8fcb86e33e55a83..1bb038aea654a17 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -153,6 +153,13 @@ pyobject_is_unique_temporary_new_object(PyObject *self, PyObject *unused) return PyLong_FromLong(result); } +static PyObject * +pyobject_clear_managed_dict(PyObject *self, PyObject *obj) +{ + PyObject_ClearManagedDict(obj); + Py_RETURN_NONE; +} + static int MyObject_dealloc_called = 0; static void @@ -358,6 +365,7 @@ static PyMethodDef test_methods[] = { {"pyobject_enable_deferred_refcount", pyobject_enable_deferred_refcount, METH_O}, {"pyobject_is_unique_temporary", pyobject_is_unique_temporary, METH_O}, {"pyobject_is_unique_temporary_new_object", pyobject_is_unique_temporary_new_object, METH_NOARGS}, + {"pyobject_clear_managed_dict", pyobject_clear_managed_dict, METH_O}, {"test_py_try_inc_ref", test_py_try_inc_ref, METH_NOARGS}, {"test_py_set_immortal", test_py_set_immortal, METH_NOARGS}, #ifdef Py_REF_DEBUG diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ce6f4a61ad3c262..5d3edbc4ed9578a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -8146,6 +8146,7 @@ PyObject_ClearManagedDict(PyObject *obj) /* Clear the dict */ Py_BEGIN_CRITICAL_SECTION(dict); PyDictKeysObject *oldkeys = dict->ma_keys; + ensure_shared_on_resize(dict); set_keys(dict, Py_EMPTY_KEYS); dict->ma_values = NULL; dictkeys_decref(oldkeys, IS_DICT_SHARED(dict)); From c4f492884ae211a286911318a9486b6d7aee64f1 Mon Sep 17 00:00:00 2001 From: Abhinav Date: Mon, 16 Nov 2026 01:48:41 +0530 Subject: [PATCH 153/229] gh-157917: Do not leak the \\?\ extended-length prefix into sys.prefix on Windows venvs (GH-158465) When a venv interpreter is invoked through a "\\?\" extended-length path, the venv launcher reports its own path via GetModuleFileNameW, which preserves that prefix. getpath.py consumed the value verbatim, so sys.prefix, sys.exec_prefix and sys.executable all kept the prefix. Windows does not resolve ".." or any other relative component in an extended-length path, so tools that join them onto a sysconfig path break. That is what makes "pip install" of a wheel with a "scripts=" entry fail with "OSError: [Errno 22] Invalid argument". Strip the prefix back off the launcher-provided path on Windows. A path that would still exceed MAX_PATH once unprefixed keeps the prefix, since the extended-length form is the only usable one for it. --- Lib/test/test_venv.py | 22 ++++++++++++++ ...-09-30-00-00-00.gh-issue-157917.48oa0L.rst | 9 ++++++ Modules/getpath.py | 29 +++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 Misc/NEWS.d/next/Windows/2026-09-30-00-00-00.gh-issue-157917.48oa0L.rst diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index bcb782a8d10c3ae..eded0e5040d1480 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -302,6 +302,28 @@ def test_prefixes(self): self.assertEqual(pathlib.Path(out.strip().decode()), pathlib.Path(expected), prefix) + # gh-157917: the "\\?\" extended-length prefix must not leak into the prefix + @requireVenvCreate + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_prefixes_extended_length_path(self): + """ + Test that invoking a venv through a "\\?\" extended-length path does + not leave that prefix behind in the prefixes. Windows does not + resolve relative components (such as ".." or "/") in such paths, so + they break tools that join them onto sysconfig-derived paths. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + envpy = os.path.join('\\\\?\\' + self.env_dir, self.bindir, self.exe) + for prefix in ('prefix', 'exec_prefix'): + with self.subTest(prefix): + out, err = check_output( + [envpy, '-c', 'import sys; print(sys.%s)' % prefix], + encoding='utf-8') + self.assertFalse(out.strip().startswith('\\\\?\\'), out) + self.assertEqual(pathlib.Path(out.strip()), + pathlib.Path(self.env_dir), prefix) + @requireVenvCreate def test_sysconfig(self): """ diff --git a/Misc/NEWS.d/next/Windows/2026-09-30-00-00-00.gh-issue-157917.48oa0L.rst b/Misc/NEWS.d/next/Windows/2026-09-30-00-00-00.gh-issue-157917.48oa0L.rst new file mode 100644 index 000000000000000..2d5e6a14a31d53f --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-09-30-00-00-00.gh-issue-157917.48oa0L.rst @@ -0,0 +1,9 @@ +Fix :attr:`sys.prefix`, :attr:`sys.exec_prefix` and :attr:`sys.executable` +retaining the ``\\?\`` extended-length path prefix in a virtual environment +when the interpreter is invoked through such a path on Windows. The venv +launcher reports its own path through ``GetModuleFileNameW``, which preserves +the prefix, and that value was used verbatim. Windows does not resolve +relative path components in extended-length paths, so the leaked prefix made +tools that join ``..`` components onto :mod:`sysconfig` paths fail with +``OSError: [Errno 22]``. The prefix is now stripped again, except for paths +that are still too long to be used without it. diff --git a/Modules/getpath.py b/Modules/getpath.py index 2ffb1d964c0fb63..6afbe4c68985569 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -211,6 +211,28 @@ def search_up(prefix, *landmarks, test=isfile): prefix = dirname(prefix) +# The traditional Windows path length limit, not counting the terminating NUL. +MAX_PATH = 260 + + +def strip_extended_length_prefix(path): + """ + Remove the leading "\\\\?\\" of a Windows extended-length path, and + rewrite the "\\\\?\\UNC\\" form of a UNC path back to a plain "\\\\" one. + Windows does not resolve relative components (such as ".." or "/") in an + extended-length path, so keeping the prefix in a path that we later join + onto is liable to break consumers. A path that is still too long to be + used unprefixed keeps its prefix, as it has no other usable form. + """ + if path.startswith('\\\\?\\UNC\\'): + stripped = '\\\\' + path[8:] + elif path.startswith('\\\\?\\'): + stripped = path[4:] + else: + return path + return stripped if len(stripped) < MAX_PATH else path + + # ****************************************************************************** # READ VARIABLES FROM config # ****************************************************************************** @@ -320,6 +342,13 @@ def search_up(prefix, *landmarks, test=isfile): real_executable = base_executable #real_executable_dir = dirname(real_executable) executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__ + if os_name == 'nt': + # QUIRK: On Windows the venv launcher reports its own path through + # GetModuleFileNameW, which hands back a "\\?\" extended-length path + # whenever that is how the interpreter was invoked. Drop that prefix + # again, so the venv prefix and sys.executable stay usable by tools + # that join relative components onto them (gh-157917). + executable = strip_extended_length_prefix(executable) executable_dir = dirname(executable) From 15cbd9b92953d85b01fb54287893209510dea56d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Delfino?= Date: Sun, 15 Nov 2026 17:48:40 -0300 Subject: [PATCH 154/229] Fix missing comma in `test_time` assertion (#161832) Co-authored-by: Stan Ulbrych --- Lib/test/test_time.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index c5754afd0e84a81..861076357e0019a 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -562,8 +562,8 @@ def test_tzset(self): # on some operating systems (e.g. FreeBSD), which is wrong. See for # example this bug: # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=93810 - self.assertIn(time.tzname[0], ('AEST' 'EST'), time.tzname[0]) - self.assertTrue(time.tzname[1] in ('AEDT', 'EDT'), str(time.tzname[1])) + self.assertIn(time.tzname[0], ('AEST', 'EST')) + self.assertIn(time.tzname[1], ('AEDT', 'EDT')) self.assertEqual(len(time.tzname), 2) self.assertEqual(time.daylight, 1) self.assertEqual(time.timezone, -36000) From d8bc6669c7f4bcf1a72599552b99fbcb4fa7a702 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 15 Nov 2026 21:56:29 +0000 Subject: [PATCH 155/229] gh-138122: Simplify Tachyon's macOS and Windows inspection (#161956) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Maurycy Pawłowski-Wieroński --- Lib/test/test_external_inspection.py | 22 + Modules/_remote_debugging/_remote_debugging.h | 22 - Modules/_remote_debugging/asyncio.c | 35 +- Modules/_remote_debugging/module.c | 41 +- Modules/_remote_debugging/threads.c | 112 ++-- Python/remote_debug.h | 547 ++++++++---------- 6 files changed, 336 insertions(+), 443 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index a3f2eabacf6e88f..104efaebd85c16f 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -4687,6 +4687,28 @@ def test_dead_process_raises_not_degrades(self): else: self.fail("ProcessLookupError never raised for dead process") + def test_dead_process_before_first_async_read(self): + script_body = """\ + import asyncio + sock.sendall(b"ready") + sock.recv(16) + """ + with self._target_process(script_body) as (p, client_socket, _): + _wait_for_signal(client_socket, b"ready") + unwinders = [ + RemoteUnwinder(p.pid, debug=debug) + for debug in (False, True) + ] + + p.kill() + p.wait() + + for debug, unwinder in zip((False, True), unwinders): + for method in ("get_async_stack_trace", "get_all_awaited_by"): + with self.subTest(debug=debug, method=method): + with self.assertRaises(ProcessLookupError): + getattr(unwinder, method)() + @requires_remote_subprocess_debugging() class TestFrameChainLimits(RemoteInspectionTestBase): diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 79fa4a92e745bd0..6080dc4e3815067 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -56,7 +56,6 @@ extern "C" { # if TARGET_OS_OSX # include # include -# define MAX_NATIVE_THREADS 4096 # endif #endif @@ -104,18 +103,7 @@ typedef struct { #ifdef MS_WINDOWS #define STATUS_SUCCESS ((NTSTATUS)0x00000000L) -#define STATUS_INFO_LENGTH_MISMATCH ((NTSTATUS)0xC0000004L) #define STATUS_NO_MORE_ENTRIES ((NTSTATUS)0x8000001AL) -typedef enum _WIN32_THREADSTATE { - WIN32_THREADSTATE_INITIALIZED = 0, - WIN32_THREADSTATE_READY = 1, - WIN32_THREADSTATE_RUNNING = 2, - WIN32_THREADSTATE_STANDBY = 3, - WIN32_THREADSTATE_TERMINATED = 4, - WIN32_THREADSTATE_WAITING = 5, - WIN32_THREADSTATE_TRANSITION = 6, - WIN32_THREADSTATE_UNKNOWN = 7 -} WIN32_THREADSTATE; #endif /* ============================================================================ @@ -387,14 +375,6 @@ typedef struct { #ifdef Py_GIL_DISABLED uint32_t tlbc_generation; _Py_hashtable_t *tlbc_cache; -#endif -#ifdef __APPLE__ - uint64_t thread_id_offset; - int thread_id_offset_initialized; -#endif -#ifdef MS_WINDOWS - PVOID win_process_buffer; - ULONG win_process_buffer_size; #endif // Thread stopping state (only on platforms that support it) #ifdef Py_REMOTE_DEBUG_SUPPORTS_BLOCKING @@ -681,8 +661,6 @@ extern int find_running_frame( uintptr_t *frame ); -extern int get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread_id); - extern PyObject* unwind_stack_for_thread( RemoteUnwinderObject *unwinder, uintptr_t *current_tstate, diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index d31f783186a2c30..11fa40eb3e5f00c 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -22,7 +22,7 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio", NULL); if (address == 0) { - if (!_Py_RemoteDebug_HasPermissionError()) { + if (!_Py_RemoteDebug_IsFatalReadError()) { PyObject *exc = PyErr_GetRaisedException(); PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); _PyErr_ChainExceptions1(exc); @@ -33,23 +33,16 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) address = search_linux_map_for_section(handle, "AsyncioDebug", "python", NULL); if (address == 0) { - if (!_Py_RemoteDebug_HasPermissionError()) { + if (!_Py_RemoteDebug_IsFatalReadError()) { PyObject *exc = PyErr_GetRaisedException(); PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); _PyErr_ChainExceptions1(exc); } } #elif defined(__APPLE__) && TARGET_OS_OSX - // On macOS, try libpython first, then fall back to python - address = search_map_for_section(handle, "AsyncioDebug", "libpython", - NULL); - if (address == 0 && !_Py_RemoteDebug_HasPermissionError()) { - PyErr_Clear(); - address = search_map_for_section(handle, "AsyncioDebug", "python", - NULL); - } + address = search_loaded_images_for_section(handle, "AsyncioDebug", NULL); if (address == 0) { - if (!_Py_RemoteDebug_HasPermissionError()) { + if (!_Py_RemoteDebug_IsFatalReadError()) { PyObject *exc = PyErr_GetRaisedException(); PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); _PyErr_ChainExceptions1(exc); @@ -99,12 +92,22 @@ ensure_async_debug_offsets(RemoteUnwinderObject *unwinder) return -1; } if (result < 0) { - if (!_Py_RemoteDebug_HasPermissionError()) { - PyErr_Clear(); - PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); - set_exception_cause(unwinder, PyExc_RuntimeError, - "AsyncioDebug section unavailable - asyncio module may not be loaded in target process"); + if (_Py_RemoteDebug_IsFatalReadError()) { + return -1; } + PyErr_Clear(); + /* Image discovery can fail without reporting that the target exited. + Read a known runtime address before reporting unavailable asyncio. */ + char runtime_byte; + if (_Py_RemoteDebug_ReadRemoteMemory(&unwinder->handle, + unwinder->runtime_start_address, sizeof(runtime_byte), + &runtime_byte) < 0) + { + return -1; + } + PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); + set_exception_cause(unwinder, PyExc_RuntimeError, + "AsyncioDebug section unavailable - asyncio module may not be loaded in target process"); return -1; } diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 28f73a86dcdbda1..303af4a4505fea7 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -384,12 +384,6 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, return -1; } - self->runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(&self->handle); - if (self->runtime_start_address == 0) { - set_exception_cause(self, PyExc_RuntimeError, "Failed to get Python runtime address"); - return -1; - } - if (_Py_RemoteDebug_ReadDebugOffsets(&self->handle, &self->runtime_start_address, &self->debug_offsets) < 0) @@ -404,20 +398,8 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, return -1; } - // Try to read async debug offsets, but don't fail if they're not available - self->async_debug_offsets_available = 1; - int async_debug_result = read_async_debug(self); - if (async_debug_result == PY_REMOTE_DEBUG_INVALID_ASYNC_DEBUG_OFFSETS) { - return -1; - } - if (async_debug_result < 0) { - if (_Py_RemoteDebug_HasPermissionError()) { - return -1; - } - PyErr_Clear(); - memset(&self->async_debug_offsets, 0, sizeof(self->async_debug_offsets)); - self->async_debug_offsets_available = 0; - } + self->async_debug_offsets_available = 0; + memset(&self->async_debug_offsets, 0, sizeof(self->async_debug_offsets)); if (populate_initial_state_data(all_threads, self, self->runtime_start_address, &self->interpreter_addr ,&self->tstate_addr) < 0) @@ -456,15 +438,6 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, } #endif -#if defined(__APPLE__) - self->thread_id_offset = 0; - self->thread_id_offset_initialized = 0; -#endif - -#ifdef MS_WINDOWS - self->win_process_buffer = NULL; - self->win_process_buffer_size = 0; -#endif #ifdef __linux__ self->thread_tids = NULL; self->thread_tids_capacity = 0; @@ -1347,11 +1320,6 @@ RemoteUnwinder_dealloc(PyObject *op) if (self->code_object_cache) { _Py_hashtable_destroy(self->code_object_cache); } -#ifdef MS_WINDOWS - if (self->win_process_buffer != NULL) { - PyMem_Free(self->win_process_buffer); - } -#endif #ifdef Py_GIL_DISABLED if (self->tlbc_cache) { @@ -1406,11 +1374,6 @@ init_runtime_offsets(RuntimeOffsets *offsets, pid_t pid, int debug) set_exception_cause(offsets, PyExc_RuntimeError, "Failed to initialize process handle"); return -1; } - offsets->runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(&offsets->handle); - if (offsets->runtime_start_address == 0) { - set_exception_cause(offsets, PyExc_RuntimeError, "Failed to get Python runtime address"); - goto error; - } if (_Py_RemoteDebug_ReadDebugOffsets(&offsets->handle, &offsets->runtime_start_address, &offsets->debug_offsets) < 0) diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index a3bd6c9edfb58eb..66c40d875ab3b31 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -18,6 +18,14 @@ #include #endif +#ifdef MS_WINDOWS +enum { + /* ThreadSystemThreadInformation is not named in the Windows SDK. */ + WIN32_THREAD_SYSTEM_INFORMATION = 40, + WIN32_THREAD_RUNNING = 2, +}; +#endif + /* Bound traversal of corrupted remote exception chains. */ #define MAX_EXCEPTION_CHAIN_DEPTH (2 << 15) @@ -158,45 +166,22 @@ find_running_frame( * THREAD STATUS FUNCTIONS * ============================================================================ */ -int -get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread_id) { +static int +get_thread_status(RemoteUnwinderObject *unwinder, unsigned long tid) { #if defined(__APPLE__) && TARGET_OS_OSX - if (!unwinder->thread_id_offset_initialized) { - uint64_t *tids = (uint64_t *)PyMem_Malloc(MAX_NATIVE_THREADS * sizeof(uint64_t)); - if (!tids) { - // Non-fatal: thread status is best-effort - return THREAD_STATE_UNKNOWN; - } - int n = proc_pidinfo(unwinder->handle.pid, PROC_PIDLISTTHREADS, 0, tids, MAX_NATIVE_THREADS * sizeof(uint64_t)) / sizeof(uint64_t); - if (n <= 0) { - PyMem_Free(tids); - return THREAD_STATE_UNKNOWN; - } - uint64_t min_offset = UINT64_MAX; - for (int i = 0; i < n; i++) { - uint64_t offset = tids[i] - pthread_id; - if (offset < min_offset) { - min_offset = offset; - } - } - unwinder->thread_id_offset = min_offset; - unwinder->thread_id_offset_initialized = 1; - PyMem_Free(tids); - } - struct proc_threadinfo ti; - uint64_t tid_with_offset = pthread_id + unwinder->thread_id_offset; - if (proc_pidinfo(unwinder->handle.pid, PROC_PIDTHREADINFO, tid_with_offset, &ti, sizeof(ti)) != sizeof(ti)) { + struct proc_threadinfo thread; + int size = proc_pidinfo(unwinder->handle.pid, PROC_PIDTHREADID64INFO, + tid, &thread, sizeof(thread)); + if (size != sizeof(thread)) { return THREAD_STATE_UNKNOWN; } - if (ti.pth_run_state == TH_STATE_RUNNING) { - return THREAD_STATE_RUNNING; - } - return THREAD_STATE_IDLE; + return thread.pth_run_state == TH_STATE_RUNNING + ? THREAD_STATE_RUNNING : THREAD_STATE_IDLE; #elif defined(__linux__) char stat_path[256]; char buffer[2048] = ""; - snprintf(stat_path, sizeof(stat_path), "/proc/%d/task/%" PRIu64 "/stat", unwinder->handle.pid, tid); + snprintf(stat_path, sizeof(stat_path), "/proc/%d/task/%lu/stat", unwinder->handle.pid, tid); int fd = open(stat_path, O_RDONLY); if (fd == -1) { @@ -232,46 +217,24 @@ get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread return THREAD_STATE_UNKNOWN; } #elif defined(MS_WINDOWS) - ULONG n; - NTSTATUS status = NtQuerySystemInformation( - SystemProcessInformation, - unwinder->win_process_buffer, - unwinder->win_process_buffer_size, - &n - ); - if (status == STATUS_INFO_LENGTH_MISMATCH) { - // Buffer was too small so we reallocate a larger one and try again. - unwinder->win_process_buffer_size = n; - PVOID new_buffer = PyMem_Realloc(unwinder->win_process_buffer, n); - if (!new_buffer) { - // Match Linux/macOS: degrade gracefully on alloc failure - return THREAD_STATE_UNKNOWN; - } - unwinder->win_process_buffer = new_buffer; - return get_thread_status(unwinder, tid, pthread_id); - } - if (status != STATUS_SUCCESS) { + HANDLE thread = OpenThread(THREAD_QUERY_LIMITED_INFORMATION, FALSE, (DWORD)tid); + if (thread == NULL) { return THREAD_STATE_UNKNOWN; } - - SYSTEM_PROCESS_INFORMATION *pi = (SYSTEM_PROCESS_INFORMATION *)unwinder->win_process_buffer; - while ((ULONG)(ULONG_PTR)pi->UniqueProcessId != unwinder->handle.pid) { - if (pi->NextEntryOffset == 0) { - // Process not found (may have exited) - return THREAD_STATE_UNKNOWN; - } - pi = (SYSTEM_PROCESS_INFORMATION *)(((BYTE *)pi) + pi->NextEntryOffset); - } - - SYSTEM_THREAD_INFORMATION *ti = (SYSTEM_THREAD_INFORMATION *)((char *)pi + sizeof(SYSTEM_PROCESS_INFORMATION)); - for (size_t i = 0; i < pi->NumberOfThreads; i++, ti++) { - if (ti->ClientId.UniqueThread == (HANDLE)tid) { - return ti->ThreadState != WIN32_THREADSTATE_RUNNING ? THREAD_STATE_IDLE : THREAD_STATE_RUNNING; - } + SYSTEM_THREAD_INFORMATION info; + ULONG size = 0; + NTSTATUS result = NtQueryInformationThread( + thread, (THREADINFOCLASS)WIN32_THREAD_SYSTEM_INFORMATION, + &info, sizeof(info), &size); + CloseHandle(thread); + if (result != STATUS_SUCCESS || size != sizeof(info) || + (ULONG_PTR)info.ClientId.UniqueProcess != (ULONG_PTR)unwinder->handle.pid || + (ULONG_PTR)info.ClientId.UniqueThread != tid) + { + return THREAD_STATE_UNKNOWN; } - - // Thread not found (may have exited) - return THREAD_STATE_UNKNOWN; + return info.ThreadState == WIN32_THREAD_RUNNING + ? THREAD_STATE_RUNNING : THREAD_STATE_IDLE; #else return THREAD_STATE_UNKNOWN; #endif @@ -398,7 +361,8 @@ unwind_stack_for_thread( STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); } - long tid = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id); + unsigned long tid = GET_MEMBER(unsigned long, ts, + unwinder->debug_offsets.thread_state.native_thread_id); // Calculate thread status using flags (always) int status_flags = 0; @@ -490,13 +454,9 @@ unwind_stack_for_thread( status_flags |= THREAD_STATUS_HAS_EXCEPTION; } - // Check CPU status - long pthread_id = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.thread_id); - - // Optimization: only check CPU status if needed by mode because it's expensive int cpu_status = THREAD_STATE_UNKNOWN; if (unwinder->mode == PROFILING_MODE_CPU || unwinder->mode == PROFILING_MODE_ALL) { - cpu_status = get_thread_status(unwinder, tid, pthread_id); + cpu_status = get_thread_status(unwinder, tid); } if (cpu_status == THREAD_STATE_UNKNOWN) { @@ -597,7 +557,7 @@ unwind_stack_for_thread( } } if (thread_id == NULL) { - thread_id = PyLong_FromLongLong(tid); + thread_id = PyLong_FromUnsignedLong(tid); if (thread_id == NULL) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); goto error; diff --git a/Python/remote_debug.h b/Python/remote_debug.h index a4a3a010926a7bc..a68e4c6102e5a7d 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -60,19 +60,13 @@ extern "C" { #endif #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX -# include -# include +# include # include -# include # include # include # include -# include # include # include -# include -# include -# include #endif #ifdef MS_WINDOWS @@ -201,7 +195,7 @@ _Py_RemoteDebug_ValidatePyRuntimeCookie(proc_handle_t *handle, uintptr_t address } char buf[sizeof(_Py_Debug_Cookie) - 1]; if (_Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(buf), buf) != 0) { - if (!_Py_RemoteDebug_HasPermissionError()) { + if (!_Py_RemoteDebug_IsFatalReadError()) { PyErr_Clear(); } return 0; @@ -282,6 +276,11 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { CloseHandle(handle->hProcess); handle->hProcess = NULL; } +#elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX + if (handle->task != MACH_PORT_NULL) { + mach_port_deallocate(mach_task_self(), handle->task); + handle->task = MACH_PORT_NULL; + } #elif defined(__linux__) if (handle->memfd != -1) { close(handle->memfd); @@ -294,237 +293,134 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX -static uintptr_t -return_section_address64( - const char* section, - uintptr_t base, - void* map -) { - struct mach_header_64* hdr = (struct mach_header_64*)map; - int ncmds = hdr->ncmds; - - int cmd_cnt = 0; - struct segment_command_64* cmd = map + sizeof(struct mach_header_64); - - uintptr_t vmaddr = 0; - - for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { - if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__TEXT") == 0) { - vmaddr = cmd->vmaddr; - } - if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { - // The section address only needs the image's ASLR slide. - // VM regions need not match the segment's on-disk size. - - int nsects = cmd->nsects; - struct section_64* sec = (struct section_64*)( - (void*)cmd + sizeof(struct segment_command_64) - ); - for (int j = 0; j < nsects; j++) { - if (strcmp(sec[j].sectname, section) == 0) { - return base + sec[j].addr - vmaddr; - } - } - cmd_cnt++; - } - - cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize); - } +enum { + MACHO_NAME_SIZE = 16, + MACHO_MAX_LOAD_COMMAND_BYTES = 16 * 1024 * 1024, + DYLD_MAX_IMAGE_COUNT = 65536, +}; - return 0; -} +/* These 64-bit little-endian layouts work on both arm64 and x86_64 (Rosetta). */ +static_assert(sizeof(uintptr_t) == sizeof(uint64_t), "64-bit macOS required"); static uintptr_t -return_section_address32( - const char* section, - uintptr_t base, - void* map -) { - struct mach_header* hdr = (struct mach_header*)map; - int ncmds = hdr->ncmds; - - int cmd_cnt = 0; - struct segment_command* cmd = map + sizeof(struct mach_header); - - uintptr_t vmaddr = 0; - - for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { - if (cmd->cmd == LC_SEGMENT && strcmp(cmd->segname, "__TEXT") == 0) { - vmaddr = cmd->vmaddr; - } - if (cmd->cmd == LC_SEGMENT && strcmp(cmd->segname, "__DATA") == 0) { - // The section address only needs the image's ASLR slide. - // VM regions need not match the segment's on-disk size. - - int nsects = cmd->nsects; - struct section* sec = (struct section*)( - (void*)cmd + sizeof(struct segment_command) - ); - for (int j = 0; j < nsects; j++) { - if (strcmp(sec[j].sectname, section) == 0) { - return base + sec[j].addr - vmaddr; - } - } - cmd_cnt++; - } - - cmd = (struct segment_command*)((void*)cmd + cmd->cmdsize); - } - - return 0; -} - -static uintptr_t -return_section_address_fat( - const char* section, - uintptr_t base, - void* map -) { - struct fat_header* fat_hdr = (struct fat_header*)map; - - // Determine host CPU type for architecture selection - cpu_type_t cpu; - int is_abi64; - size_t cpu_size = sizeof(cpu), abi64_size = sizeof(is_abi64); - - if (sysctlbyname("hw.cputype", &cpu, &cpu_size, NULL, 0) != 0) { - int err = errno; - _set_debug_oserror_from_errno(err, - "Failed to determine CPU type via sysctlbyname " - "for fat binary analysis at 0x%lx: %s", - base, strerror(err)); +macho_section_address(const void *image, size_t image_size, + uintptr_t base, const char *name) +{ + struct mach_header_64 header; + if (image_size < sizeof(header)) { return 0; } - if (sysctlbyname("hw.cpu64bit_capable", &is_abi64, &abi64_size, NULL, 0) != 0) { - int err = errno; - _set_debug_oserror_from_errno(err, - "Failed to determine CPU ABI capability via sysctlbyname " - "for fat binary analysis at 0x%lx: %s", - base, strerror(err)); + memcpy(&header, image, sizeof(header)); + if (header.magic != MH_MAGIC_64 || + header.sizeofcmds > image_size - sizeof(header)) + { return 0; } - cpu |= is_abi64 * CPU_ARCH_ABI64; - - // Check endianness - int swap = fat_hdr->magic == FAT_CIGAM; - struct fat_arch* arch = (struct fat_arch*)(map + sizeof(struct fat_header)); - - // Get number of architectures in fat binary - uint32_t nfat_arch = swap ? __builtin_bswap32(fat_hdr->nfat_arch) : fat_hdr->nfat_arch; - - // Search for matching architecture - for (uint32_t i = 0; i < nfat_arch; i++) { - cpu_type_t arch_cpu = swap ? __builtin_bswap32(arch[i].cputype) : arch[i].cputype; - - if (arch_cpu == cpu) { - // Found matching architecture, now process it - uint32_t offset = swap ? __builtin_bswap32(arch[i].offset) : arch[i].offset; - struct mach_header_64* hdr = (struct mach_header_64*)(map + offset); - - // Determine which type of Mach-O it is and process accordingly - switch (hdr->magic) { - case MH_MAGIC: - case MH_CIGAM: - return return_section_address32(section, base, (void*)hdr); - - case MH_MAGIC_64: - case MH_CIGAM_64: - return return_section_address64(section, base, (void*)hdr); - - default: - PyErr_Format(PyExc_RuntimeError, - "Unknown Mach-O magic number 0x%x in fat binary architecture %u at base 0x%lx", - hdr->magic, i, base); - return 0; + const unsigned char *cursor = (const unsigned char *)image + sizeof(header); + size_t remaining = header.sizeofcmds; + uint64_t header_address = 0, section_address = 0; + int have_header = 0, have_section = 0; + for (uint32_t index = 0; index < header.ncmds; index++) { + struct load_command command; + if (remaining < sizeof(command)) { + return 0; + } + memcpy(&command, cursor, sizeof(command)); + size_t command_size = command.cmdsize; + if (command_size < sizeof(command) || command_size > remaining) { + return 0; + } + if (command.cmd == LC_SEGMENT_64) { + struct segment_command_64 segment; + if (command_size < sizeof(segment)) { + return 0; + } + memcpy(&segment, cursor, sizeof(segment)); + if (segment.nsects > + (command_size - sizeof(segment)) / sizeof(struct section_64)) + { + return 0; + } + /* __PAGEZERO has no file contents and does not define the slide. */ + if (segment.fileoff == 0 && segment.filesize != 0) { + header_address = segment.vmaddr; + have_header = 1; + } + if (strncmp(segment.segname, "__DATA", MACHO_NAME_SIZE) == 0) { + const unsigned char *record = cursor + sizeof(segment); + for (uint32_t item = 0; item < segment.nsects; + item++, record += sizeof(struct section_64)) + { + struct section_64 section; + memcpy(§ion, record, sizeof(section)); + if (strncmp(section.sectname, name, MACHO_NAME_SIZE) == 0) { + section_address = section.addr; + have_section = 1; + } + } } } + cursor += command_size; + remaining -= command_size; } - - PyErr_Format(PyExc_RuntimeError, - "No matching architecture found for CPU type 0x%x " - "in fat binary at base 0x%lx (%u architectures examined)", - cpu, base, nfat_arch); - return 0; + if (!have_header || !have_section || section_address < header_address || + section_address - header_address > UINTPTR_MAX - base) + { + return 0; + } + return base + (uintptr_t)(section_address - header_address); } static uintptr_t -search_section_in_file(const char* secname, char* path, uintptr_t base) +search_section_in_memory(proc_handle_t *handle, uintptr_t base, + const char *section) { - int fd = open(path, O_RDONLY); - if (fd == -1) { - int err = errno; - _set_debug_oserror_from_errno_with_filename(err, path, - "Cannot open binary file '%s' for section '%s' search: %s", - path, secname, strerror(err)); - return 0; + /* Most images fit in one read. Stay within the smallest macOS page size. */ + unsigned char prefix[4096]; + size_t prefix_size = sizeof(prefix) - base % sizeof(prefix); + struct mach_header_64 header; + if (prefix_size < sizeof(header) || + _Py_RemoteDebug_ReadRemoteMemory(handle, base, prefix_size, prefix) < 0) + { + if (_Py_RemoteDebug_IsFatalReadError()) { + return 0; + } + PyErr_Clear(); + prefix_size = sizeof(header); + if (_Py_RemoteDebug_ReadRemoteMemory(handle, base, prefix_size, prefix) < 0) { + return 0; + } } - - struct stat fs; - if (fstat(fd, &fs) == -1) { - int err = errno; - _set_debug_oserror_from_errno_with_filename(err, path, - "Cannot get file size for binary '%s' during section '%s' search: %s", - path, secname, strerror(err)); - close(fd); + memcpy(&header, prefix, sizeof(header)); + if (header.magic != MH_MAGIC_64 || + header.sizeofcmds > MACHO_MAX_LOAD_COMMAND_BYTES) + { return 0; } - - void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); - if (map == MAP_FAILED) { - int err = errno; - _set_debug_oserror_from_errno_with_filename(err, path, - "Cannot memory map binary file '%s' (size: %lld bytes) for section '%s' search: %s", - path, (long long)fs.st_size, secname, strerror(err)); - close(fd); + size_t size = sizeof(header) + header.sizeofcmds; + if (base > UINTPTR_MAX - size) { return 0; } - - uintptr_t result = 0; - uint32_t magic = *(uint32_t*)map; - - switch (magic) { - case MH_MAGIC: - case MH_CIGAM: - result = return_section_address32(secname, base, map); - break; - case MH_MAGIC_64: - case MH_CIGAM_64: - result = return_section_address64(secname, base, map); - break; - case FAT_MAGIC: - case FAT_CIGAM: - result = return_section_address_fat(secname, base, map); - break; - default: - PyErr_Format(PyExc_RuntimeError, - "Unrecognized Mach-O magic number 0x%x in binary file '%s' for section '%s' search", - magic, path, secname); - break; + if (size <= prefix_size) { + return macho_section_address(prefix, size, base, section); } - - if (munmap(map, fs.st_size) != 0) { - if (!PyErr_Occurred()) { - int err = errno; - _set_debug_oserror_from_errno_with_filename(err, path, - "Failed to unmap binary file '%s' (size: %lld bytes): %s", - path, (long long)fs.st_size, strerror(err)); - } - result = 0; + unsigned char *image = PyMem_Malloc(size); + if (image == NULL) { + PyErr_NoMemory(); + return 0; } - if (close(fd) != 0) { - if (!PyErr_Occurred()) { - int err = errno; - _set_debug_oserror_from_errno_with_filename(err, path, - "Failed to close binary file '%s': %s", - path, strerror(err)); - } - result = 0; + memcpy(image, prefix, prefix_size); + uintptr_t result = 0; + if (_Py_RemoteDebug_ReadRemoteMemory(handle, base + prefix_size, + size - prefix_size, image + prefix_size) == 0) + { + result = macho_section_address(image, size, base, section); } + PyMem_Free(image); return result; } - static mach_port_t pid_to_task(pid_t pid) { @@ -542,86 +438,166 @@ pid_to_task(pid_t pid) return task; } -static uintptr_t -search_map_for_section(proc_handle_t *handle, const char* secname, const char* substr, - section_validator_t validator) { - mach_vm_address_t address = 0; - mach_vm_size_t size = 0; - mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); - vm_region_basic_info_data_64_t region_info; - mach_port_t object_name; - - mach_port_t proc_ref = pid_to_task(handle->pid); - if (proc_ref == 0) { - if (!PyErr_Occurred()) { - PyErr_Format(PyExc_PermissionError, - "Cannot get task port for PID %d during section search", - handle->pid); - } - return 0; +static int +read_dyld_image_metadata(proc_handle_t *handle, struct dyld_all_image_infos *images) +{ + task_dyld_info_data_t info = {0}; + mach_msg_type_number_t count = TASK_DYLD_INFO_COUNT; + kern_return_t status = task_info(handle->task, TASK_DYLD_INFO, + (task_info_t)&info, &count); + if (status != KERN_SUCCESS) { + PyErr_Format(status == KERN_PROTECTION_FAILURE + ? PyExc_PermissionError : PyExc_RuntimeError, + "Cannot read dyld information for PID %d (Mach error %d)", + handle->pid, status); + return -1; } - - char map_filename[MAXPATHLEN + 1]; - - kern_return_t kr; - while ((kr = mach_vm_region( - proc_ref, - &address, - &size, - VM_REGION_BASIC_INFO_64, - (vm_region_info_t)®ion_info, - &count, - &object_name)) == KERN_SUCCESS) + if (count < TASK_DYLD_INFO_COUNT || + info.all_image_info_format != TASK_DYLD_ALL_IMAGE_INFO_64) + { + PyErr_SetString(PyExc_RuntimeError, "Unsupported dyld image-info format"); + return -1; + } + size_t uuid_end = offsetof(struct dyld_all_image_infos, uuidArray) + + sizeof(images->uuidArray); + size_t size = (size_t)Py_MIN(info.all_image_info_size, uuid_end); + if (size < offsetof(struct dyld_all_image_infos, infoArray) + + sizeof(images->infoArray) || + info.all_image_info_addr > UINTPTR_MAX - size) { + PyErr_SetString(PyExc_RuntimeError, "Invalid dyld image information"); + return -1; + } + if (_Py_RemoteDebug_ReadRemoteMemory(handle, info.all_image_info_addr, + size, images) < 0) + { + return -1; + } + /* The UUID array was added in version 8 of dyld_all_image_infos. */ + if (images->version < 8 || size < uuid_end) { + images->uuidArray = NULL; + images->uuidArrayCount = 0; + } + return 0; +} - if ((region_info.protection & VM_PROT_READ) == 0 - || (region_info.protection & VM_PROT_EXECUTE) == 0) { - address += size; - continue; +static uintptr_t +search_memory_regions_for_section(proc_handle_t *handle, const char *section, + section_validator_t validator) +{ + mach_vm_address_t address = 0; + while (1) { + mach_vm_size_t size = 0; + vm_region_basic_info_data_64_t region; + mach_msg_type_number_t count = VM_REGION_BASIC_INFO_COUNT_64; + mach_port_t object = MACH_PORT_NULL; + kern_return_t status = mach_vm_region( + handle->task, &address, &size, VM_REGION_BASIC_INFO_64, + (vm_region_info_t)®ion, &count, &object); + if (object != MACH_PORT_NULL) { + mach_port_deallocate(mach_task_self(), object); } - - int path_len = proc_regionfilename( - handle->pid, address, map_filename, MAXPATHLEN); - if (path_len == 0) { - address += size; - continue; + if (status == KERN_INVALID_ADDRESS) { + return 0; } - - char* filename = strrchr(map_filename, '/'); - if (filename != NULL) { - filename++; // Move past the '/' - } else { - filename = map_filename; // No path, use the whole string + if (status != KERN_SUCCESS) { + PyErr_Format(status == KERN_PROTECTION_FAILURE + ? PyExc_PermissionError : PyExc_RuntimeError, + "Cannot read memory regions for PID %d (Mach error %d)", + handle->pid, status); + return 0; } - - if (strncmp(filename, substr, strlen(substr)) == 0) { - PyErr_Clear(); - uintptr_t result = search_section_in_file( - secname, map_filename, address); - if (result != 0) { - if (validator == NULL || validator(handle, result)) { - return result; - } - if (_Py_RemoteDebug_HasPermissionError()) { - return 0; - } + vm_prot_t required = VM_PROT_READ | VM_PROT_EXECUTE; + if ((region.protection & required) == required) { + uintptr_t candidate = search_section_in_memory(handle, address, section); + if (candidate != 0 && (validator == NULL || validator(handle, candidate))) { + return candidate; } - else if (_Py_RemoteDebug_HasPermissionError()) { + if (_Py_RemoteDebug_IsFatalReadError()) { return 0; } + PyErr_Clear(); + } + if (size == 0 || size > UINT64_MAX - address) { + return 0; } - address += size; } +} - if (kr != KERN_INVALID_ADDRESS && !PyErr_Occurred()) { - PyErr_Format(PyExc_RuntimeError, - "mach_vm_region failed while searching PID %d for section '%s' " - "(kern_return_t: %d)", - handle->pid, secname, kr); +static uintptr_t +search_dyld_image_array(proc_handle_t *handle, uintptr_t array_address, + size_t count, size_t record_size, + const char *section, section_validator_t validator) +{ + if (array_address == 0 || count == 0 || count > DYLD_MAX_IMAGE_COUNT || + array_address > UINTPTR_MAX - count * record_size) + { + return 0; + } + size_t size = count * record_size; + unsigned char *images = PyMem_Malloc(size); + if (images == NULL) { + PyErr_NoMemory(); + return 0; + } + uintptr_t result = 0; + if (_Py_RemoteDebug_ReadRemoteMemory(handle, array_address, size, images) < 0) { + goto done; + } + for (size_t index = 0; index < count; index++) { + /* Both dyld_image_info and dyld_uuid_info begin with imageLoadAddress. */ + uintptr_t address; + memcpy(&address, images + index * record_size, sizeof(address)); + if (address == 0) { + continue; + } + uintptr_t candidate = search_section_in_memory(handle, address, section); + if (candidate != 0 && (validator == NULL || validator(handle, candidate))) { + result = candidate; + break; + } + if (_Py_RemoteDebug_IsFatalReadError()) { + break; + } + /* A library can unload after the image list is read. */ + PyErr_Clear(); } +done: + PyMem_Free(images); + if (!_Py_RemoteDebug_IsFatalReadError()) { + PyErr_Clear(); + } + return result; +} - return 0; +static uintptr_t +search_loaded_images_for_section(proc_handle_t *handle, const char *section, + section_validator_t validator) +{ + struct dyld_all_image_infos images = {0}; + if (read_dyld_image_metadata(handle, &images) == 0) { + /* Python is normally outside the shared cache: try the shorter list. */ + uintptr_t result = search_dyld_image_array( + handle, (uintptr_t)images.uuidArray, images.uuidArrayCount, + sizeof(struct dyld_uuid_info), section, validator); + if (result != 0 || _Py_RemoteDebug_IsFatalReadError()) { + return result; + } + result = search_dyld_image_array( + handle, (uintptr_t)images.infoArray, images.infoArrayCount, + sizeof(struct dyld_image_info), section, validator); + if (result != 0 || _Py_RemoteDebug_IsFatalReadError()) { + return result; + } + } + if (_Py_RemoteDebug_IsFatalReadError()) { + return 0; + } + /* dyld can leave both lists unavailable while a stopped target is loading + a library. Treat copied lists as hints and fall back to mapped images. */ + PyErr_Clear(); + return search_memory_regions_for_section(handle, section, validator); } #endif // (__APPLE__ && defined(TARGET_OS_OSX) && TARGET_OS_OSX) @@ -1173,22 +1149,14 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) } } #elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX - // On macOS, try libpython first, then fall back to python - const char* candidates[] = {"libpython", "python", "Python", NULL}; - for (const char** candidate = candidates; *candidate; candidate++) { - PyErr_Clear(); - address = search_map_for_section(handle, "PyRuntime", *candidate, - _Py_RemoteDebug_ValidatePyRuntimeCookie); - if (address != 0 || _Py_RemoteDebug_HasPermissionError()) { - break; - } - } + address = search_loaded_images_for_section( + handle, "PyRuntime", _Py_RemoteDebug_ValidatePyRuntimeCookie); if (address == 0) { if (!_Py_RemoteDebug_HasPermissionError()) { PyObject *exc = PyErr_GetRaisedException(); PyErr_Format(PyExc_RuntimeError, "Failed to find the PyRuntime section in process %d " - "on macOS platform (tried both libpython and python)", + "on macOS platform", handle->pid); _PyErr_ChainExceptions1(exc); } @@ -1665,7 +1633,6 @@ _Py_RemoteDebug_ReadDebugOffsets( "Failed to locate PyRuntime address for PID %d", handle->pid); } - _set_debug_exception_cause(PyExc_RuntimeError, "PyRuntime address lookup failed during debug offsets initialization"); return -1; } size_t size = sizeof(struct _Py_DebugOffsets); From 2e14c838544c6c36b9095544b4b6c994d3832678 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 16 Nov 2026 10:04:35 +0900 Subject: [PATCH 156/229] no-issue: Add duplicate-issue checks to AGENTS.md (gh-162001) --- AGENTS.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index d23d0c9315d3f31..ce4489146a9ee92 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,3 +14,16 @@ When acting on this repository, apply the policy's core principles: - Follow existing coding style and patterns. - Write tests that exercise the change. - Keep backwards compatibility with prior releases in mind. + +## Opening issues + +Do not auto-generate issues with AI tools. + +Search GitHub for duplicate issues before opening a new issue. Do not open +duplicates. + +```bash +gh search issues --repo python/cpython "" +gh search issues --repo python/cpython "" +gh issue list --repo python/cpython --state all --label "" --search "" +``` From 7c59816cb4d3db8387c454ce33d6b56c7d93fa6e Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 16 Nov 2026 11:05:00 +0900 Subject: [PATCH 157/229] gh-161934: Split TextIOWrapper.write into text and byte helpers (#161991) --- Modules/_io/textio.c | 206 +++++++++++++++++++++++-------------------- 1 file changed, 111 insertions(+), 95 deletions(-) diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 4929e41ba9ccb41..719f9ef36f49570 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1752,95 +1752,14 @@ textiowrapper_check_write(textio *self) return Py_None; } -/*[clinic input] -@critical_section -_io.TextIOWrapper.write - text: unicode - / -[clinic start generated code]*/ - -static PyObject * -_io_TextIOWrapper_write_impl(textio *self, PyObject *text) -/*[clinic end generated code: output=d2deb0d50771fcec input=73ec95c5c4a3489c]*/ +/* Consume b, which is either bytes or an ASCII string. */ +static int +textiowrapper_write_bytes(textio *self, PyObject *b, + int needflush, int text_needflush) { - PyObject *ret; - PyObject *b; - Py_ssize_t textlen; - int haslf = 0; - int needflush = 0, text_needflush = 0; - - if (textiowrapper_check_write(self) == NULL) { - return NULL; - } - - Py_INCREF(text); - - textlen = PyUnicode_GET_LENGTH(text); - - if ((self->writetranslate && self->writenl != NULL) || self->line_buffering) - if (PyUnicode_FindChar(text, '\n', 0, PyUnicode_GET_LENGTH(text), 1) != -1) - haslf = 1; - - if (haslf && self->writetranslate && self->writenl != NULL) { - PyObject *newtext = _PyObject_CallMethod(text, &_Py_ID(replace), - "ss", "\n", self->writenl); - Py_DECREF(text); - if (newtext == NULL) - return NULL; - if (!PyUnicode_Check(newtext)) { - PyErr_Format(PyExc_TypeError, - "replace() should return a str, not %T", newtext); - Py_DECREF(newtext); - return NULL; - } - /* replace() may have closed, detached or reinitialized the stream. */ - if (textiowrapper_check_write(self) == NULL) { - Py_DECREF(newtext); - return NULL; - } - text = newtext; - } - - if (self->write_through) - text_needflush = 1; - if (self->line_buffering && - (haslf || - PyUnicode_FindChar(text, '\r', 0, PyUnicode_GET_LENGTH(text), 1) != -1)) - needflush = 1; - - /* XXX What if we were just reading? */ - if (self->encodefunc != NULL) { - if (PyUnicode_IS_ASCII(text) && - // See bpo-43260 - PyUnicode_GET_LENGTH(text) <= self->chunk_size && - is_asciicompat_encoding(self->encodefunc)) { - b = Py_NewRef(text); - } - else { - b = (*self->encodefunc)((PyObject *) self, text); - } - self->encoding_start_of_stream = 0; - } - else { - /* A codec callback may reconfigure the stream. */ - PyObject *encoder = Py_NewRef(self->encoder); - b = PyObject_CallMethodOneArg(encoder, &_Py_ID(encode), text); - Py_DECREF(encoder); - } - - Py_DECREF(text); - if (b == NULL) - return NULL; - if (b != text && !PyBytes_Check(b)) { - PyErr_Format(PyExc_TypeError, - "encoder should return a bytes object, not '%.200s'", - Py_TYPE(b)->tp_name); - Py_DECREF(b); - return NULL; - } - Py_ssize_t bytes_len; - if (b == text) { + if (PyUnicode_Check(b)) { + assert(PyUnicode_IS_ASCII(b)); bytes_len = PyUnicode_GET_LENGTH(b); } else { @@ -1860,7 +1779,7 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) while (self->pending_bytes != NULL) { if (_textiowrapper_writeflush(self) < 0) { Py_DECREF(b); - return NULL; + return -1; } } } @@ -1874,7 +1793,7 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) PyObject *list = PyList_New(2); if (list == NULL) { Py_DECREF(b); - return NULL; + return -1; } // Since Python 3.12, allocating GC object won't trigger GC and release // GIL. See https://github.com/python/cpython/issues/97922 @@ -1886,7 +1805,7 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) else { if (PyList_Append(self->pending_bytes, b) < 0) { Py_DECREF(b); - return NULL; + return -1; } Py_DECREF(b); } @@ -1900,18 +1819,18 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) if (self->pending_bytes_count >= self->chunk_size || needflush || text_needflush) { if (_textiowrapper_writeflush(self) < 0) - return NULL; + return -1; } if (needflush) { PyObject *buffer = buffer_access_safe(self); if (buffer == NULL) { - return NULL; + return -1; } int res = _PyFile_Flush(buffer); Py_DECREF(buffer); if (res < 0) { - return NULL; + return -1; } } @@ -1921,15 +1840,112 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) } if (self->decoder) { - ret = PyObject_CallMethodNoArgs(self->decoder, &_Py_ID(reset)); + PyObject *ret = PyObject_CallMethodNoArgs(self->decoder, &_Py_ID(reset)); if (ret == NULL) - return NULL; + return -1; Py_DECREF(ret); } + return 0; +} + +static PyObject * +textiowrapper_write_text(textio *self, PyObject *text) +{ + PyObject *b; + Py_ssize_t textlen; + int haslf = 0; + int needflush = 0, text_needflush = 0; + + Py_INCREF(text); + + textlen = PyUnicode_GET_LENGTH(text); + + if ((self->writetranslate && self->writenl != NULL) || self->line_buffering) + if (PyUnicode_FindChar(text, '\n', 0, PyUnicode_GET_LENGTH(text), 1) != -1) + haslf = 1; + + if (haslf && self->writetranslate && self->writenl != NULL) { + PyObject *newtext = _PyObject_CallMethod(text, &_Py_ID(replace), + "ss", "\n", self->writenl); + Py_DECREF(text); + if (newtext == NULL) + return NULL; + if (!PyUnicode_Check(newtext)) { + PyErr_Format(PyExc_TypeError, + "replace() should return a str, not %T", newtext); + Py_DECREF(newtext); + return NULL; + } + /* replace() may have closed, detached or reinitialized the stream. */ + if (textiowrapper_check_write(self) == NULL) { + Py_DECREF(newtext); + return NULL; + } + text = newtext; + } + + if (self->write_through) + text_needflush = 1; + if (self->line_buffering && + (haslf || + PyUnicode_FindChar(text, '\r', 0, PyUnicode_GET_LENGTH(text), 1) != -1)) + needflush = 1; + + /* XXX What if we were just reading? */ + if (self->encodefunc != NULL) { + if (PyUnicode_IS_ASCII(text) && + // See bpo-43260 + PyUnicode_GET_LENGTH(text) <= self->chunk_size && + is_asciicompat_encoding(self->encodefunc)) { + b = Py_NewRef(text); + } + else { + b = (*self->encodefunc)((PyObject *) self, text); + } + self->encoding_start_of_stream = 0; + } + else { + /* A codec callback may reconfigure the stream. */ + PyObject *encoder = Py_NewRef(self->encoder); + b = PyObject_CallMethodOneArg(encoder, &_Py_ID(encode), text); + Py_DECREF(encoder); + } + + Py_DECREF(text); + if (b == NULL) + return NULL; + if (b != text && !PyBytes_Check(b)) { + PyErr_Format(PyExc_TypeError, + "encoder should return a bytes object, not '%.200s'", + Py_TYPE(b)->tp_name); + Py_DECREF(b); + return NULL; + } + + if (textiowrapper_write_bytes(self, b, needflush, text_needflush) < 0) { + return NULL; + } return PyLong_FromSsize_t(textlen); } +/*[clinic input] +@critical_section +_io.TextIOWrapper.write + text: unicode + / +[clinic start generated code]*/ + +static PyObject * +_io_TextIOWrapper_write_impl(textio *self, PyObject *text) +/*[clinic end generated code: output=d2deb0d50771fcec input=73ec95c5c4a3489c]*/ +{ + if (textiowrapper_check_write(self) == NULL) { + return NULL; + } + return textiowrapper_write_text(self, text); +} + /* Steal a reference to chars and store it in the decoded_char buffer; */ static void From ea678b2e08b57e4c23fdd993e168d3642d9ba7bb Mon Sep 17 00:00:00 2001 From: Ferdinando Ametrano Date: Mon, 16 Nov 2026 09:19:28 +0100 Subject: [PATCH 158/229] gh-162034: Quote the utils.sh path in the Apple testbed build phase (#162035) Modifies the iOS testbed to ensure that the path to the `utils.sh` script is quoted so that spaces in the path don't prevent the script from operating. --- Doc/using/ios.rst | 2 +- .../next/Tests/2026-11-15-09-43-15.gh-issue-162034.egx5QV.rst | 2 ++ Platforms/Apple/testbed/Python.xcframework/build/utils.sh | 2 +- Platforms/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj | 2 +- Platforms/Apple/testbed/tvOSTestbed.xcodeproj/project.pbxproj | 2 +- .../Apple/testbed/visionOSTestbed.xcodeproj/project.pbxproj | 2 +- .../Apple/testbed/watchOSTestbed.xcodeproj/project.pbxproj | 2 +- 7 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-11-15-09-43-15.gh-issue-162034.egx5QV.rst diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index d7cf699d5bc3bfb..4dbb6e77459c480 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -226,7 +226,7 @@ To add Python to an iOS Xcode project: .. code-block:: bash set -e - source $PROJECT_DIR/Python.xcframework/build/build_utils.sh + source "$PROJECT_DIR/Python.xcframework/build/utils.sh" install_python Python.xcframework app If you have placed your XCframework somewhere other than the root of your diff --git a/Misc/NEWS.d/next/Tests/2026-11-15-09-43-15.gh-issue-162034.egx5QV.rst b/Misc/NEWS.d/next/Tests/2026-11-15-09-43-15.gh-issue-162034.egx5QV.rst new file mode 100644 index 000000000000000..1b7d2c4ad5f27c6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-11-15-09-43-15.gh-issue-162034.egx5QV.rst @@ -0,0 +1,2 @@ +The Apple testbed projects now build when the path of the project contains a +space. diff --git a/Platforms/Apple/testbed/Python.xcframework/build/utils.sh b/Platforms/Apple/testbed/Python.xcframework/build/utils.sh index bbe3620a1397311..7e2cda78be57690 100755 --- a/Platforms/Apple/testbed/Python.xcframework/build/utils.sh +++ b/Platforms/Apple/testbed/Python.xcframework/build/utils.sh @@ -14,7 +14,7 @@ # * There is an `app_packages` folder that contains installed Python packages. # ----- # set -e -# source $PROJECT_DIR/Python.xcframework/build/build_utils.sh +# source "$PROJECT_DIR/Python.xcframework/build/utils.sh" # install_python Python.xcframework app app_packages # ----- diff --git a/Platforms/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj b/Platforms/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj index 6ac40a325045bd6..b2131822424044e 100644 --- a/Platforms/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/Platforms/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -250,7 +250,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\nsource $PROJECT_DIR/Python{{abiflags}}.xcframework/build/utils.sh\ninstall_python Python{{abiflags}}.xcframework app app_packages\n"; + shellScript = "set -e\nsource \"$PROJECT_DIR/Python{{abiflags}}.xcframework/build/utils.sh\"\ninstall_python Python{{abiflags}}.xcframework app app_packages\n"; showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ diff --git a/Platforms/Apple/testbed/tvOSTestbed.xcodeproj/project.pbxproj b/Platforms/Apple/testbed/tvOSTestbed.xcodeproj/project.pbxproj index e14489ad0770a0f..941f6a266bc42a0 100644 --- a/Platforms/Apple/testbed/tvOSTestbed.xcodeproj/project.pbxproj +++ b/Platforms/Apple/testbed/tvOSTestbed.xcodeproj/project.pbxproj @@ -236,7 +236,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\nsource $PROJECT_DIR/Python{{abiflags}}.xcframework/build/utils.sh\n\ninstall_python Python{{abiflags}}.xcframework app app_packages\n"; + shellScript = "set -e\n\nsource \"$PROJECT_DIR/Python{{abiflags}}.xcframework/build/utils.sh\"\n\ninstall_python Python{{abiflags}}.xcframework app app_packages\n"; showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ diff --git a/Platforms/Apple/testbed/visionOSTestbed.xcodeproj/project.pbxproj b/Platforms/Apple/testbed/visionOSTestbed.xcodeproj/project.pbxproj index ee19e22d533cc83..53ddec1b6f29e40 100644 --- a/Platforms/Apple/testbed/visionOSTestbed.xcodeproj/project.pbxproj +++ b/Platforms/Apple/testbed/visionOSTestbed.xcodeproj/project.pbxproj @@ -250,7 +250,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\nsource $PROJECT_DIR/Python{{abiflags}}.xcframework/build/utils.sh\n\ninstall_python Python{{abiflags}}.xcframework app app_packages\n"; + shellScript = "set -e\n\nsource \"$PROJECT_DIR/Python{{abiflags}}.xcframework/build/utils.sh\"\n\ninstall_python Python{{abiflags}}.xcframework app app_packages\n"; showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ diff --git a/Platforms/Apple/testbed/watchOSTestbed.xcodeproj/project.pbxproj b/Platforms/Apple/testbed/watchOSTestbed.xcodeproj/project.pbxproj index d5220c60e65c441..02b98a685cea6c0 100644 --- a/Platforms/Apple/testbed/watchOSTestbed.xcodeproj/project.pbxproj +++ b/Platforms/Apple/testbed/watchOSTestbed.xcodeproj/project.pbxproj @@ -236,7 +236,7 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\nsource $PROJECT_DIR/Python{{abiflags}}.xcframework/build/utils.sh\n\ninstall_python Python{{abiflags}}.xcframework app app_packages\n"; + shellScript = "set -e\n\nsource \"$PROJECT_DIR/Python{{abiflags}}.xcframework/build/utils.sh\"\n\ninstall_python Python{{abiflags}}.xcframework app app_packages\n"; showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ From 91ab655eb519d7c48e78934e00295e3c755b04a9 Mon Sep 17 00:00:00 2001 From: Timofei Ivankov <128279579+deadlovelll@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:31:30 +0300 Subject: [PATCH 159/229] Add a documentation URL field to the docs issue template (#162086) --- .github/ISSUE_TEMPLATE/documentation.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml index d720cf9c4de91e2..d21e2a88604087f 100644 --- a/.github/ISSUE_TEMPLATE/documentation.yml +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -7,10 +7,16 @@ body: value: | > [!NOTE] > Trivial changes (for example typos) don’t require an issue before opening a PR. + - type: input + id: url + attributes: + label: "Documentation URL" + description: "A link to the relevant documentation page, if applicable." + placeholder: "https://docs.python.org/3/..." - type: textarea id: description attributes: label: "Documentation" - description: "A clear and concise description of the issue. Include a link to the page." + description: "A clear and concise description of the issue." validations: required: true From 585755d87aa57db35b233cbdca2dfcda2b3c20e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Mon, 16 Nov 2026 12:49:16 +0100 Subject: [PATCH 160/229] gh-149584: _remote_debugging: Simplify asyncio result construction (GH-162101) Use a shared CoroInfo constructor for tasks and futures to remove duplicate allocation and cleanup. --- Lib/test/test_external_inspection.py | 16 +++++++ Modules/_remote_debugging/asyncio.c | 70 ++++++++++------------------ 2 files changed, 40 insertions(+), 46 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 104efaebd85c16f..edcac9d141ff5aa 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -520,6 +520,22 @@ async def rec(n): self.assertEqual(asyncio.run(rec(3)), ["rec"] * 4) + @skip_if_not_supported + def test_future_waiter_has_empty_stack(self): + async def main(): + task = asyncio.current_task() + future = asyncio.get_running_loop().create_future() + asyncio.future_add_to_awaited_by(task, future) + try: + trace = RemoteUnwinder(os.getpid()).get_all_awaited_by() + task_info = self._get_task_id_map(trace)[id(task)] + self.assertEqual(task_info.awaited_by, [([], id(future))]) + self.assertTrue(task_info.coroutine_stack[0].call_stack) + finally: + asyncio.future_discard_from_awaited_by(task, future) + + asyncio.run(main()) + @skip_if_not_supported def test_async_generator_stack_is_not_truncated(self): # gh-159620 diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index 11fa40eb3e5f00c..f9fcea54cacefea 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -422,13 +422,12 @@ parse_coro_chain( static PyObject* create_task_result( RemoteUnwinderObject *unwinder, - uintptr_t task_address + uintptr_t task_address, + char is_task ) { PyObject* result = NULL; PyObject *call_stack = NULL; PyObject *tn = NULL; - char task_obj[SIZEOF_TASK_OBJ]; - uintptr_t coro_addr; // Create call_stack first since it's the first tuple element call_stack = PyList_New(0); @@ -444,25 +443,27 @@ create_task_result( goto error; } - // Parse coroutine chain - if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, - (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, - task_obj) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object for coro chain"); - goto error; - } - - coro_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_coro); - - if ((void*)coro_addr != NULL) { - if (parse_coro_chain(unwinder, coro_addr, call_stack) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse coroutine chain"); + if (is_task) { + char task_obj[SIZEOF_TASK_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object for coro chain"); goto error; } - if (PyList_Reverse(call_stack)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to reverse call stack"); - goto error; + uintptr_t coro_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_coro); + + if ((void*)coro_addr != NULL) { + if (parse_coro_chain(unwinder, coro_addr, call_stack) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse coroutine chain"); + goto error; + } + + if (PyList_Reverse(call_stack)) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to reverse call stack"); + goto error; + } } } @@ -506,33 +507,10 @@ parse_task( goto error; } - if (is_task) { - result = create_task_result(unwinder, task_address); - if (!result) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task result"); - goto error; - } - } else { - // Create an empty CoroInfo for non-task objects - RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); - result = PyStructSequence_New(state->CoroInfo_Type); - if (result == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create empty CoroInfo"); - goto error; - } - PyObject *empty_list = PyList_New(0); - if (empty_list == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create empty list"); - goto error; - } - PyObject *task_name = PyLong_FromUnsignedLongLong(task_address); - if (task_name == NULL) { - Py_DECREF(empty_list); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task name"); - goto error; - } - PyStructSequence_SetItem(result, 0, empty_list); // This steals the reference - PyStructSequence_SetItem(result, 1, task_name); // This steals the reference + result = create_task_result(unwinder, task_address, is_task); + if (result == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task result"); + goto error; } if (PyList_Append(render_to, result)) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append task result to render list"); From 6f049d37d9b655c14f183784d8a6ddd9eb9dd147 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Mon, 16 Nov 2026 13:05:46 +0100 Subject: [PATCH 161/229] gh-149584: _remote_debugging: Deduplicate AwaitedInfo construction (GH-162104) Use one helper to construct and append AwaitedInfo results for both asyncio APIs. Keep reference ownership and cleanup in the shared helper. --- Lib/test/test_external_inspection.py | 19 +++++++- Modules/_remote_debugging/asyncio.c | 73 ++++++++++++---------------- 2 files changed, 50 insertions(+), 42 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index edcac9d141ff5aa..cb9aae3345cff9c 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -55,7 +55,7 @@ try: from _remote_debugging import PROCESS_VM_READV_SUPPORTED from _remote_debugging import RemoteUnwinder - from _remote_debugging import FrameInfo, CoroInfo, TaskInfo + from _remote_debugging import FrameInfo, CoroInfo, TaskInfo, AwaitedInfo except ImportError: raise unittest.SkipTest( "Test only runs when _remote_debugging is available" @@ -487,6 +487,23 @@ def _extract_coroutine_funcnames(self, trace): for task in info.awaited_by } + @skip_if_not_supported + def test_async_awaited_info(self): + async def main(): + task = asyncio.current_task() + unwinder = RemoteUnwinder(os.getpid()) + for method in (unwinder.get_async_stack_trace, + unwinder.get_all_awaited_by): + with self.subTest(method=method.__name__): + info, = (info for info in method() + if info.thread_id == threading.get_native_id()) + self.assertIsInstance(info, AwaitedInfo) + task_info, = info.awaited_by + self.assertEqual(task_info.task_id, id(task)) + self.assertEqual(task_info.task_name, task.get_name()) + + asyncio.run(main()) + @skip_if_not_supported def test_long_task_name_is_truncated(self): # gh-157788 diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index f9fcea54cacefea..9fc04cf348d144e 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -872,11 +872,11 @@ append_awaited_by_for_thread( return 0; } -int -append_awaited_by( +static int +append_awaited_info( RemoteUnwinderObject *unwinder, unsigned long tid, - uintptr_t head_addr, + PyObject *task_list, PyObject *result) { PyObject *tid_py = PyLong_FromUnsignedLong(tid); @@ -885,24 +885,16 @@ append_awaited_by( return -1; } - PyObject* awaited_by_for_thread = PyList_New(0); - if (awaited_by_for_thread == NULL) { - Py_DECREF(tid_py); - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by thread list"); - return -1; - } - RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); PyObject *result_item = PyStructSequence_New(state->AwaitedInfo_Type); if (result_item == NULL) { Py_DECREF(tid_py); - Py_DECREF(awaited_by_for_thread); set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create AwaitedInfo"); return -1; } - PyStructSequence_SetItem(result_item, 0, tid_py); // steals ref - PyStructSequence_SetItem(result_item, 1, awaited_by_for_thread); // steals ref + PyStructSequence_SetItem(result_item, 0, tid_py); + PyStructSequence_SetItem(result_item, 1, Py_NewRef(task_list)); if (PyList_Append(result, result_item)) { Py_DECREF(result_item); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append awaited_by result item"); @@ -910,8 +902,30 @@ append_awaited_by( } Py_DECREF(result_item); - if (append_awaited_by_for_thread(unwinder, head_addr, awaited_by_for_thread)) - { + return 0; +} + +int +append_awaited_by( + RemoteUnwinderObject *unwinder, + unsigned long tid, + uintptr_t head_addr, + PyObject *result) +{ + PyObject *task_list = PyList_New(0); + if (task_list == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by thread list"); + return -1; + } + + if (append_awaited_info(unwinder, tid, task_list, result) < 0) { + Py_DECREF(task_list); + return -1; + } + + int err = append_awaited_by_for_thread(unwinder, head_addr, task_list); + Py_DECREF(task_list); + if (err < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append awaited_by for thread"); return -1; } @@ -1016,32 +1030,9 @@ process_thread_for_async_stack_trace( return -1; } - // Create AwaitedInfo structure for this thread - PyObject *tid_py = PyLong_FromUnsignedLong(tid); - if (tid_py == NULL) { - Py_DECREF(task_list); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); - return -1; - } - - RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); - PyObject *awaited_info = PyStructSequence_New(state->AwaitedInfo_Type); - if (awaited_info == NULL) { - Py_DECREF(tid_py); - Py_DECREF(task_list); - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create AwaitedInfo"); - return -1; - } - - PyStructSequence_SetItem(awaited_info, 0, tid_py); // steals ref - PyStructSequence_SetItem(awaited_info, 1, task_list); // steals ref - - if (PyList_Append(result, awaited_info)) { - Py_DECREF(awaited_info); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append AwaitedInfo to result"); - return -1; - } - Py_DECREF(awaited_info); + int err = append_awaited_info(unwinder, tid, task_list, result); + Py_DECREF(task_list); + return err; } return 0; From 812bd8772b3ba8de0a0c2a0bd74a287dafc27df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Mon, 16 Nov 2026 13:07:10 +0100 Subject: [PATCH 162/229] gh-149584: _remote_debugging: Simplify page cache state (GH-162094) Use page_cache_count to identify active page cache entries. Remove the redundant validity flag, unused next pointer, and unused size macro. --- Modules/_remote_debugging/_remote_debugging.h | 1 - Python/remote_debug.h | 10 +--------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 6080dc4e3815067..7bbcd8942de61f6 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -121,7 +121,6 @@ typedef struct { #define SIZEOF_GEN_OBJ sizeof(PyGenObject) #define SIZEOF_INTERP_FRAME sizeof(_PyInterpreterFrame) #define SIZEOF_LLIST_NODE sizeof(struct llist_node) -#define SIZEOF_PAGE_CACHE_ENTRY sizeof(page_cache_entry_t) #define SIZEOF_PYOBJECT sizeof(PyObject) #define SIZEOF_SET_OBJ sizeof(PySetObject) #define SIZEOF_TASK_OBJ 4096 diff --git a/Python/remote_debug.h b/Python/remote_debug.h index a68e4c6102e5a7d..0c55fd3a0354e3a 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -155,8 +155,6 @@ get_page_size(void) { typedef struct page_cache_entry { uintptr_t page_addr; // page-aligned base address char *data; - int valid; - struct page_cache_entry *next; } page_cache_entry_t; #define MAX_PAGES 1024 @@ -211,7 +209,6 @@ _Py_RemoteDebug_FreePageCache(proc_handle_t *handle) PyMem_RawFree(handle->pages[i].data); } handle->pages[i].data = NULL; - handle->pages[i].valid = 0; } handle->page_cache_count = 0; } @@ -219,9 +216,6 @@ _Py_RemoteDebug_FreePageCache(proc_handle_t *handle) UNUSED static void _Py_RemoteDebug_ClearCache(proc_handle_t *handle) { - for (int i = 0; i < handle->page_cache_count; i++) { - handle->pages[i].valid = 0; - } handle->page_cache_count = 0; } @@ -244,7 +238,6 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { handle->page_cache_count = 0; for (int i = 0; i < MAX_PAGES; i++) { handle->pages[i].data = NULL; - handle->pages[i].valid = 0; } handle->pid = pid; @@ -1539,7 +1532,7 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, // between profiler samples, so entries are packed at the front. for (int i = 0; i < handle->page_cache_count; i++) { page_cache_entry_t *entry = &handle->pages[i]; - if (entry->valid && entry->page_addr == page_base) { + if (entry->page_addr == page_base) { memcpy(out, entry->data + offset_in_page, size); return 0; } @@ -1566,7 +1559,6 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, } entry->page_addr = page_base; - entry->valid = 1; handle->page_cache_count++; memcpy(out, entry->data + offset_in_page, size); return 0; From bd1e9afa74a38697c7e51df935b3348fa178391e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Mon, 16 Nov 2026 13:28:13 +0100 Subject: [PATCH 163/229] gh-149584: _remote_debugging: Deduplicate frame parsing (GH-162105) Use the shared frame parser for copied stack chunks. Remove unused output parameters while preserving remote reads and fallback behavior. --- Modules/_remote_debugging/_remote_debugging.h | 2 - Modules/_remote_debugging/asyncio.c | 5 +- Modules/_remote_debugging/frames.c | 74 +++++-------------- 3 files changed, 19 insertions(+), 62 deletions(-) diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 7bbcd8942de61f6..532fed28741738b 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -574,7 +574,6 @@ extern int parse_frame_object( RemoteUnwinderObject *unwinder, PyObject** result, uintptr_t address, - uintptr_t* address_of_code_object, uintptr_t* previous_frame ); @@ -583,7 +582,6 @@ extern int parse_frame_from_chunks( PyObject **result, uintptr_t address, uintptr_t *previous_frame, - uintptr_t *stackpointer, StackChunkList *chunks ); diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index 9fc04cf348d144e..787f7921d7d74aa 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -386,8 +386,7 @@ parse_coro_chain( // Parse the previous frame using the gi_iframe from local copy uintptr_t prev_frame; uintptr_t gi_iframe_addr = coro_address + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe; - uintptr_t address_of_code_object = 0; - if (parse_frame_object(unwinder, &name, gi_iframe_addr, &address_of_code_object, &prev_frame) < 0) { + if (parse_frame_object(unwinder, &name, gi_iframe_addr, &prev_frame) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in coro chain"); return -1; } @@ -776,14 +775,12 @@ parse_async_frame_chain( return -1; } PyObject* frame_info = NULL; - uintptr_t address_of_code_object; uintptr_t this_frame = address_of_current_frame; int res = parse_frame_object( unwinder, &frame_info, address_of_current_frame, - &address_of_code_object, &address_of_current_frame ); diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index 19a9b5369148a98..5df309109fcfe35 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -195,11 +195,8 @@ parse_frame_buffer( RemoteUnwinderObject *unwinder, PyObject** result, const char *frame, - uintptr_t* address_of_code_object, uintptr_t* previous_frame ) { - *address_of_code_object = 0; - *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable); int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); @@ -217,8 +214,6 @@ parse_frame_buffer( } #endif - *address_of_code_object = code_object; - CodeObjectContext code_ctx = { .code_addr = code_object, .instruction_pointer = instruction_pointer, @@ -232,7 +227,6 @@ parse_frame_object( RemoteUnwinderObject *unwinder, PyObject** result, uintptr_t address, - uintptr_t* address_of_code_object, uintptr_t* previous_frame ) { char frame[SIZEOF_INTERP_FRAME]; @@ -249,7 +243,7 @@ parse_frame_object( STATS_INC(unwinder, memory_reads); STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); - return parse_frame_buffer(unwinder, result, frame, address_of_code_object, previous_frame); + return parse_frame_buffer(unwinder, result, frame, previous_frame); } int @@ -258,41 +252,16 @@ parse_frame_from_chunks( PyObject **result, uintptr_t address, uintptr_t *previous_frame, - uintptr_t *stackpointer, StackChunkList *chunks ) { - void *frame_ptr = find_frame_in_chunks(chunks, address); - if (!frame_ptr) { + const char *frame = find_frame_in_chunks(chunks, address); + if (!frame) { PyErr_Format(PyExc_RuntimeError, "Frame at address 0x%lx not found in stack chunks", address); set_exception_cause(unwinder, PyExc_RuntimeError, "Frame not found in stack chunks"); return -1; } - char *frame = (char *)frame_ptr; - *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); - *stackpointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.stackpointer); - uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame_ptr, unwinder->debug_offsets.interpreter_frame.executable); - int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); - if (frame_valid != 1) { - return frame_valid; - } - - uintptr_t instruction_pointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.instr_ptr); - - // Get tlbc_index for free threading builds - int32_t tlbc_index = 0; -#ifdef Py_GIL_DISABLED - if (unwinder->debug_offsets.interpreter_frame.tlbc_index != 0) { - tlbc_index = GET_MEMBER(int32_t, frame, unwinder->debug_offsets.interpreter_frame.tlbc_index); - } -#endif - - CodeObjectContext code_ctx = { - .code_addr = code_object, - .instruction_pointer = instruction_pointer, - .tlbc_index = tlbc_index, - }; - return parse_code_object(unwinder, result, &code_ctx); + return parse_frame_buffer(unwinder, result, frame, previous_frame); } /* ============================================================================ @@ -316,7 +285,6 @@ process_frame_chain( int parse_result = 0; PyObject *frame = NULL; uintptr_t next_frame_addr = 0; - uintptr_t stackpointer = 0; last_frame_addr = frame_addr; if (++frame_count > MAX_FRAME_CHAIN_DEPTH) { @@ -328,28 +296,23 @@ process_frame_chain( if (ctx->chunks && ctx->chunks->count > 0) { parse_result = parse_frame_from_chunks( - unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, ctx->chunks); + unwinder, &frame, frame_addr, &next_frame_addr, ctx->chunks); if (parse_result == 0) { goto parsed_frame; } PyErr_Clear(); } - { - uintptr_t address_of_code_object = 0; - if (ctx->prefetch.frame && ctx->prefetch.frame_addr == frame_addr) { - parse_result = parse_frame_buffer( - unwinder, &frame, ctx->prefetch.frame, - &address_of_code_object, &next_frame_addr); - } - else { - parse_result = parse_frame_object( - unwinder, &frame, frame_addr, - &address_of_code_object, &next_frame_addr); - } - if (parse_result < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in chain"); - return -1; - } + if (ctx->prefetch.frame && ctx->prefetch.frame_addr == frame_addr) { + parse_result = parse_frame_buffer( + unwinder, &frame, ctx->prefetch.frame, &next_frame_addr); + } + else { + parse_result = parse_frame_object( + unwinder, &frame, frame_addr, &next_frame_addr); + } + if (parse_result < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in chain"); + return -1; } parsed_frame: @@ -543,17 +506,16 @@ try_full_cache_hit( } PyObject *current_frame = NULL; - uintptr_t code_object_addr = 0; uintptr_t previous_frame = 0; int parse_result; if (ctx->prefetch.frame && ctx->prefetch.frame_addr == ctx->frame_addr) { parse_result = parse_frame_buffer(unwinder, ¤t_frame, ctx->prefetch.frame, - &code_object_addr, &previous_frame); + &previous_frame); } else { parse_result = parse_frame_object(unwinder, ¤t_frame, ctx->frame_addr, - &code_object_addr, &previous_frame); + &previous_frame); } if (parse_result < 0) { return -1; From be3dfe5b1890d43c71d6c58c76b65b0ce33eff57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Mon, 16 Nov 2026 13:38:02 +0100 Subject: [PATCH 164/229] _remote_debugging: Use PyBytesWriter in read_py_bytes() (GH-162110) Use PyBytesWriter to construct remote bytes objects. Remove the separate raw buffer allocation, copy, and cleanup. --- Lib/test/test_external_inspection.py | 11 ++++++++ Modules/_remote_debugging/object_reading.c | 29 ++++++---------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index cb9aae3345cff9c..5aebe1fb6b18e95 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -696,6 +696,17 @@ async def main(): stacks = self._extract_coroutine_funcnames(asyncio.run(main())) self.assertEqual(stacks["consumer"], ["consumer"]) + @skip_if_not_supported + def test_self_trace_with_empty_linetable(self): + def sample(): + return RemoteUnwinder(os.getpid()).get_stack_trace() + + sample.__code__ = sample.__code__.replace(co_linetable=b"") + frame = self._find_frame_in_trace( + sample(), lambda f: f.funcname == sample.__qualname__ + ) + self.assertEqual(frame.location, (-1, -1, -1, -1)) + @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, diff --git a/Modules/_remote_debugging/object_reading.c b/Modules/_remote_debugging/object_reading.c index 6f138f3de3503b6..ed1f07c9143e0fc 100644 --- a/Modules/_remote_debugging/object_reading.c +++ b/Modules/_remote_debugging/object_reading.c @@ -185,9 +185,6 @@ read_py_bytes( uintptr_t address, Py_ssize_t max_len ) { - PyObject *result = NULL; - char *buf = NULL; - // Read the entire PyBytesObject at once char bytes_obj[SIZEOF_BYTES_OBJ]; int res = _Py_RemoteDebug_PagedReadRemoteMemory( @@ -198,7 +195,7 @@ read_py_bytes( ); if (res < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read PyBytesObject"); - goto err; + return NULL; } Py_ssize_t len = GET_MEMBER(Py_ssize_t, bytes_obj, unwinder->debug_offsets.bytes_object.ob_size); @@ -209,36 +206,26 @@ read_py_bytes( return NULL; } - buf = (char *)PyMem_RawMalloc(len+1); - if (buf == NULL) { - PyErr_NoMemory(); + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate buffer for bytes reading"); return NULL; } size_t offset = (size_t)unwinder->debug_offsets.bytes_object.ob_sval; - res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, len, buf); + res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, + len, PyBytesWriter_GetData(writer)); if (res < 0) { + PyBytesWriter_Discard(writer); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read bytes data from remote memory"); - goto err; + return NULL; } - buf[len] = '\0'; - result = PyBytes_FromStringAndSize(buf, len); + PyObject *result = PyBytesWriter_Finish(writer); if (result == NULL) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create PyBytes from remote bytes data"); - goto err; } - - PyMem_RawFree(buf); - assert(result != NULL); return result; - -err: - if (buf != NULL) { - PyMem_RawFree(buf); - } - return NULL; } long From f4c13c80cdb9c350cf474ea5b9cf85245cbf89c7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 16 Nov 2026 17:03:36 +0100 Subject: [PATCH 165/229] gh-162092: Fix test_check_interned() of test_capi (#162099) Fix fresh_string() for Free Threaded build to make sure that the string is not interned. Co-authored-by: Maik Schulze --- Lib/test/test_capi/test_unicode.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 84303bc379cf130..fe4e8e3b5046e62 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1883,7 +1883,9 @@ def test_check_interned(self): def fresh_string(text): # Create a new string object by decoding a bytes string. - return text.encode().decode() + # Add a suffix to make sure that it's not interned, even if + # the string is interned in the PYC file (Free Threading). + return (text + ' suffix').encode().decode() s = fresh_string('hello') self.assertEqual(check_interned(s), SSTATE_NOT_INTERNED) From 5e18bfcf3e27d69731384f7240e063f7eac4ef50 Mon Sep 17 00:00:00 2001 From: Thejesh Date: Mon, 16 Nov 2026 09:00:08 -0800 Subject: [PATCH 166/229] gh-162028: Fix the pure Python JSON decoder error position for invalid escapes (#162029) The C decoder reports the position of the backslash. The pure Python decoder reported the position of the character after it. --- Lib/json/decoder.py | 2 +- Lib/test/test_json/test_fail.py | 18 ++++++++++++++++++ ...6-11-15-02-06-21.gh-issue-162028.jAnDAg.rst | 2 ++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-15-02-06-21.gh-issue-162028.jAnDAg.rst diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index ee0464318458117..31f410632d4f87e 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -112,7 +112,7 @@ def py_scanstring(s, end, strict=True, char = _b[esc] except KeyError: msg = "Invalid \\escape: {0!r}".format(esc) - raise JSONDecodeError(msg, s, end) + raise JSONDecodeError(msg, s, end - 1) end += 1 else: uni = _decode_uXXXX(s, end) diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index 7f99a6331d36251..76ab6f9ced89016 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -201,6 +201,24 @@ def test_unexpected_data(self): '%s: line 1 column %d (char %d)' % (msg, idx + 1, idx)) + def test_invalid_escape_position(self): + # The position points to the backslash of the invalid escape. + test_cases = [ + ('"\\a"', 1), + ('"ab\\x"', 3), + ('["\\\'"]', 2), + ('{"k": "v\\ "}', 8), + ] + for data, idx in test_cases: + with self.subTest(data=data): + with self.assertRaises(self.JSONDecodeError) as cm: + self.loads(data) + err = cm.exception + self.assertStartsWith(err.msg, 'Invalid \\escape') + self.assertEqual(err.pos, idx) + self.assertEqual(err.lineno, 1) + self.assertEqual(err.colno, idx + 1) + def test_extra_data(self): test_cases = [ ('[]]', 'Extra data', 2), diff --git a/Misc/NEWS.d/next/Library/2026-11-15-02-06-21.gh-issue-162028.jAnDAg.rst b/Misc/NEWS.d/next/Library/2026-11-15-02-06-21.gh-issue-162028.jAnDAg.rst new file mode 100644 index 000000000000000..a52a970039d3df0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-15-02-06-21.gh-issue-162028.jAnDAg.rst @@ -0,0 +1,2 @@ +Fix the position reported by the pure Python :mod:`json` decoder for an +invalid escape. It now points to the backslash, like the C decoder. From 910a92babf40b7b34baa22184e5df0868f6c27ee Mon Sep 17 00:00:00 2001 From: Kirti Aggarwal <47375858+kirtiagwl1595@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:08:31 +0530 Subject: [PATCH 167/229] gh-110891: Document `PyType_GetDict` as returning a new reference (GH-162107) --- Doc/data/refcounts.dat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index c19a8a3083a4302..b488237b31b642e 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -2417,6 +2417,9 @@ PyType_GenericNew:PyObject*:type:0: PyType_GenericNew:PyObject*:args:0: PyType_GenericNew:PyObject*:kwds:0: +PyType_GetDict:PyObject*::+1: +PyType_GetDict:PyTypeObject*:type:0: + PyType_GetFlags:unsigned long::: PyType_GetFlags:PyTypeObject*:type:0: From 6903425cd913e4a2b78a312c81d74248322165e9 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 16 Nov 2026 18:38:35 -0800 Subject: [PATCH 168/229] Add a `this` module doc, with an In Memoriam links (#161925) Add a `this` module doc, with an In Memoriam link for Tim Peters (RIP). Co-authored-by: Victor Stinner --- Doc/library/this.rst | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 Doc/library/this.rst diff --git a/Doc/library/this.rst b/Doc/library/this.rst new file mode 100644 index 000000000000000..2893d5584addc4b --- /dev/null +++ b/Doc/library/this.rst @@ -0,0 +1,42 @@ +:orphan: + +:mod:`!this` --- The Zen of Python +================================== + +.. module:: this + :synopsis: The Zen of Python + +.. + This easter egg module is deliberately missing from the table of contents. + +:pep:`20` -- The Zen of Python, by Tim Peters (RIP). + + | Beautiful is better than ugly. + | Explicit is better than implicit. + | Simple is better than complex. + | Complex is better than complicated. + | Flat is better than nested. + | Sparse is better than dense. + | Readability counts. + | Special cases aren't special enough to break the rules. + | Although practicality beats purity. + | Errors should never pass silently. + | Unless explicitly silenced. + | In the face of ambiguity, refuse the temptation to guess. + | There should be one-\- and preferably only one -\-obvious way to do it. + | Although that way may not be obvious at first unless you're Dutch. + | Now is better than never. + | Although never is often better than *right* now. + | If the implementation is hard to explain, it's a bad idea. + | If the implementation is easy to explain, it may be a good idea. + | Namespaces are one honking great idea -\- let's do more of those! + +`In Memoriam: Tim Peters 1952-2026 `__. + + +Usage +----- + +.. code-block:: shell-session + + $ python3 -m this From b90572af8f94568d9373edb7b1f22ee09d49b8a4 Mon Sep 17 00:00:00 2001 From: An Long Date: Tue, 17 Nov 2026 12:00:31 +0900 Subject: [PATCH 169/229] gh-161996: Fix crash in `Interpreter.exec` when the uncaught exception's `__str__` raises (GH-162000) --- Lib/test/test_interpreters/test_api.py | 15 +++++++++++++++ ...2026-11-14-21-04-56.gh-issue-161996.35uL2d.rst | 2 ++ Python/crossinterp.c | 8 ++++++++ 3 files changed, 25 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-11-14-21-04-56.gh-issue-161996.35uL2d.rst diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index a9e296891bb8705..e9a64915a1b4a87 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -947,6 +947,21 @@ def test_failure(self): with self.assertRaises(ExecutionFailed): interp.exec('raise Exception') + def test_failure_with_broken_exception_str(self): + interp = interpreters.create() + for raised, expected in [ + ('ValueError', interpreters.InterpreterError), + ('MemoryError', MemoryError), + ]: + with self.subTest(raised=raised): + with self.assertRaises(expected): + interp.exec(dedent(f""" + class BrokenError(Exception): + def __str__(self): + raise {raised} + raise BrokenError + """)) + @force_not_colorized def test_display_preserved_exception(self): tempdir = self.temp_dir() diff --git a/Misc/NEWS.d/next/Library/2026-11-14-21-04-56.gh-issue-161996.35uL2d.rst b/Misc/NEWS.d/next/Library/2026-11-14-21-04-56.gh-issue-161996.35uL2d.rst new file mode 100644 index 000000000000000..561bb62d6537073 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-14-21-04-56.gh-issue-161996.35uL2d.rst @@ -0,0 +1,2 @@ +Fix a crash in :meth:`concurrent.interpreters.Interpreter.exec` when the +uncaught exception's ``__str__`` raises. diff --git a/Python/crossinterp.c b/Python/crossinterp.c index d189d7bcefe313e..a91b45ba2cbcf45 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -2018,6 +2018,14 @@ _PyXI_ApplyError(_PyXI_error *error, const char *failure) PyThreadState *tstate = PyThreadState_Get(); if (failure != NULL) { + if (error->override != NULL + && error->override->code == _PyXI_ERR_NO_MEMORY) + { + PyErr_NoMemory(); + } + else { + PyErr_SetString(PyExc_InterpreterError, failure); + } xi_error_clear(error); return NULL; } From 4497fe9889105b1abc6d633971e84ee9f78bd3b8 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Mon, 16 Nov 2026 22:43:02 -0700 Subject: [PATCH 170/229] gh-161792: Fix crashes in the callable iterator on free-threading (#161793) --- .../test_free_threading/test_iteration.py | 26 +++++ Lib/test/test_iter.py | 17 +++ ...1-11-04-58-51.gh-issue-161792.42EcBI8x.rst | 4 + Objects/iterobject.c | 102 ++++++++++++------ 4 files changed, 119 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst diff --git a/Lib/test/test_free_threading/test_iteration.py b/Lib/test/test_free_threading/test_iteration.py index 540e8edcc676285..bf42304b41e8b96 100644 --- a/Lib/test/test_free_threading/test_iteration.py +++ b/Lib/test/test_free_threading/test_iteration.py @@ -1,3 +1,4 @@ +import itertools import sys import threading import unittest @@ -170,6 +171,31 @@ def drain(it, barrier, results): self.assertEqual(sys.getrefcount(seq), refcount_before) +class ContendedCallIterExhaustionTest(unittest.TestCase): + """Test exhausting a shared iter(callable, sentinel) iterator. + + The values seen by each thread are unspecified, but exhausting the + iterator from several threads at once must not crash. + """ + + def test_shared_iterator_exhaustion(self): + niters = 2000 if support.check_sanitizer(thread=True) else 20000 + + def make_iter(): + counter = itertools.count().__next__ + return iter(lambda: counter(), 3) + + shared = [make_iter()] + + def worker(): + for _ in range(niters): + it = shared[0] + if next(it, None) is None: + shared[0] = make_iter() + + threading_helper.run_concurrently(worker, nthreads=NUMTHREADS) + + class ContendedRangeIterationTest(ContendedTupleIterationTest): def make_testdata(self, n): return range(n) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index fe8617309da29d0..6ebde4848a5a491 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -532,6 +532,23 @@ def spam(): with self.assertRaises(StopIteration): next(spam.iterator) + def test_iter_function_sentinel_replaced_during_compare(self): + # gh-158031: the sentinel's __eq__ replaces the sentinel, which must + # stay alive until the comparison is finished. + class Sentinel: + def __eq__(self, other): + it.__setstate__(((), StopIteration)) + return NotImplemented + + class Result: + def __eq__(self, other): + self.other_type = type(other) + return NotImplemented + + it = iter(Result, Sentinel()) + result = next(it) + self.assertIs(result.other_type, Sentinel) + # Test exception propagation through function iterator def test_exception_function(self): def spam(state=[0]): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst new file mode 100644 index 000000000000000..a4b685d96bc8967 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-11-04-58-51.gh-issue-161792.42EcBI8x.rst @@ -0,0 +1,4 @@ +Fix crashes in the callable iterator returned by :func:`iter` with a +sentinel, when :meth:`!__setstate__` is called while the iterator is in use, +or when the iterator is shared between threads in the :term:`free-threaded +build`. diff --git a/Objects/iterobject.c b/Objects/iterobject.c index fcef1cc9c301434..683683ceaa6600c 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -238,11 +238,33 @@ PyCallIter_New(PyObject *callable, PyObject *sentinel) return _PyCallIter_NewEx(callable, sentinel, NULL); } +// Sharing a callable iterator between threads, or calling __setstate__() +// on an iterator that is in use, is not supported: the values produced are +// unspecified, but the interpreter must not crash. + +static inline PyObject * +calliter_swap(PyObject **field, PyObject *value) +{ +#ifdef Py_GIL_DISABLED + return _Py_atomic_exchange_ptr(field, value); +#else + PyObject *old = *field; + *field = value; + return old; +#endif +} + +static inline PyObject * +calliter_load(PyObject **field) +{ + return Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(*field)); +} + static void calliter_exhaust(calliterobject *it) { - Py_CLEAR(it->it_callable); - Py_CLEAR(it->it_sentinel); + _PyObject_XDecRefDelayed(calliter_swap(&it->it_callable, NULL)); + _PyObject_XDecRefDelayed(calliter_swap(&it->it_sentinel, NULL)); } static void @@ -270,19 +292,26 @@ static PyObject * calliter_iternext(PyObject *op) { calliterobject *it = (calliterobject*)op; - PyObject *result; + PyObject *callable, *sentinel, *stop_exc, *result; - if (it->it_callable == NULL) { + callable = calliter_load(&it->it_callable); + if (callable == NULL) { return NULL; } + result = _PyObject_CallNoArgs(callable); + Py_DECREF(callable); - result = _PyObject_CallNoArgs(it->it_callable); /* The call can exhaust the iterator re-entrantly. */ - if (result != NULL && it->it_callable != NULL) { - if (it->it_sentinel == NULL) { + if (result != NULL && + FT_ATOMIC_LOAD_PTR_RELAXED(it->it_callable) != NULL) + { + sentinel = calliter_load(&it->it_sentinel); + if (sentinel == NULL) { return result; /* Common case, fast path */ } - int ok = PyObject_RichCompareBool(it->it_sentinel, result, Py_EQ); + /* The comparison can run code that replaces the sentinel. */ + int ok = PyObject_RichCompareBool(sentinel, result, Py_EQ); + Py_DECREF(sentinel); if (ok == 0) { return result; /* Common case, fast path */ } @@ -291,14 +320,19 @@ calliter_iternext(PyObject *op) calliter_exhaust(it); } } - else if (PyErr_ExceptionMatches(it->it_stop_exc)) { - PyErr_Clear(); - calliter_exhaust(it); - } - else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - /* It would be mistaken for the end of the iteration (see PEP 479). */ - _PyErr_FormatFromCause(PyExc_RuntimeError, - "callable raised StopIteration"); + else if (result == NULL) { + stop_exc = calliter_load(&it->it_stop_exc); + if (PyErr_ExceptionMatches(stop_exc)) { + PyErr_Clear(); + calliter_exhaust(it); + } + else if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + /* It would be mistaken for the end of the iteration (see + PEP 479). */ + _PyErr_FormatFromCause(PyExc_RuntimeError, + "callable raised StopIteration"); + } + Py_DECREF(stop_exc); } Py_XDECREF(result); return NULL; @@ -314,22 +348,29 @@ calliter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) * call must be before access of iterator pointers. * see issue #101765 */ - if (it->it_callable == NULL) { - return Py_BuildValue("N(())", iter); + PyObject *callable = calliter_load(&it->it_callable); + PyObject *sentinel = calliter_load(&it->it_sentinel); + PyObject *stop_exc = calliter_load(&it->it_stop_exc); + PyObject *res; + if (callable == NULL) { + res = Py_BuildValue("N(())", iter); } /* Only the sentinel can be passed as an argument of iter(), so other attributes are restored from the state (see calliter_setstate()). */ - if (it->it_sentinel == NULL) { - return Py_BuildValue("N(OO)(()O)", iter, it->it_callable, Py_None, - it->it_stop_exc); + else if (sentinel == NULL) { + res = Py_BuildValue("N(OO)(()O)", iter, callable, Py_None, stop_exc); } - else if (it->it_stop_exc == PyExc_StopIteration) { - return Py_BuildValue("N(OO)", iter, it->it_callable, it->it_sentinel); + else if (stop_exc == PyExc_StopIteration) { + res = Py_BuildValue("N(OO)", iter, callable, sentinel); } else { - return Py_BuildValue("N(OO)((O)O)", iter, it->it_callable, Py_None, - it->it_sentinel, it->it_stop_exc); + res = Py_BuildValue("N(OO)((O)O)", iter, callable, Py_None, + sentinel, stop_exc); } + Py_XDECREF(callable); + Py_XDECREF(sentinel); + Py_DECREF(stop_exc); + return res; } static PyObject * @@ -349,11 +390,12 @@ calliter_setstate(PyObject *op, PyObject *state) if (_PyEval_CheckExceptTypeValid(_PyThreadState_GET(), stop_exc) < 0) { return NULL; } - if (it->it_callable != NULL) { - Py_XSETREF(it->it_sentinel, - PyTuple_GET_SIZE(sentinel) ? - Py_NewRef(PyTuple_GET_ITEM(sentinel, 0)) : NULL); - Py_SETREF(it->it_stop_exc, Py_NewRef(stop_exc)); + if (FT_ATOMIC_LOAD_PTR_RELAXED(it->it_callable) != NULL) { + PyObject *value = PyTuple_GET_SIZE(sentinel) ? + Py_NewRef(PyTuple_GET_ITEM(sentinel, 0)) : NULL; + _PyObject_XDecRefDelayed(calliter_swap(&it->it_sentinel, value)); + _PyObject_XDecRefDelayed(calliter_swap(&it->it_stop_exc, + Py_NewRef(stop_exc))); } Py_RETURN_NONE; From c7fef83a149bf0e8bb427ccadb622ac3fbed8cca Mon Sep 17 00:00:00 2001 From: Romuald Brunet Date: Tue, 17 Nov 2026 19:02:55 +0100 Subject: [PATCH 171/229] Docs: update OpenSSL cipher list format URL (#162155) --- Doc/library/ssl.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 868ef0667597435..80a1a05eb6e5e32 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1965,7 +1965,7 @@ to speed up repeated connections from the same clients. Set the allowed ciphers for sockets created with this context when connecting using TLS 1.2 and earlier. The *ciphers* argument should be a string in the `OpenSSL cipher list format - `_. + `_. To set allowed TLS 1.3 ciphers, use :meth:`SSLContext.set_ciphersuites`. If no cipher can be selected (because compile-time options or other From aedb2c1d3d2bfaf60cef84c549606cc08542b63c Mon Sep 17 00:00:00 2001 From: An Long Date: Wed, 18 Nov 2026 04:50:33 +0900 Subject: [PATCH 172/229] gh-158414: Fix hang when writing incomplete UTF-8 sequence to Windows console (#158435) Co-authored-by: Cody Maloney --- Lib/test/test_io/test_winconsoleio.py | 2 +- .../Windows/2026-09-30-01-12-58.gh-issue-158414.738yJa.rst | 1 + Modules/_io/winconsoleio.c | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2026-09-30-01-12-58.gh-issue-158414.738yJa.rst diff --git a/Lib/test/test_io/test_winconsoleio.py b/Lib/test/test_io/test_winconsoleio.py index 1bae884ed9ae3ea..b2474007da98935 100644 --- a/Lib/test/test_io/test_winconsoleio.py +++ b/Lib/test/test_io/test_winconsoleio.py @@ -160,7 +160,7 @@ def test_write(self): testcases.append(data + b'\xff') # incomplete multibyte sequence with self.subTest(data=data): - self.assertEqual(f.write(data), len(a)) + self.assertEqual(f.write(data), len(a or data)) for data in testcases: with self.subTest(data=data): self.assertEqual(f.write(data), len(data)) diff --git a/Misc/NEWS.d/next/Windows/2026-09-30-01-12-58.gh-issue-158414.738yJa.rst b/Misc/NEWS.d/next/Windows/2026-09-30-01-12-58.gh-issue-158414.738yJa.rst new file mode 100644 index 000000000000000..dba12665deddd93 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-09-30-01-12-58.gh-issue-158414.738yJa.rst @@ -0,0 +1 @@ +Fix a hang when writing an incomplete UTF-8 sequence to the Windows console. diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 3da4f1df1708a78..183506adc678ea2 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -1097,7 +1097,10 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, len = Py_MIN(len, max_wlen * 3); while (1) { /* Fix for github issues gh-110913 and gh-82052. */ - len = _find_last_utf8_boundary(b->buf, len); + DWORD boundary = _find_last_utf8_boundary(b->buf, len); + if (boundary != 0) { + len = boundary; + } wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); if (wlen <= max_wlen) { break; From c885978bb600ac94012776877070381a50e4d4f2 Mon Sep 17 00:00:00 2001 From: Kirti Aggarwal <47375858+kirtiagwl1595@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:41:54 +0530 Subject: [PATCH 173/229] gh-116245: Document `getentropy()` use in `os.urandom()` (#162108) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/os.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7f4b698eac38cd8..9cf4a4efe5c0ee9 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -6466,9 +6466,12 @@ Random numbers random bytes in non-blocking mode (using the :data:`GRND_NONBLOCK` flag) or to poll until the system urandom entropy pool is initialized. - On a Unix-like system, random bytes are read from the ``/dev/urandom`` - device. If the ``/dev/urandom`` device is not available or not readable, the - :exc:`NotImplementedError` exception is raised. + On other Unix-like systems, the C ``getrandom()`` function is used if it is + available, otherwise the C ``getentropy()`` function. + + If none of these functions can be used, random bytes are read from the + ``/dev/urandom`` device. If the ``/dev/urandom`` device is not available or + not readable, the :exc:`NotImplementedError` exception is raised. On Windows, it will use ``BCryptGenRandom()``. From e28063ef9bc5c79ab8576a7293ec97e5bd62b37a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 17 Nov 2026 21:48:31 +0100 Subject: [PATCH 174/229] gh-162144: Skip `@nomemtest` tests when using a thread sanitizer (TSan) (#162154) --- Lib/test/support/__init__.py | 8 +++++++- .../Tests/2026-11-17-15-57-41.gh-issue-162144.pL5k29.rst | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-11-17-15-57-41.gh-issue-162144.pL5k29.rst diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f69734037a8fe57..2cfe4b7f20e1015 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1407,7 +1407,13 @@ def internal(*args, **kwargs): import_module('_testcapi') return test(*args, **kwargs) - return cpython_only(internal) + # gh-162144: set_nomemory() and call_with_nomemory() of _testcapi are not + # thread safe and emit warnings when using a thread sanitizer (TSan). + use_tsan = check_sanitizer(thread=True) + reason ='not working with thread sanitizer (gh-157415)' + skip_if_tsan = unittest.skipIf(use_tsan, reason) + + return cpython_only(skip_if_tsan(internal)) def bigaddrspacetest(f): """Decorator for tests that fill the address space.""" diff --git a/Misc/NEWS.d/next/Tests/2026-11-17-15-57-41.gh-issue-162144.pL5k29.rst b/Misc/NEWS.d/next/Tests/2026-11-17-15-57-41.gh-issue-162144.pL5k29.rst new file mode 100644 index 000000000000000..bbd4a9cf54752e0 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-11-17-15-57-41.gh-issue-162144.pL5k29.rst @@ -0,0 +1,3 @@ +Skip ``@support.nomemtest`` tests when using a thread sanitizer (TSan). +``set_nomemory()`` and ``call_with_nomemory()`` functions of the +:mod:`!_testcapi` extension are not thread safe. Patch by Victor Stinner. From 715d34fc38f84bbccf9feaf937477290e575b060 Mon Sep 17 00:00:00 2001 From: Kevin Modzelewski Date: Tue, 17 Nov 2026 13:03:13 -0800 Subject: [PATCH 175/229] gh-162170: Makefile: have "install" depend on pgo (#95287) Previously, `make install` would ignore PGO. This isn't an issue if one runs `make && make install`, since `make` will build with pgo (if requested) and then `make install` won't rebuild anything. But if one simply does `./configure --enable-optimizations && make install`, the build system would build a non-PGO build and then install that. This commit changes the Makefile so that `install` depends on `all`, which includes PGO when requested. --- Makefile.pre.in | 2 +- .../next/Build/2026-11-17-20-12-03.gh-issue-162170.0SpcRZ.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2026-11-17-20-12-03.gh-issue-162170.0SpcRZ.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index 1d109dac77fa906..d0dcf163e380248 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2485,7 +2485,7 @@ sharedinstall: all # Install the interpreter with $(VERSION) affixed # This goes into $(exec_prefix) .PHONY: altbininstall -altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ +altbininstall: all @FRAMEWORKPYTHONW@ @for i in $(BINDIR) $(LIBDIR); \ do \ if test ! -d $(DESTDIR)$$i; then \ diff --git a/Misc/NEWS.d/next/Build/2026-11-17-20-12-03.gh-issue-162170.0SpcRZ.rst b/Misc/NEWS.d/next/Build/2026-11-17-20-12-03.gh-issue-162170.0SpcRZ.rst new file mode 100644 index 000000000000000..caa4b13ce303395 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-11-17-20-12-03.gh-issue-162170.0SpcRZ.rst @@ -0,0 +1,3 @@ +``make install`` now depends on the ``all`` target, so running it directly +after ``./configure --enable-optimizations`` installs the PGO-optimized +interpreter rather than an unoptimized build. From 3ad9316217ba4e2788abc7d19f469762c6c38309 Mon Sep 17 00:00:00 2001 From: Ferdinando Ametrano Date: Tue, 17 Nov 2026 23:15:06 +0100 Subject: [PATCH 176/229] gh-162034: Quote paths in the Apple testbed's utils.sh (#162161) Add additional protection against spaces and other special characters in iOS Xcode build scripts. --- ...026-11-17-00-39-51.gh-issue-162034.FmiuNi.rst | 2 ++ .../testbed/Python.xcframework/build/utils.sh | 16 ++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-11-17-00-39-51.gh-issue-162034.FmiuNi.rst diff --git a/Misc/NEWS.d/next/Tests/2026-11-17-00-39-51.gh-issue-162034.FmiuNi.rst b/Misc/NEWS.d/next/Tests/2026-11-17-00-39-51.gh-issue-162034.FmiuNi.rst new file mode 100644 index 000000000000000..1764fa2a458e599 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-11-17-00-39-51.gh-issue-162034.FmiuNi.rst @@ -0,0 +1,2 @@ +Fix the Apple testbed's extension module processing for an XCframework or +package path with a space, or a build path with brackets or a backslash. diff --git a/Platforms/Apple/testbed/Python.xcframework/build/utils.sh b/Platforms/Apple/testbed/Python.xcframework/build/utils.sh index 7e2cda78be57690..53801215377fd86 100755 --- a/Platforms/Apple/testbed/Python.xcframework/build/utils.sh +++ b/Platforms/Apple/testbed/Python.xcframework/build/utils.sh @@ -101,9 +101,9 @@ install_dylib () { MODULE_PATH=$(dirname "$FULL_EXT") MODULE_NAME=$(echo $EXT | cut -d "." -f 1) # The location of the extension file, relative to the bundle - RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} + RELATIVE_EXT=${FULL_EXT#"$CODESIGNING_FOLDER_PATH"/} # The path to the extension file, relative to the install base - PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} + PYTHON_EXT=${RELATIVE_EXT/"$INSTALL_BASE"/} # The full dotted name of the extension module, constructed from the file path. FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); # A bundle identifier; not actually used, but required by Xcode framework packaging @@ -152,8 +152,8 @@ install_dylib () { process_dylibs () { PYTHON_XCFRAMEWORK_PATH=$1 LIB_PATH=$2 - find "$CODESIGNING_FOLDER_PATH/$LIB_PATH" -name "*.so" | while read FULL_EXT; do - install_dylib $PYTHON_XCFRAMEWORK_PATH "$LIB_PATH/" "$FULL_EXT" + find "$CODESIGNING_FOLDER_PATH/$LIB_PATH" -name "*.so" | while read -r FULL_EXT; do + install_dylib "$PYTHON_XCFRAMEWORK_PATH" "$LIB_PATH/" "$FULL_EXT" done } @@ -168,13 +168,13 @@ install_python() { PYTHON_XCFRAMEWORK_PATH=$1 shift - install_stdlib $PYTHON_XCFRAMEWORK_PATH + install_stdlib "$PYTHON_XCFRAMEWORK_PATH" PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib" | grep -E "^python3\.\d+[a-z]*$") echo "Install Python $PYTHON_VER standard library extension modules..." - process_dylibs $PYTHON_XCFRAMEWORK_PATH python/lib/$PYTHON_VER/lib-dynload + process_dylibs "$PYTHON_XCFRAMEWORK_PATH" python/lib/$PYTHON_VER/lib-dynload - for package_path in $@; do + for package_path in "$@"; do echo "Installing $package_path extension modules ..." - process_dylibs $PYTHON_XCFRAMEWORK_PATH $package_path + process_dylibs "$PYTHON_XCFRAMEWORK_PATH" "$package_path" done } From 94276402cf3b3a5e1fa96cedb33a4dc498a77ed2 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 18 Nov 2026 02:32:31 +0000 Subject: [PATCH 177/229] Improve PyManager documentation around tag matching (GH-161940) --- Doc/using/windows.rst | 142 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 1 deletion(-) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 2b824fd70e7d34c..a53e3040ab3bdcc 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -169,7 +169,8 @@ It should be separated from the tag by a slash (either ``/`` or ``\``), and may be shortened to any prefix of its full value. Specifying the company is optional when it is ``PythonCore``, and specifying the tag is optional (but not the slash) when you want the latest release from a -specific company. +specific company. For more details about how tags are matched, see +:ref:`pymanager-tag-matching`. .. code:: @@ -998,6 +999,145 @@ Credentials may be embedded in either setting, however, when ``HTTPS_PROXY`` has credentials embedded they will be used in preference to other credentials. Only one set of credentials will be used for both types of proxy. +.. _pymanager-tag-matching: + +Tags and tag matching +--------------------- + +.. versionchanged:: 26.5 + Previously the behavior was very similar to what is described here, but had + many more inconsistencies. We have streamlined it and ensured the behavior is + well-defined, which may result in some small differences from previous + versions. + +Tags identify Python runtimes that may be installed, modified, or launched by +the Python install manager. They are based on :pep:`514`, with matching and +ordering rules specific to the install manager. These rules do not change +PEP 514's specification. + +A tag has three components: Company, Tag, and Platform. Each part is +case-insensitive, but the exact comparison semantics vary. + +.. code-block:: text + + [/][-] + +The Company is optional and defaults to ``PythonCore``. It may contain slashes, +while the rest may not, and so the last forward slash or backslash separates it +from the Tag. When omitting Company, also omit any slash. In general, Company is +matched as a prefix. + +The optional Platform suffix is one of the known platform suffixes, currently +``-32``, ``-64``, or ``-arm64``. Broadly speaking, the Platform suffix behaves +as part of the Tag, but it may be implied from the current configuration or +system in a way that is different from the rest of the tag. It also does not +parse or compare as a number. + +Runtimes are defined as having a primary tag, and offer several more tags for +installation (``install-for``) and launching (``run-for``). Whether the install +tags or the launch tags are considered depends on context, but both sets are +used to filter to matching runtimes, from which the best option is selected +using more complex logic. If a runtime doesn't have specific tags, its primary +tag may be used for installation and uninstallation (but not launch - it will be +unlaunchable without any ``run-for`` tags). + +You can observe this behaviour using ``py list`` with and without the ``--one`` +option (to show the "best" or all candidates) and with and without the +``--online``/``--source`` options (to show install matches or launch matches). + +Tags are not version numbers, though numeric sequences are treated as version +numbers. Hyphens divide the Tag into segments, where each segment is either text +or a version number with a text suffix. The version number part includes those +ending in ``a``, ``b``, or ``rc`` followed by a serial number, but any other +text is considered a separate suffix. Without a serial number, an ``a``, ``b`` +or ``rc`` suffix is also treated as text. + +Broadly, numbers are compared as numerical versions, so ``3.10`` is greater than +``3.1``, ``3.14`` is greater than ``3.14a1``, and ``3`` is equal to both ``3.0`` +and ``3.0.0``. Text is compared case-insensitively, and must match, so ``2024b`` +is not equivalent to ``2024.0`` because ``'b' != ''``. + +.. note:: + + Do not interpret the depth of detail of the following few paragraphs about + constraints as representing their importance. Constraints are a necessary + evil, and we would prefer nobody rely on them instead of specific tags, which + are much simpler to reason about for publishers and end users. Unfortunately, + we are obliged to explain how our implementation handles constraints, and + this is the place for that explanation. + +Constraints can select multiple tags. A matching tag will start with the same +segments as the constraining tag, where text segments are equal and numeric +segments are above, below, or equal, according to the constraint. The first +non-equal segment determines the result: non-equal text results in failure, +while a non-equal version either succeeds or fails based on the constraint. +Remaining segments in the matching tag are ignored. + +When applying a constraint check to a version segment that was shorter than the +constraint (e.g. ``3`` has one field and so is shorter than ``==3.14``), it is +interpreted as representing any possible version. This will result in many +failed constraint checks that will usually be overridden by a more specific one. +For example ``3`` could represent ``3.99`` and so it is not equal to ``3.14``, +but if the install is offering tags ``['3', '3.14']`` then the later tag will +result in the constraint matching. + +Here are some example cases showing the different applications of constraints: + +* ``>=3.13`` matches ``3.13.4`` and ``3.14`` +* ``>3.13`` matches ``3.14``, but not ``3.13.4`` +* ``>=3.13t`` matches ``3.14t``, while ``>=3.13`` does not +* ``>=3.13`` matches ``3.14-abc`` but not ``3.14abc`` +* ``>3.13-abc-123`` matches ``3.14`` (number is higher) and ``3.13-abc-124`` + (number ``124`` is higher) but not ``3.13-def-123`` (text mismatch), + ``3.14def`` (text ``def`` is not in the constraint), + ``3.13-abc-123`` (number is not higher) or + ``3.13-abc`` (not enough segments) +* ``>=3.0`` matches ``3``, but ``>3.0`` does not +* ``<4`` matches ``3``, but ``<3.12`` and ``<=3.12`` do not +* ``==3.0`` does not match ``3``, while ``==3.10`` still matches ``3.10.1`` +* ``!=3.13`` rejects ``3`` (because it could be ``3.13``), but ``!=3.13t`` + accepts ``3`` (different suffix) + +The difference between hyphen-separated segments and adjacent text and numbers +is subtle, but should not be relevant to most users. Publishers will choose and +advertise the tags for their packages, and so should take care to choose between +adjacent text or separated text. For example, the core python\.org releases use +``3.14t`` for free-threaded builds, which should not be confused with original +builds and so ``3.14t`` should not match ``3.14``. However, ``3.14`` should be +allowed to match with ``3.14-64`` (recalling that known platform tags are +treated at text), and so the hyphenated convention is chosen instead. End users +get intuitive behavior because the publisher considered the impact of their +choices. + +Note that constraints are not complementary, that is, the ``>=`` constraint does +not necessarily include every tag rejected by the ``<`` constraint. +For example, ``3.14t`` satisfies neither ``>=3.14`` nor ``<3.14``. + +Publishers should provide tags for each way they want users to select a +runtime, commonly from most specific to least specific. For example, a +free-threaded runtime might offer ``3.14.4t``, ``3.14t``, and ``3t``. Include +a generic tag such as ``3.14`` only if that runtime should be eligible for +generic requests. Exact matches of user requested tags are preferred over +partial matches, so include the tags you expect your users to use. + +The primary tag should be orderable against other runtimes from the same +publisher, but should not change with in-place updates. Also be aware that +operations reading an online feed assume that the feed is already ordered, and +will select the first runtime that matches requested tags. + +After filtering, runtimes are ordered by their primary tag and then the best is +selected. "Best" is very context sensitive, and may be influenced by user +configuration (such as the default tag), active virtual environments, prerelease +marks on a runtime's sortable version, and whether a windowless interpreter is +desired (such as launching through ``pyw.exe`` instead of ``py.exe``). + +Most notably, a request containing the text field ``dev``, such as ``3-dev``, +causes prereleases to be considered on equal footing with stable releases. +Typically stable releases are preferred unless filtering excludes all viable +options. However, this behavior only takes place after filtering, and so a +runtime that does not offer a tag containing ``dev`` will not be selected as a +candidate. + .. _pymanager-troubleshoot: Troubleshooting From f5ec90d79e108e588a942ea5bfe4f479b2ec382d Mon Sep 17 00:00:00 2001 From: Faris Basha TM <72191505+farisbasha@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:33:51 +0530 Subject: [PATCH 178/229] gh-103484: Remove an outdated footnote in the `unittest.mock` examples (#162166) Co-authored-by: Stan Ulbrych --- Doc/library/unittest.mock-examples.rst | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index da57f05edcf78fa..7b736b5aecb6412 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -642,7 +642,7 @@ Mocking a generator method ~~~~~~~~~~~~~~~~~~~~~~~~~~ A Python generator is a function or method that uses the :keyword:`yield` statement -to return a series of values when iterated over [#]_. +to return a series of values when iterated over. A generator method / function is called to return the generator object. It is the generator object that is then iterated over. The protocol method for @@ -671,12 +671,6 @@ To configure the values returned from the iteration (implicit in the call to >>> list(mock_foo.iter()) [1, 2, 3] -.. [#] There are also generator expressions and more `advanced uses - `_ of generators, but we aren't - concerned about them here. A very good introduction to generators and how - powerful they are is: `Generator Tricks for Systems Programmers - `_. - Applying the same patch to every test method ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From b219e551e86bc1a78d09bb246880ed9fb3839678 Mon Sep 17 00:00:00 2001 From: Tan Long Date: Wed, 18 Nov 2026 19:14:53 +0800 Subject: [PATCH 179/229] Correct `DynamicClassAttribute` from function to class in types docs (GH-144968) Also: add sphinx roles to the text --- Doc/library/types.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 9e928e6dd3637b0..7a6e7aa3fae6a57 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -515,14 +515,14 @@ Additional Utility Classes and Functions .. versionchanged:: 3.13 Added support for an optional positional argument. -.. function:: DynamicClassAttribute(fget=None, fset=None, fdel=None, doc=None) +.. class:: DynamicClassAttribute(fget=None, fset=None, fdel=None, doc=None) - Route attribute access on a class to __getattr__. + Route attribute access on a class to :meth:`~object.__getattr__`. This is a descriptor, used to define attributes that act differently when accessed through an instance and through a class. Instance access remains normal, but access to an attribute through a class will be routed to the - class's __getattr__ method; this is done by raising AttributeError. + class's ``__getattr__`` method; this is done by raising :exc:`AttributeError`. This allows one to have properties active on an instance, and have virtual attributes on the class with the same name (see :class:`enum.Enum` for an example). From 0d5e2ed94fc130156957c359cd8176fd33312e75 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 18 Nov 2026 14:51:32 +0300 Subject: [PATCH 180/229] gh-161767: Fix crash in `PyDict_AsFrozenDictAndClear` on `embedded` values (#162139) --- Doc/c-api/dict.rst | 2 ++ Lib/test/test_capi/test_dict.py | 36 +++++++++++++++++++++++++++++++++ Objects/dictobject.c | 17 +++++++++++++--- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index e986c8b6456b749..cdb3d8fba5af80c 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -526,6 +526,8 @@ Dictionary objects as an argument, prefer :c:func:`PyFrozenDict_New` to leave the source intact. + Does not work for dicts with embedded values. + .. impl-detail:: Works with O(1) complexity. diff --git a/Lib/test/test_capi/test_dict.py b/Lib/test/test_capi/test_dict.py index 9a2e79d4da7e715..4028b56a9cc22c0 100644 --- a/Lib/test/test_capi/test_dict.py +++ b/Lib/test/test_capi/test_dict.py @@ -635,11 +635,47 @@ class DictSubtype(dict): ... self.assertIs(type(d), DictSubtype) self.assertEqual(d, DictSubtype({})) + # non str keys: + d = {1: 'a', 2: 'b', 3: 'c', None: 'f'} + f = as_frozendict(d) + self.assertIs(type(f), frozendict) + self.assertEqual(f, frozendict({1: 'a', 2: 'b', 3: 'c', None: 'f'})) + self.assertIs(type(d), dict) + self.assertEqual(d, {}) + + d = DictSubtype({1: 'a', 2: 'b', 3: 'c', None: 'f'}) + f = as_frozendict(d) + self.assertIs(type(f), frozendict) + self.assertEqual(f, frozendict({1: 'a', 2: 'b', 3: 'c', None: 'f'})) + self.assertIs(type(d), DictSubtype) + self.assertEqual(d, DictSubtype({})) + + # wrong inputs: for wrong_input in (frozendict(), [], None): with self.subTest(wrong_input=wrong_input): with self.assertRaises(SystemError): as_frozendict(wrong_input) + # gh-161767: check type with inline values and dict with embedded values + class Regular: ... + + for klass in (Regular, _testcapi.ManagedDictType): + with self.subTest(klass=klass): + instance = klass() # empty `.__dict__` + with self.assertRaisesRegex(ValueError, 'dict with embedded values'): + as_frozendict(instance.__dict__) + self.assertEqual(instance.__dict__, {}) + + instance = klass() # non-empty `.__dict__` + instance.key = 'value' + with self.assertRaisesRegex(ValueError, 'dict with embedded values'): + as_frozendict(instance.__dict__) + + self.assertEqual(instance.__dict__, {'key': 'value'}) + # Both of these actions were needed to cause a crash: + instance.other = 'new' + del instance # must not crash + def test_frozendict_check(self): # Test PyFrozenDict_Check() check = _testcapi.frozendict_check diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 5d3edbc4ed9578a..ed8b53b8a08d495 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -8663,16 +8663,22 @@ frozendict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return d; } -static void +static int transfer_keys_and_values_lock_held(PyObject *res, PyObject *dict) { PyDictObject *new = (PyDictObject *)res; PyDictObject *old = (PyDictObject *)dict; assert(can_modify_dict(old)); + if (old->ma_values && old->ma_values->embedded) { + PyErr_SetString(PyExc_ValueError, + "can't convert dict with embedded values"); + return -1; + } + // Fast path: do nothing on an empty dict: if (old->ma_keys == Py_EMPTY_KEYS) { - return; + return 0; } Py_ssize_t used = old->ma_used; @@ -8690,6 +8696,7 @@ transfer_keys_and_values_lock_held(PyObject *res, PyObject *dict) new->ma_keys = keys; new->ma_values = values; ASSERT_CONSISTENT(new); + return 0; } PyObject * @@ -8705,9 +8712,13 @@ PyDict_AsFrozenDictAndClear(PyObject *dict) return NULL; } + int transfer; Py_BEGIN_CRITICAL_SECTION(dict); - transfer_keys_and_values_lock_held(res, dict); + transfer = transfer_keys_and_values_lock_held(res, dict); Py_END_CRITICAL_SECTION(); + if (transfer < 0) { + return NULL; + } _PyObject_GC_TRACK(res); assert(_PyFrozenDictObject_CAST(res)->ma_hash == -1); From 117ff5af6fc25a04b7b2769c6de86be574644acc Mon Sep 17 00:00:00 2001 From: ivonastojanovic <80911834+ivonastojanovic@users.noreply.github.com> Date: Wed, 18 Nov 2026 12:01:34 +0000 Subject: [PATCH 181/229] gh-159207: Store Tachyon samples in bounded columnar blocks (GH-160313) Store samples in independently decodable columnar blocks so binary replay can process a profile without decompressing the complete sample stream. Use per-stream timestamp and stack deltas, status bitmaps, and compact frame IDs within each block. --- InternalDocs/profiling_binary_format.md | 296 ++-- Lib/profiling/sampling/binary_collector.py | 2 +- Lib/profiling/sampling/sample.py | 8 +- .../test_binary_format.py | 812 +++++++++- .../test_sampling_profiler/test_cli.py | 4 +- Makefile.pre.in | 2 +- ...-10-20-12-49-32.gh-issue-159207.Lccezg.rst | 3 + Modules/_remote_debugging/binary_io.h | 270 +++- Modules/_remote_debugging/binary_io_reader.c | 1394 ++++++++++------- Modules/_remote_debugging/binary_io_writer.c | 1019 +++++++----- Modules/_remote_debugging/clinic/module.c.h | 4 +- Modules/_remote_debugging/module.c | 39 +- 12 files changed, 2724 insertions(+), 1129 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-20-12-49-32.gh-issue-159207.Lccezg.rst diff --git a/InternalDocs/profiling_binary_format.md b/InternalDocs/profiling_binary_format.md index a3d86b1a4ab5693..b467514ca228e3d 100644 --- a/InternalDocs/profiling_binary_format.md +++ b/InternalDocs/profiling_binary_format.md @@ -41,7 +41,7 @@ The file consists of five required sections and one optional extension: | Header | 64 bytes (fixed) +------------------+ Offset 64 | | -| Sample Data | Variable size (optionally compressed) +| Sample Data | Blocks of columns (optionally compressed) | | +------------------+ string_table_offset | String Table | Variable size @@ -60,8 +60,8 @@ so these tables must be built incrementally and written at the end. The header comes first so readers can quickly validate the file and locate the metadata tables. The sample data follows immediately, allowing the writer -to stream samples directly to disk (or through a compression stream) without -buffering the entire dataset in memory. +to write each block of samples as soon as it is full without buffering the +entire dataset in memory. The string and frame tables are placed after sample data because they grow as new unique entries are discovered during profiling. By deferring their @@ -120,86 +120,141 @@ network streams. ## Sample Data Sample data begins at offset 64 and extends to `string_table_offset`. Samples -use delta compression to minimize redundancy when consecutive samples from the -same thread have identical or similar call stacks. +are grouped into blocks of at most 1 MiB of decoded data. Within a block, +samples use delta compression to minimize redundancy when consecutive samples +from the same thread have identical or similar call stacks. Each block can be +decoded on its own once the string and frame tables are loaded, so a reader +only needs to decode one block at a time. + +### Block Format + +A block stores values of the same kind together, in six columns: + +``` ++------------------------+ +| Block header | 21 to 88 bytes ++------------------------+ +| STREAM_TABLE | (thread_id, interpreter_id) pairs ++------------------------+ +| SAMPLE_STREAM_IDS | One stream index per sample ++------------------------+ +| TIMESTAMP_RESIDUALS | One timestamp residual per sample ++------------------------+ +| STATUS_BITMAPS | One bitmap per varying status flag ++------------------------+ +| STACK_INSTRUCTIONS | Stack encodings ++------------------------+ +| FRAME_IDS | Frame table indices ++------------------------+ +``` + +**Block header:** +``` ++---------------------+-----------+----------------------------------------+ +| magic | 4 bytes | "TBLK" | +| sample_count | varint | Number of samples in this block | +| base_timestamp | varint | Timestamp of the first sample | +| frame_id_width | 1 byte | Bytes per frame index (1 to 4) | +| common_status_flags | 1 byte | Status bits set in every sample | +| status_bitmap_flags | 1 byte | Status bits stored as bitmaps | +| columns | varies | One column descriptor per column | ++---------------------+-----------+----------------------------------------+ +``` + +**Column descriptor:** +``` ++---------------------+-----------+----------------------------------------+ +| decoded_size | varint | Size of the decoded column | +| encoding | 1 byte | 0=raw, 1=zstd, 2=constant | +| stored_size | varint | Compressed size (zstd only) | ++---------------------+-----------+----------------------------------------+ +``` + +A raw column is stored as is, and a constant column is a single byte repeated +`decoded_size` times, so neither needs a stored size. The number of streams +and frame indices in a block follows from the decoded sizes of +`STREAM_TABLE` and `FRAME_IDS`. + +A block holds at most 4,096 streams. There is no separate limit on the number +of samples: the stream index and timestamp columns need at least one byte per +sample, so the 1 MiB limit already bounds it. ### Stack Encoding Types -Each sample record begins with thread identification, then an encoding byte: +Each thread's stack is reconstructed from the previous stack of the same +thread in the block. The first sample of a thread in a block starts from an +empty stack. `STACK_INSTRUCTIONS` holds one of these encodings for every +sample not covered by a `REPEAT`: | Code | Name | Description | |------|------|-------------| | 0x00 | REPEAT | RLE: identical stack repeated N times | -| 0x01 | FULL | Complete stack (first sample or no match) | -| 0x02 | SUFFIX | Shares N frames from bottom of previous stack | -| 0x03 | POP_PUSH | Remove M frames from top, add N new frames | +| 0x01 | REPLACE_SUFFIX | Keeps N outer frames of the previous stack, adds M new frames | +| 0x02 | REUSE | Complete stack stored earlier in the block | + +Stacks hold at most 256 frames. The writer keeps the innermost 256 frames of +deeper stacks. -### Record Formats +### Stack Encoding Formats + +All fields are varints. Stacks are written outermost frame first. **REPEAT (0x00) - Run-Length Encoded Identical Stacks:** ``` +-----------------+-----------+----------------------------------------+ -| thread_id | 8 bytes | Thread identifier (uint64, fixed) | -| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | -| encoding | 1 byte | 0x00 (REPEAT) | +| encoding | varint | 0x00 (REPEAT) | | count | varint | Number of samples in this RLE group | -| samples | varies | Interleaved: [delta: varint, status: 1]| -| | | repeated count times | +-----------------+-----------+----------------------------------------+ ``` -The stack is inherited from this thread's previous sample. Each sample in the -group gets its own timestamp delta and status byte, stored as interleaved pairs -(delta1, status1, delta2, status2, ...) rather than separate arrays. +The stack is inherited from this thread's previous sample, for this sample +and the thread's next `count - 1` samples. Samples from other threads can +appear in between: the reader keeps a remaining count for each thread and +reads no encoding for that thread until the count reaches zero. Each sample +in the group keeps its own timestamp and status in the other columns. -**FULL (0x01) - Complete Stack:** +**REPLACE_SUFFIX (0x01) - Shared Outer Frames:** ``` +-----------------+-----------+----------------------------------------+ -| thread_id | 8 bytes | Thread identifier (uint64, fixed) | -| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | -| encoding | 1 byte | 0x01 (FULL) | -| timestamp_delta | varint | Microseconds since thread's last sample| -| status | 1 byte | Thread state flags | -| stack_depth | varint | Number of frames in call stack | -| frame_indices | varint[] | Array of frame table indices | +| encoding | varint | 0x01 (REPLACE_SUFFIX) | +| keep_count | varint | Outer frames kept from previous stack | +| new_count | varint | Inner frames read from FRAME_IDS | +-----------------+-----------+----------------------------------------+ ``` -Used for the first sample from a thread, or when delta encoding would not -provide savings. +Used when a call or return changed the inner part of the stack: the outer +frames are kept, and `new_count` new frames are read from `FRAME_IDS`. A +complete stack, such as the first sample of a thread, uses `keep_count` 0. -**SUFFIX (0x02) - Shared Suffix Match:** +**REUSE (0x02) - Repeated Complete Stack:** ``` +-----------------+-----------+----------------------------------------+ -| thread_id | 8 bytes | Thread identifier (uint64, fixed) | -| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | -| encoding | 1 byte | 0x02 (SUFFIX) | -| timestamp_delta | varint | Microseconds since thread's last sample| -| status | 1 byte | Thread state flags | -| shared_count | varint | Frames shared from bottom of prev stack| -| new_count | varint | New frames at top of stack | -| new_frames | varint[] | Array of new_count frame indices | +| encoding | varint | 0x02 (REUSE) | +| offset | varint | Index of the first frame in FRAME_IDS | +| count | varint | Number of frames in the stack | +-----------------+-----------+----------------------------------------+ ``` -Used when a function call added frames to the top of the stack. The shared -frames from the previous stack are kept, and new frames are prepended. +Used when a complete stack was already written earlier in the same block, +for example by another thread running the same code. The frames are copied +from `FRAME_IDS` instead of being stored again. -**POP_PUSH (0x03) - Pop and Push:** -``` -+-----------------+-----------+----------------------------------------+ -| thread_id | 8 bytes | Thread identifier (uint64, fixed) | -| interpreter_id | 4 bytes | Interpreter ID (uint32, fixed) | -| encoding | 1 byte | 0x03 (POP_PUSH) | -| timestamp_delta | varint | Microseconds since thread's last sample| -| status | 1 byte | Thread state flags | -| pop_count | varint | Frames to remove from top of prev stack| -| push_count | varint | New frames to add at top | -| new_frames | varint[] | Array of push_count frame indices | -+-----------------+-----------+----------------------------------------+ -``` -Used when the code path changed: some frames were popped (function returns) -and new frames were pushed (different function calls). +For example, two threads produce four samples: + +| Sample | Thread | Stack | Encoding | +|--------|--------|----------------|----------------------| +| 0 | A | `[4, 21, 37]` | `REPLACE_SUFFIX 0 3` | +| 1 | B | `[4, 21, 37]` | `REUSE 0 3` | +| 2 | A | `[4, 21, 52]` | `REPLACE_SUFFIX 2 1` | +| 3 | A | `[4, 21, 52]` | `REPEAT 1` | + +`FRAME_IDS` holds only `[4, 21, 37, 52]`. ### Thread and Interpreter Identification +Each `(thread_id, interpreter_id)` pair used in a block is stored once in +`STREAM_TABLE`, as an 8-byte thread ID followed by a 4-byte interpreter ID. +`SAMPLE_STREAM_IDS` then holds one varint per sample: the index of its pair +in the table. Samples are kept in arrival order, so samples from different +threads are interleaved. + Thread IDs are 64-bit values that can be large (memory addresses on some platforms) and vary unpredictably. Using a fixed 8-byte encoding avoids the overhead of varint encoding for large values and simplifies parsing @@ -226,23 +281,30 @@ Multiple flags can be set simultaneously (e.g., a thread can hold the GIL while also running on CPU). Analysis tools use these to filter samples or visualize thread states over time. +Most flags rarely change within a block, so the status bytes are stored as +bit planes. Bits set in every sample are recorded once in +`common_status_flags`. Each bit in `status_bitmap_flags` has a bitmap of +`ceil(sample_count / 8)` bytes in `STATUS_BITMAPS`, in increasing bit order, +with sample `i` in bit `i % 8` of byte `i / 8`. Bits that never occur take no +space. The reader rebuilds each status byte from `common_status_flags` and +the sample's bit in each bitmap. + ### Timestamp Delta Encoding Timestamps use delta encoding rather than absolute values. Absolute timestamps in microseconds require 8 bytes each, but consecutive samples from the same thread are typically separated by the sampling interval -(e.g., 1000 microseconds), so the delta between them is small and fits -in 1-2 varint bytes. The writer tracks the previous timestamp for each -thread separately. The first sample from a thread encodes its delta from -the profiling start time; subsequent samples encode the delta from that -thread's previous sample. This per-thread tracking is necessary because -samples are interleaved across threads in arrival order, not grouped by -thread. - -For REPEAT (RLE) records, timestamp deltas and status bytes are stored as -interleaved pairs (delta, status, delta, status, ...) - one pair per -repeated sample - allowing efficient batching while preserving the exact -timing and state of each sample. +(e.g., 1000 microseconds), so each timestamp is expected to be the thread's +previous timestamp plus the interval, and only the difference from that +expected timestamp is stored. The writer tracks the previous timestamp for +each thread separately. The expected timestamp for a thread's first sample +in a block is the block's `base_timestamp`. This per-thread tracking is +necessary because samples are interleaved across threads in arrival order, +not grouped by thread. + +Each residual is a signed 64-bit zigzag varint. A sample taken exactly on +schedule has a residual of 0 and takes one byte, while timestamps are still +preserved exactly. ### Frame Indexing @@ -251,15 +313,25 @@ rather than inline data. This provides massive space savings because call stacks are highly repetitive: the same function appears in many samples (hot functions), call stacks often share common prefixes (main -> app -> handler -> ...), and recursive functions create repeated frame sequences. -A frame index is typically 1-2 varint bytes. Inline frame data would be -20-200+ bytes (two strings plus a line number). For a profile with 100,000 -samples averaging 30 frames each, this reduces frame data from potentially +A frame index is typically 1-2 bytes. Inline frame data would be 20-200+ +bytes (two strings plus a line number). For a profile with 100,000 samples +averaging 30 frames each, this reduces frame data from potentially gigabytes to tens of megabytes. -Frame indices are written innermost-first (the currently executing frame -has index 0 in the array). This ordering works well with delta compression: -function calls typically add frames at the top (index 0), while shared -frames remain at the bottom. +`FRAME_IDS` stores the indices read by `REPLACE_SUFFIX`, using the smallest +byte width that fits the largest index in the block. The bytes are split into +planes, low bytes first, so similar bytes sit together for compression: + +``` +indices: 0x1234 0x5678 0x9abc +low-byte plane: 0x34 0x78 0xbc +high-byte plane: 0x12 0x56 0x9a +``` + +Frame indices are written outermost-first (the outermost frame has index 0 +in the array). This ordering works well with delta compression: function +calls typically change frames at the end of the array, while shared frames +remain at the start. ## String Table @@ -453,29 +525,34 @@ bytes. Zigzag encoding interleaves positive and negative values: ``` This ensures small-magnitude values (whether positive or negative) encode -in few bytes. +in few bytes. Frame table deltas use 32-bit zigzag values and timestamp +residuals use 64-bit zigzag values. ## Compression -When compression is enabled, the sample data region contains a zstd stream. +The header's compression type controls whether zstd may be used. Each column +of each block is encoded separately, using whichever representation is +smallest: + +- **constant**, when every byte of the column is the same, such as the stream + IDs of a single-threaded block or residuals of on-time samples; +- **zstd**, when compression is enabled and the compressed column plus its + stored-size varint is smaller than the raw column; +- **raw** otherwise. + The string table, frame table, and footer remain uncompressed so readers can access metadata without decompressing the entire file. A tool that only needs to report "this file contains 50,000 samples of 3 threads" can read the header -and footer without touching the compressed sample data. This also simplifies -the format: the header's offset fields point directly to the tables rather -than to positions within a decompressed stream. +and footer without touching the compressed sample data. Zstd provides an excellent balance of compression ratio and speed. Profiling data compresses very well (often 5-10x) due to repetitive patterns: the same -small set of frame indices appears repeatedly, and delta-encoded timestamps -cluster around the sampling interval. Zstd's streaming API allows compression -without buffering the entire dataset. The writer feeds sample data through -the compressor incrementally, flushing compressed chunks to disk as they -become available. +small set of frame indices appears repeatedly, and timestamp residuals +cluster around zero. -Level 5 compression is used as a default. Lower levels (1-3) are faster but -compress less; higher levels (6+) compress more but slow down writing. Level -5 provides good compression with minimal impact on profiling overhead. +Level 3 compression is used as a default. It gives good compression without +slowing down each block flush. Compressing columns separately keeps blocks +independent, at the cost of restarting compression history at every block. ## Reading and Writing @@ -486,9 +563,9 @@ compress less; higher levels (6+) compress more but slow down writing. Level 3. For each sample: - Intern any new strings, assigning sequential indices - Intern any new frames, assigning sequential indices - - Encode the sample record and write to the buffer - - Flush the buffer through compression (if enabled) when full -4. Flush remaining buffered data and finalize compression + - Append the sample to the current block's columns + - Flush the block when it reaches its size or stream limit +4. Flush the last block 5. Write the string table (length-prefixed strings in index order) 6. Write the frame table (varint-encoded entries in index order) 7. Write measured profile statistics, when available @@ -497,7 +574,13 @@ compress less; higher levels (6+) compress more but slow down writing. Level The writer maintains two dictionaries: one mapping strings to indices, one mapping (filename_idx, funcname_idx, lineno) tuples to frame indices. These -enable O(1) lookup during interning. +enable O(1) lookup during interning. To find `REUSE` candidates, it also keeps +a small per-block cache of complete stacks, confirmed by a full comparison. + +When a block is flushed, the status and frame ID columns are transposed into +bit and byte planes, the `REPEAT` counts are inserted into the instruction +column, and each column is encoded as described in +[Compression](#compression). ### Reading @@ -510,13 +593,32 @@ enable O(1) lookup during interning. 6. Parse the string table, populating the array 7. Allocate the frame array 8. Parse the frame table, populating the array -9. If compressed, decompress the sample data region -10. Iterate through samples, resolving indices to strings/frames - (byte-swapping thread_id and interpreter_id if needed) +9. For each block, decode its columns into a reusable buffer and replay its + samples in order, resolving indices to strings/frames (byte-swapping the + stream table if needed) The reader builds lookup arrays rather than dictionaries since it only needs index-to-value mapping, not value-to-index. +Consecutive samples of one stream with the same stack and status are passed +to the collector in a single call, with up to 8,192 timestamps. + +The block header and columns are validated before any sample of the block is +replayed. Stack instructions are checked as they are replayed, so a block +with a bad instruction may already have passed earlier samples to the +collector before the error is raised. The reader rejects a block when: + +- a size, encoding, or stream or frame reference is out of range, or a + column has trailing data; +- the stream table lists a thread twice or has a stream with no samples; +- a stream's first instruction in the block is `REPEAT`, or a `REPEAT` + count is zero or runs past the block; +- a stack would be deeper than 256 frames, or `REUSE` refers to frame IDs + that have not been read yet; +- not every frame ID is used; +- status bitmap padding bits are not zero; +- it uses zstd in an uncompressed file. + ## Platform Considerations ### Byte Ordering and Cross-Platform Portability @@ -540,7 +642,7 @@ zeros instead of the actual value. The reader tracks whether byte-swapping is needed via a `needs_swap` flag set during header parsing. All fixed-width fields in the header, footer, and -sample data are conditionally byte-swapped using Python's internal byte-swap +stream tables are conditionally byte-swapped using Python's internal byte-swap functions (`_Py_bswap32`, `_Py_bswap64` from `pycore_bitutils.h`). Variable-length integers (varints) are byte-order independent since they @@ -568,10 +670,8 @@ On Windows, the implementation falls back to standard file I/O with full file buffering. Profiling data files are typically small enough (tens to hundreds of megabytes) that this is acceptable. -The writer uses a 512 KB buffer to batch small writes. Each sample record -is typically tens of bytes; writing these individually would incur excessive -syscall overhead. The buffer accumulates data until full, then flushes in -one write() call (or feeds through the compression stream). +The writer buffers samples in the current block and writes each completed +block with a single call, so individual samples never cause a write. ## Future Considerations diff --git a/Lib/profiling/sampling/binary_collector.py b/Lib/profiling/sampling/binary_collector.py index 3d6d988077cfa2a..21f58daf76f3dd4 100644 --- a/Lib/profiling/sampling/binary_collector.py +++ b/Lib/profiling/sampling/binary_collector.py @@ -146,7 +146,7 @@ def get_stats(self): """Get encoding statistics. Returns: - Dict with encoding statistics including repeat/full/suffix/pop-push + Dict with encoding statistics including repeat/full/suffix/reuse record counts, frames written/saved, and compression ratio. """ return self._writer.get_stats() diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 1dff6529ade0408..e50af2528e93bf0 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -411,22 +411,22 @@ def _print_binary_stats(self, collector): repeat_samples = stats.get('repeat_samples', 0) full_records = stats.get('full_records', 0) suffix_records = stats.get('suffix_records', 0) - pop_push_records = stats.get('pop_push_records', 0) + reuse_records = stats.get('reuse_records', 0) total_records = stats.get('total_records', 0) if total_records > 0: repeat_pct = repeat_records / total_records * 100 full_pct = full_records / total_records * 100 suffix_pct = suffix_records / total_records * 100 - pop_push_pct = pop_push_records / total_records * 100 + reuse_pct = reuse_records / total_records * 100 else: - repeat_pct = full_pct = suffix_pct = pop_push_pct = 0 + repeat_pct = full_pct = suffix_pct = reuse_pct = 0 print(f" Records: {total_records:,}") print(f" RLE repeat: {repeat_records:,} ({ANSIColors.GREEN}{repeat_pct:.1f}%{ANSIColors.RESET}) [{repeat_samples:,} samples]") print(f" Full stack: {full_records:,} ({full_pct:.1f}%)") print(f" Suffix match: {suffix_records:,} ({suffix_pct:.1f}%)") - print(f" Pop-push: {pop_push_records:,} ({pop_push_pct:.1f}%)") + print(f" Stack reuse: {reuse_records:,} ({reuse_pct:.1f}%)") frames_written = stats.get('total_frames_written', 0) frames_saved = stats.get('frames_saved', 0) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index 66a9c68a61c47dc..d73af62d3b02351 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -10,7 +10,7 @@ from unittest import mock from collections import defaultdict -from test.support import captured_stderr +from test.support import captured_stderr, import_helper try: import _remote_debugging @@ -127,6 +127,28 @@ def export(self, filename): pass +def encode_varint(value): + """Encode an unsigned varint.""" + out = bytearray() + while value >= 0x80: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + return bytes(out) + + +def decode_varint(data, offset): + """Decode an unsigned varint, returning (value, next offset).""" + value = shift = 0 + while True: + byte = data[offset] + offset += 1 + value |= (byte & 0x7F) << shift + shift += 7 + if byte < 0x80: + return value, offset + + def samples_to_by_thread(samples): """Convert input samples to by-thread format for comparison.""" by_thread = defaultdict(list) @@ -147,6 +169,20 @@ def samples_to_by_thread(samples): class BinaryFormatTestBase(unittest.TestCase): """Base class with common setup/teardown for binary format tests.""" + HDR_OFF_STR_TABLE = 40 + FILE_HEADER_PLACEHOLDER_SIZE = 64 + + # Sample block layout (see binary_io.h). + BLOCK_MAGIC = b"TBLK" + COLUMN_COUNT = 6 + COLUMN_STREAM_IDS = 1 + COLUMN_TIMESTAMP_RESIDUALS = 2 + COLUMN_STATUS_BITMAPS = 3 + COLUMN_STACK_INSTRUCTIONS = 4 + COLUMN_FRAME_IDS = 5 + COLUMN_ZSTD = 1 + COLUMN_CONSTANT = 2 + def setUp(self): self.temp_files = [] @@ -155,6 +191,13 @@ def tearDown(self): if os.path.exists(f): os.unlink(f) + def temp_filename(self): + """Return a new temporary file name that is removed on cleanup.""" + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + return filename + def create_binary_file(self, samples, interval=1000, compression="none", mode=None, capture_config=None): """Create a test binary file and track it for cleanup.""" @@ -166,9 +209,7 @@ def create_binary_file(self, samples, interval=1000, compression="none", def write_binary_file(self, samples, interval=1000, compression="none", mode=None, capture_config=None): """Like create_binary_file but also returns the writer collector.""" - with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: - filename = f.name - self.temp_files.append(filename) + filename = self.temp_filename() collector = BinaryCollector( filename, interval, compression=compression, mode=mode, @@ -179,6 +220,45 @@ def write_binary_file(self, samples, interval=1000, compression="none", collector.export(None) return filename, collector + def read_block_columns(self, filename): + """Return the header flags and (decoded size, encoding) per column. + + Each block is a dict with frame_width, common_status_flags, + status_bitmap_flags and columns. + """ + with open(filename, "rb") as raw: + data = raw.read() + sample_end = struct.unpack_from("=Q", data, self.HDR_OFF_STR_TABLE)[0] + offset = self.FILE_HEADER_PLACEHOLDER_SIZE + blocks = [] + while offset < sample_end: + magic_end = offset + len(self.BLOCK_MAGIC) + self.assertEqual(data[offset:magic_end], self.BLOCK_MAGIC) + _, offset = decode_varint(data, magic_end) # sample count + _, offset = decode_varint(data, offset) # base timestamp + block = {"frame_width": data[offset], + "common_status_flags": data[offset + 1], + "status_bitmap_flags": data[offset + 2]} + offset += 3 + columns = [] + stored_total = 0 + for _ in range(self.COLUMN_COUNT): + decoded_size, offset = decode_varint(data, offset) + encoding = data[offset] + offset += 1 + if encoding == self.COLUMN_ZSTD: + stored_size, offset = decode_varint(data, offset) + elif encoding == self.COLUMN_CONSTANT: + stored_size = 1 + else: + stored_size = decoded_size + columns.append((decoded_size, encoding)) + stored_total += stored_size + offset += stored_total + block["columns"] = columns + blocks.append(block) + return blocks + def roundtrip(self, samples, interval=1000, compression="none"): """Write samples to binary and read back.""" filename = self.create_binary_file(samples, interval, compression) @@ -393,6 +473,29 @@ def test_binary_replay_preserves_main_thread_for_gecko(self): self.assertTrue(threads[1]["isMainThread"]) self.assertFalse(threads[2]["isMainThread"]) + def test_binary_replay_starts_gecko_at_earliest_sample(self): + """Binary replay uses the earliest sample as Gecko's time origin.""" + filename = self.temp_filename() + collector = BinaryCollector(filename, 1000, compression="none") + for thread_id, name, timestamp in [(1, "first", 1000), + (2, "second", 2000), + (2, "changed", 3000)]: + frame = make_frame(f"{name}.py", 10, name) + collector.collect( + [make_interpreter(0, [make_thread(thread_id, [frame])])], + timestamp_us=timestamp, + ) + collector.export(None) + + gecko = GeckoCollector(1000) + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(gecko), 3) + + profile = gecko._build_profile() + threads = {thread["tid"]: thread for thread in profile["threads"]} + self.assertEqual(threads[1]["samples"]["time"], [0.0]) + self.assertEqual(threads[2]["samples"]["time"], [1.0, 2.0]) + def test_multiple_threads_per_sample(self): """Multiple threads in one sample roundtrip exactly.""" threads = [ @@ -897,6 +1000,277 @@ def test_rejects_non_pathlike(self): with self.assertRaises(TypeError): _remote_debugging.BinaryReader(123) + def test_writer_rejects_invalid_compression(self): + """BinaryWriter rejects an unknown compression type.""" + with self.assertRaisesRegex(ValueError, "compression"): + _remote_debugging.BinaryWriter( + self.temp_filename(), 1000, 0, compression=2 + ) + + def test_writer_rejects_status_outside_byte_range(self): + """A status above 255 raises OverflowError and drops that sample.""" + filename = self.temp_filename() + writer = _remote_debugging.BinaryWriter(filename, 1000, 0) + valid = [make_interpreter(0, [make_thread(1, [], status=1)])] + invalid = [make_interpreter(0, [make_thread(1, [], status=256)])] + writer.write_sample(valid, 1000) + with self.assertRaisesRegex(OverflowError, "thread status"): + writer.write_sample(invalid, 2000) + self.assertTrue(writer.limit_reached) + writer.write_sample(valid, 3000) + writer.finalize() + self.assertEqual(writer.total_samples, 2) + + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(RawCollector()), 2) + + def test_writer_reports_stack_reuse(self): + """A stack seen earlier in the block is stored as STACK_REUSE.""" + filename = self.temp_filename() + frames = [make_frame("shared.py", line, "shared") for line in range(3)] + sample = [make_interpreter(0, [ + make_thread(1, frames), + make_thread(2, frames), + ])] + writer = _remote_debugging.BinaryWriter(filename, 1000, 0) + writer.write_sample(sample, 1000) + stats = writer.get_stats() + self.assertEqual(stats["reuse_records"], 1) + self.assertEqual(stats["total_records"], 2) + self.assertEqual(stats["total_samples"], 2) + writer.close() + + def test_writer_buffered_bytes_match_flushed_bytes(self): + """bytes_written before finalize() matches the flushed size.""" + filename = self.temp_filename() + frames = [make_frame("a.py", line, "f") for line in range(300)] + writer = _remote_debugging.BinaryWriter(filename, 1000, 0) + for i in range(100): + sample = [make_interpreter(0, [ + make_thread(1, frames[i:i + 5], status=i % 3), + make_thread(2, frames[:3]), + ])] + writer.write_sample(sample, 1000 * (i + 1)) + buffered = writer.get_stats()["bytes_written"] + writer.finalize() + + [block] = self.read_block_columns(filename) + self.assertEqual(buffered, + sum(size for size, _ in block["columns"])) + + def test_samples_span_multiple_blocks(self): + """Replay preserves samples and progress across block boundaries.""" + # Stream 4097 starts the second block. Thread 1 then returns. + frame = make_frame("many_threads.py", 1, "work") + samples = [ + [make_interpreter(0, [make_thread(i + 1, [frame])])] + for i in range(4097) + ] + samples.append(samples[0]) + filename = self.create_binary_file(samples) + self.assertEqual(len(self.read_block_columns(filename)), 2) + + collector = RawCollector() + progress = mock.Mock() + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(collector, progress), + len(samples)) + self.assert_samples_equal(samples, collector) + total = len(samples) + self.assertEqual(progress.call_args_list, + [mock.call(current, total) + for current in (0, 1000, 2000, 3000, 4000, total)]) + + def test_block_size_limit_starts_new_block(self): + """A thread whose samples exceed 1 MiB is split across blocks.""" + # Each sample writes a new 256-frame stack, about 1 KiB while + # buffered, so 1,100 samples need more than one block. + frames = [make_frame("deep.py", line, "f") for line in range(1356)] + samples = [ + [make_interpreter(0, [make_thread(1, frames[i:i + 256])])] + for i in range(1100) + ] + filename = self.create_binary_file(samples) + blocks = self.read_block_columns(filename) + self.assertGreaterEqual(len(blocks), 2) + for block in blocks: + self.assertLessEqual(sum(size for size, _ in block["columns"]), + 1024 * 1024) + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(RawCollector()), + len(samples)) + + def test_writer_uses_constant_columns_and_narrow_frame_ids(self): + """Repeated bytes are stored once and frame IDs use the least width.""" + frame = make_frame("steady.py", 1, "steady") + sample = [make_interpreter(0, [make_thread(1, [frame])])] + filename = self.temp_filename() + writer = _remote_debugging.BinaryWriter(filename, 1000, 0) + for i in range(100): + writer.write_sample(sample, 1000 * (i + 1)) + writer.finalize() + # One stream and on-time samples: every stream ID and residual is 0 + [block] = self.read_block_columns(filename) + self.assertEqual(block["frame_width"], 1) + self.assertEqual(block["columns"][self.COLUMN_STREAM_IDS], + (100, self.COLUMN_CONSTANT)) + self.assertEqual(block["columns"][self.COLUMN_TIMESTAMP_RESIDUALS], + (100, self.COLUMN_CONSTANT)) + + # 300 distinct frames need two bytes per frame ID + frames = [make_frame("wide.py", line, "f") for line in range(300)] + samples = [ + [make_interpreter(0, [make_thread(1, frames[:150])])], + [make_interpreter(0, [make_thread(1, frames[150:])])], + ] + filename = self.create_binary_file(samples) + [block] = self.read_block_columns(filename) + self.assertEqual(block["frame_width"], 2) + self.assertEqual(block["columns"][self.COLUMN_FRAME_IDS][0], 300 * 2) + + def test_writer_stores_common_status_bits_once(self): + """Status bits set in every sample go in the header, not a bitmap.""" + frame = make_frame("status.py", 1, "f") + filename = self.temp_filename() + writer = _remote_debugging.BinaryWriter(filename, 1000, 0) + for i in range(20): + # HAS_GIL in every sample; ON_CPU in every other one + status = THREAD_STATUS_HAS_GIL | (THREAD_STATUS_ON_CPU * (i % 2)) + sample = [make_interpreter(0, [make_thread(1, [frame], status)])] + writer.write_sample(sample, 1000 * (i + 1)) + writer.finalize() + [block] = self.read_block_columns(filename) + self.assertEqual(block["common_status_flags"], THREAD_STATUS_HAS_GIL) + self.assertEqual(block["status_bitmap_flags"], THREAD_STATUS_ON_CPU) + # One bitmap of ceil(20 / 8) bytes, for ON_CPU only + self.assertEqual(block["columns"][self.COLUMN_STATUS_BITMAPS][0], 3) + + @unittest.skipUnless(ZSTD_AVAILABLE, "zstd compression not available") + def test_writer_compresses_repetitive_columns(self): + """Compressed files store repetitive columns with zstd.""" + root = [make_frame("root.py", line, "root") for line in range(20)] + leaves = [make_frame("leaf.py", 1, "even"), + make_frame("leaf.py", 2, "odd")] + samples = [ + [make_interpreter(0, [make_thread(1, [leaves[i % 2]] + root)])] + for i in range(2000) + ] + filename = self.create_binary_file(samples, compression="zstd") + [block] = self.read_block_columns(filename) + columns = block["columns"] + self.assertEqual(columns[self.COLUMN_STACK_INSTRUCTIONS][1], + self.COLUMN_ZSTD) + self.assertEqual(columns[self.COLUMN_FRAME_IDS][1], self.COLUMN_ZSTD) + + def test_replay_byte_swapped_file(self): + """A file written with the other byte order replays the same.""" + samples = [[make_interpreter(0, [ + make_thread(1, [make_frame("a.py", 1, "f")]), + make_thread(2, [make_frame("b.py", 2, "g")]), + ])] for _ in range(3)] + filename = self.temp_filename() + writer = _remote_debugging.BinaryWriter(filename, 1000, 0) + for i, sample in enumerate(samples): + writer.write_sample(sample, 1000 * (i + 1)) + writer.finalize() + + data = bytearray(pathlib.Path(filename).read_bytes()) + # Header fields: magic, version, start time, interval, sample + # count, thread count, table offsets, compression and config + fields = [(0, 4), (4, 4), (12, 8), (20, 8), (28, 8), (36, 4), + (40, 8), (48, 8), (56, 4), (60, 4)] + # Footer string and frame counts + fields += [(len(data) - 32, 4), (len(data) - 28, 4)] + # The block's stream table entries follow its header + offset = self.FILE_HEADER_PLACEHOLDER_SIZE + len(self.BLOCK_MAGIC) + _, offset = decode_varint(data, offset) # sample count + _, offset = decode_varint(data, offset) # base timestamp + offset += 3 + for _ in range(self.COLUMN_COUNT): + _, offset = decode_varint(data, offset) + offset += 1 # raw encoding, no stored size + for entry in range(2): + fields += [(offset + entry * 12, 8), (offset + entry * 12 + 8, 4)] + for start, size in fields: + data[start:start + size] = data[start:start + size][::-1] + pathlib.Path(filename).write_bytes(data) + + collector = RawCollector() + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(collector), 6) + self.assert_samples_equal(samples, collector) + + def test_reader_replays_twice(self): + """Replaying twice gives the same samples and statistics.""" + class RetainingCollector(RawCollector): + def __init__(self): + super().__init__() + self.frame_lists = [] + + def collect(self, stack_frames, timestamps_us): + self.frame_lists.extend( + thread.frame_info + for interp in stack_frames + for thread in interp.threads + ) + super().collect(stack_frames, timestamps_us) + + samples = [ + [make_interpreter(0, [ + make_thread(1, [make_frame("a.py", i % 3, "f")]), + make_thread(2, [make_frame("b.py", 1, "g")]), + ])] + for i in range(10) + ] + filename = self.create_binary_file(samples) + with BinaryReader(filename) as reader: + first = RetainingCollector() + second = RetainingCollector() + self.assertEqual(reader.replay_samples(first), 20) + self.assertEqual(reader.replay_samples(second), 20) + self.assertEqual(reader.get_stats()["total_samples"], 20) + self.assertEqual(first.by_thread, second.by_thread) + for first_frames, second_frames in zip( + first.frame_lists, second.frame_lists, strict=True + ): + self.assertIsNot(first_frames, second_frames) + self.assertEqual(len(first_frames), len(second_frames)) + for first_frame, second_frame in zip( + first_frames, second_frames, strict=True + ): + self.assertIs(first_frame, second_frame) + + def test_writer_repeat_run_spans_interleaved_streams(self): + """A thread's repeat run continues across other threads' samples.""" + filename = self.temp_filename() + idle = [make_frame("idle.py", 1, "wait")] + busy = [[make_frame("busy.py", 1, "even")], + [make_frame("busy.py", 2, "odd")]] + writer = _remote_debugging.BinaryWriter(filename, 1000, 0) + for i in range(10): + sample = [make_interpreter(0, [ + make_thread(1, idle), + make_thread(2, busy[i % 2]), + ])] + writer.write_sample(sample, 1000 * (i + 1)) + stats = writer.get_stats() + self.assertEqual(stats["repeat_records"], 1) + self.assertEqual(stats["repeat_samples"], 9) + writer.finalize() + + collector = RawCollector() + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(collector), 20) + self.assertEqual(reader.get_stats()["repeat_samples"], 9) + idle_frames = [frame_to_dict(frame) for frame in idle] + self.assertEqual( + [sample["frames"] for sample in collector.by_thread[(0, 1)]], + [idle_frames] * 10) + self.assertEqual( + [sample["frames"][0]["funcname"] + for sample in collector.by_thread[(0, 2)]], + ["even", "odd"] * 5) + def test_invalid_path_error_preserves_pathlib(self): """Missing path: OSError carries the original path object, not a string.""" missing = pathlib.Path("/i/do/not/exist") @@ -907,13 +1281,9 @@ def test_invalid_path_error_preserves_pathlib(self): def test_writer_handles_empty_stack_first_sample(self): """BinaryWriter.write_sample tolerates an empty stack on a fresh thread. - Regression test for the C-level RLE bug in process_thread_sample: a - freshly-created ThreadEntry has prev_stack_depth == 0, so an empty - curr_stack compares as STACK_REPEAT against the zero-initialized - previous stack. Before the fix, this fell through the - `&& !is_new_thread` guard into write_sample_with_encoding, which had - no handler for STACK_REPEAT and raised - RuntimeError("Invalid stack encoding type"). Goes through + Regression test for gh-149342: an empty first stack equals the empty + stack a new thread starts from, but it must still be written as a + stack (STACK_REPLACE_SUFFIX 0 0) rather than a repeat. Goes through BinaryWriter.write_sample directly so the test cannot be masked by any Python-level filtering. """ @@ -931,8 +1301,7 @@ def test_writer_handles_empty_stack_first_sample(self): # scenario that exposes the bug. writer.write_sample(empty_sample, 1000) writer.write_sample(empty_sample, 2000) - # Mix in a real sample to exercise the transition out of the - # empty-stack RLE buffer. + # Mix in a real sample after the repeated empty stack. real_sample = [ make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])]) ] @@ -942,8 +1311,8 @@ def test_writer_handles_empty_stack_first_sample(self): reader_collector = RawCollector() with BinaryReader(filename) as reader: count = reader.replay_samples(reader_collector) - # Empty-stack samples are recorded as STACK_REPEAT records with - # depth-0 stacks; the file must replay all three samples. + # The empty stack is written once and then repeated; the file must + # replay all three samples. self.assertEqual(count, 3) def test_writer_handles_mixed_empty_and_real_first_sample(self): @@ -977,11 +1346,9 @@ def test_writer_handles_mixed_empty_and_real_first_sample(self): def test_writer_total_samples_after_finalize_matches_reader(self): """BinaryWriter.total_samples after finalize() matches the reader's count.""" - # Five IDENTICAL samples force every sample beyond the first into the - # per-thread RLE buffer. Regression for the cached_total_samples - # ordering bug: capturing the cache BEFORE binary_writer_finalize() - # missed the buffered samples that flush_pending_rle() counts. Keep - # the samples identical to preserve coverage. See gh-149342. + # Five IDENTICAL samples stay in the open block, as one repeat, until + # finalize() flushes it; total_samples must still count all of them. + # Keep the samples identical to preserve coverage. See gh-149342. samples = [ [make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])])] ] * 5 @@ -992,8 +1359,8 @@ def test_writer_total_samples_after_finalize_matches_reader(self): self.assertEqual(writer_collector.total_samples, len(samples)) self.assertEqual(writer_collector.total_samples, replayed) - def test_rle_buffer_flushes_before_finalize(self): - """RLE buffer flushes before the writer is destroyed during finalize().""" + def test_open_block_flushes_on_finalize(self): + """finalize() writes the open block before the writer is destroyed.""" with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: filename = f.name self.temp_files.append(filename) @@ -1194,11 +1561,10 @@ def test_finalize_failure_breaks_writer(self): class TestBinaryFormatValidation(BinaryFormatTestBase): """Tests for malformed binary files.""" + HDR_OFF_VERSION = 4 HDR_OFF_SAMPLES = 28 HDR_OFF_THREADS = 36 - HDR_OFF_STR_TABLE = 40 HDR_OFF_FRAME_TABLE = 48 - FILE_HEADER_PLACEHOLDER_SIZE = 64 FILE_FOOTER_SIZE = 32 FTR_OFF_STRINGS = 0 FTR_OFF_FRAMES = 4 @@ -1261,7 +1627,8 @@ def test_replay_rejects_trailing_partial_sample_header(self): with BinaryReader(filename) as reader: with self.assertRaises(ValueError) as cm: reader.replay_samples(RawCollector()) - self.assertEqual(str(cm.exception), "Truncated sample data: 1 trailing bytes") + self.assertEqual(str(cm.exception), + "Invalid sample block: truncated header") # Minimum on-disk size of one table entry (see binary_io.h). MIN_STRING_ENTRY_SIZE = 1 @@ -1345,6 +1712,21 @@ def test_open_accepts_frame_count_at_capacity_boundary(self): f"possible {max_frames}", ) + def test_open_rejects_older_format_version(self): + """Open rejects version 1 files and reports version 2.""" + filename = self.create_binary_file([], compression="none") + with BinaryReader(filename) as reader: + self.assertEqual(reader.get_info()["version"], 2) + with open(filename, "r+b") as raw: + raw.seek(self.HDR_OFF_VERSION) + raw.write(struct.pack("=I", 1)) + with self.assertRaisesRegex( + ValueError, + r"Unsupported format version 1 \(this reader supports " + r"version 2\)"): + with BinaryReader(filename): + pass + def test_sample_count_reads_full_64_bits(self): """sample_count values requiring the upper 32 bits decode correctly.""" filename = self.create_binary_file([], compression="none") @@ -1370,12 +1752,331 @@ def test_sample_count_boundary_values(self): with BinaryReader(filename) as reader: self.assertEqual(reader.get_info()["sample_count"], value) + # Block column indices and encodings (see binary_io.h). + MAX_BLOCK_SIZE = 1024 * 1024 + + def _write_single_block(self, num_samples=1, instructions=b"\x01\x00\x00", + frame_ids=b"", *, header_samples=None, + file_samples=None, stream_count=1, + stream_table=None, stream_ids=None, + residuals=None, statuses=b"", status_flags=(0, 0), + frame_width=1, descriptors=None, magic=b"TBLK", + truncate=None, frame_count=1): + """Write a file whose only block is built from the given parts. + + By default every sample belongs to stream 0, has status 0 and is + exactly one sample interval after the previous one, and the only + stack is empty (STACK_REPLACE_SUFFIX 0 0). The frame table holds + *frame_count* frames: frame 0 is in "a.py" and frame N in "N.py". + *descriptors* maps a column index to (encoding, decoded size) to + replace its raw descriptor. + """ + frames = [make_frame("a.py", 1, "f")] + frames += [make_frame(f"{index}.py", 1, "f") + for index in range(1, frame_count)] + samples = [[make_interpreter(0, [make_thread(1, frames[i:i + 256])])] + for i in range(0, frame_count, 256)] + filename = self.create_binary_file(samples, compression="none") + data = bytearray(pathlib.Path(filename).read_bytes()) + sample_end = struct.unpack_from("=Q", data, self.HDR_OFF_STR_TABLE)[0] + if stream_table is None: + stream_table = b"".join(struct.pack("=QI", thread, 0) + for thread in range(1, stream_count + 1)) + columns = ( + stream_table, + b"\x00" * num_samples if stream_ids is None else stream_ids, + b"\x00" * num_samples if residuals is None else residuals, + statuses, + instructions, + frame_ids, + ) + if header_samples is None: + header_samples = num_samples + block = (magic + encode_varint(header_samples) + + b"\x01" # base timestamp + + bytes([frame_width, *status_flags])) + payload = b"" + for index, column in enumerate(columns): + encoding, decoded_size = (descriptors or {}).get( + index, (0, len(column))) + block += encode_varint(decoded_size) + bytes([encoding]) + if encoding == self.COLUMN_ZSTD: + block += encode_varint(len(column)) + if encoding == self.COLUMN_CONSTANT: + column = column[:1] + payload += column + block = (block + payload)[:truncate] + data[self.FILE_HEADER_PLACEHOLDER_SIZE:sample_end] = block + shift = len(block) - (sample_end - self.FILE_HEADER_PLACEHOLDER_SIZE) + if file_samples is None: + file_samples = num_samples + struct.pack_into("=Q", data, self.HDR_OFF_SAMPLES, file_samples) + struct.pack_into("=I", data, self.HDR_OFF_THREADS, + max(len(stream_table) // 12, 1)) + for offset in (self.HDR_OFF_STR_TABLE, self.HDR_OFF_FRAME_TABLE): + old_offset = struct.unpack_from("=Q", data, offset)[0] + struct.pack_into("=Q", data, offset, old_offset + shift) + struct.pack_into("=Q", data, len(data) - 24, len(data)) + pathlib.Path(filename).write_bytes(data) + return filename + + def assert_block_rejected(self, filename, message): + with BinaryReader(filename) as reader: + with self.assertRaises(ValueError) as cm: + reader.replay_samples(RawCollector()) + self.assertEqual(str(cm.exception), f"Invalid sample block: {message}") + + def test_replay_accepts_hand_built_block(self): + """Replay accepts a block built by hand from its columns.""" + # STACK_REPLACE_SUFFIX 0 0, then STACK_REPEAT 2. + filename = self._write_single_block(3, b"\x01\x00\x00\x00\x02") + collector = RawCollector() + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(collector), 3) + self.assertEqual(len(collector.by_thread[(0, 1)]), 3) + + def test_replay_decodes_every_frame_id_width(self): + """Frame IDs decode correctly at every byte width.""" + # STACK_REPLACE_SUFFIX 0 1 reads one frame ID, stored in one byte + # plane per byte of the width, low byte first. + for width, frame_id in ((1, 43), (2, 299), (3, 299), (4, 299)): + with self.subTest(width=width): + filename = self._write_single_block( + 1, b"\x01\x00\x01", frame_id.to_bytes(width, "little"), + frame_width=width, frame_count=300) + collector = RawCollector() + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(collector), 1) + frames = collector.by_thread[(0, 1)][0]["frames"] + self.assertEqual([frame["filename"] for frame in frames], + [f"{frame_id}.py"]) + + def test_replay_reads_constant_stream_and_residual_columns(self): + """Constant stream index and residual columns decode every sample.""" + # Every sample is in stream 0 and one microsecond late (zigzag 2). + filename = self._write_single_block( + 3, b"\x01\x00\x00\x00\x02", residuals=b"\x02" * 3, + descriptors={self.COLUMN_STREAM_IDS: (self.COLUMN_CONSTANT, 3), + self.COLUMN_TIMESTAMP_RESIDUALS: + (self.COLUMN_CONSTANT, 3)}) + collector = mock.Mock() + with BinaryReader(filename) as reader: + interval = reader.get_info()["sample_interval_us"] + self.assertEqual(reader.replay_samples(collector), 3) + timestamps = [ts for call in collector.collect.call_args_list + for ts in call.args[1]] + self.assertEqual(timestamps, + [2 + i * (interval + 1) for i in range(3)]) + + def test_replay_rejects_unterminated_constant_varint(self): + """A constant byte that never ends a varint is rejected.""" + for column, part in ((self.COLUMN_STREAM_IDS, "stream_ids"), + (self.COLUMN_TIMESTAMP_RESIDUALS, "residuals")): + with self.subTest(part): + filename = self._write_single_block( + **{part: b"\x80"}, + descriptors={column: (self.COLUMN_CONSTANT, 1)}) + with BinaryReader(filename) as reader: + with self.assertRaisesRegex(ValueError, + "incomplete varint"): + reader.replay_samples(RawCollector()) + + def test_replay_rejects_reentrant_replay(self): + """Replaying from inside the same reader's collector raises.""" + filename = self._write_single_block( + 2, b"\x01\x00\x00\x00\x01") # REPEAT 1 + + with BinaryReader(filename) as reader: + class ReentrantCollector: + def collect(self, stack_frames, timestamps_us): + reader.replay_samples(RawCollector()) + + with self.assertRaisesRegex(RuntimeError, "already replaying"): + reader.replay_samples(ReentrantCollector()) + self.assertEqual(reader.replay_samples(RawCollector()), 2) + + def test_replay_rejects_close_during_replay(self): + """Closing the reader from inside its collector raises.""" + filename = self._write_single_block( + 2, b"\x01\x00\x00\x00\x01") # REPEAT 1 + closers = { + "close": lambda reader: reader._reader.close(), + "__exit__": lambda reader: reader._reader.__exit__( + None, None, None), + "__init__": lambda reader: reader._reader.__init__(filename), + } + for name, close in closers.items(): + with self.subTest(name): + with BinaryReader(filename) as reader: + class ClosingCollector: + def collect(self, stack_frames, timestamps_us): + close(reader) + + with self.assertRaisesRegex(RuntimeError, "replaying"): + reader.replay_samples(ClosingCollector()) + self.assertEqual(reader.replay_samples(RawCollector()), 2) + + def test_replay_rejects_close_from_initial_progress_callback(self): + """Closing the reader from its initial progress callback raises.""" + filename = self._write_single_block( + 2, b"\x01\x00\x00\x00\x01") # REPEAT 1 + + with BinaryReader(filename) as reader: + def close_reader(current, total): + reader._reader.close() + + with self.assertRaisesRegex(RuntimeError, "replaying"): + reader.replay_samples(RawCollector(), close_reader) + self.assertEqual(reader.replay_samples(RawCollector()), 2) + + def test_replay_rejects_invalid_block(self): + """Replay rejects every kind of malformed block.""" + frames = self.COLUMN_FRAME_IDS + cases = [ + ("wrong magic", dict(magic=b"XBLK")), + # Magic, sample count and base timestamp, then 1 of 3 flag bytes + ("truncated header", dict(truncate=7)), + ("no samples", dict(header_samples=0)), + ("frame ID width out of range", dict(frame_width=0)), + ("frame ID width out of range", dict(frame_width=5)), + ("common and varying status flags overlap", + dict(status_flags=(1, 1))), + # Magic, sample count, base timestamp, 3 flag bytes and the + # first decoded size + ("truncated column descriptor", dict(truncate=10)), + ("unknown column encoding", dict(descriptors={frames: (3, 0)})), + ("zstd column in an uncompressed file", + dict(descriptors={self.COLUMN_STACK_INSTRUCTIONS: + (self.COLUMN_ZSTD, 3)})), + ("encoded column is empty", + dict(descriptors={frames: (self.COLUMN_CONSTANT, 0)})), + ("decoded columns exceed the block limit", + dict(frame_ids=b"\x00", + descriptors={frames: (self.COLUMN_CONSTANT, + self.MAX_BLOCK_SIZE)})), + # Rejected before anything is allocated for the samples + ("more samples than stored stream indices", + dict(header_samples=0xFFFFFFFF)), + ("column extends past the sample section", dict(truncate=-1)), + ("stream table has a partial entry", dict(stream_table=b"0" * 11)), + ("stream count out of range", dict(stream_table=b"")), + ("frame ID column has a partial value", + dict(frame_width=2, frame_ids=b"\x00")), + ("status bitmaps have the wrong size", dict(status_flags=(0, 1))), + ("status bitmap padding is not zero", + dict(status_flags=(0, 1), statuses=b"\x02")), + ("stream table repeats a thread", + dict(stream_table=struct.pack("=QI", 1, 0) * 2)), + ("sample refers to an unknown stream", dict(stream_ids=b"\x01")), + ("sample refers to an unknown stream", + dict(num_samples=2, stream_ids=b"\x01", + descriptors={self.COLUMN_STREAM_IDS: + (self.COLUMN_CONSTANT, 2)})), + ("stream index column has trailing data", + dict(num_samples=2, header_samples=1)), + # A constant column longer than the sample count is expanded + # and then rejected like any other trailing data + ("stream index column has trailing data", + dict(descriptors={self.COLUMN_STREAM_IDS: + (self.COLUMN_CONSTANT, 2)})), + ("timestamp column has trailing data", + dict(residuals=b"\x00\x00")), + ("timestamp column has trailing data", + dict(descriptors={self.COLUMN_TIMESTAMP_RESIDUALS: + (self.COLUMN_CONSTANT, 2)})), + ("frame index out of range", + dict(instructions=b"\x01\x00\x01", frame_ids=b"\x05")), + # Frame IDs 65536 and 2**24 only have a byte in the third or + # fourth plane + ("frame index out of range", + dict(instructions=b"\x01\x00\x01", frame_width=3, + frame_ids=(1 << 16).to_bytes(3, "little"), + frame_count=300)), + ("frame index out of range", + dict(instructions=b"\x01\x00\x01", frame_width=4, + frame_ids=(1 << 24).to_bytes(4, "little"), + frame_count=300)), + ("unknown stack instruction", dict(instructions=b"\x03")), + ("first stack of a stream cannot repeat", + dict(instructions=b"\x00\x01")), + ("repeat count out of range", + dict(num_samples=2, instructions=b"\x01\x00\x00\x00\x00")), + ("stack prefix out of range", dict(instructions=b"\x01\x01\x00")), + ("stack uses more frame IDs than stored", + dict(instructions=b"\x01\x00\x02", frame_ids=b"\x00")), + # STACK_REUSE 0 257 + ("stack is too deep", dict(instructions=b"\x02\x00\x81\x02")), + ("reused stack is not stored earlier", + dict(instructions=b"\x02\x00\x01", frame_ids=b"\x00")), + ("stack instruction column has trailing data", + dict(instructions=b"\x01\x00\x00\x01")), + ("frame ID column has unused values", dict(frame_ids=b"\x00")), + ("stream table has an unused entry", dict(stream_count=2)), + ("repeat count extends past the block", + dict(num_samples=2, instructions=b"\x01\x00\x00\x00\x02")), + ("more samples than the file header declares", + dict(file_samples=0)), + ] + for message, kwargs in cases: + with self.subTest(message, **kwargs): + filename = self._write_single_block(**kwargs) + self.assert_block_rejected(filename, message) + + @unittest.skipUnless(ZSTD_AVAILABLE, "zstd compression not available") + def test_replay_rejects_invalid_zstd_column(self): + """Replay rejects corrupt and mismatched zstd columns.""" + zstd = import_helper.import_module("compression.zstd") + instructions = b"\x01\x00\x00" + frame = zstd.compress( + instructions, + options={zstd.CompressionParameter.checksum_flag: 1}) + bad_checksum = frame[:-1] + bytes([frame[-1] ^ 1]) + cases = [ + ("Corrupted zstd frame header", instructions, len(instructions)), + ("zstd frame size does not match the decoded size", + frame, len(instructions) + 1), + ("zstd decompression error: ", bad_checksum, len(instructions)), + ] + for message, stored, decoded_size in cases: + with self.subTest(message): + filename = self._write_single_block( + instructions=stored, + descriptors={self.COLUMN_STACK_INSTRUCTIONS: + (self.COLUMN_ZSTD, decoded_size)}) + with open(filename, "r+b") as raw: + raw.seek(56) # compression type + raw.write(struct.pack("=I", 1)) + with BinaryReader(filename) as reader: + with self.assertRaises(ValueError) as cm: + reader.replay_samples(RawCollector()) + self.assertStartsWith(str(cm.exception), message) + + def test_replay_propagates_callback_errors(self): + """Errors raised by the collector or progress callback propagate.""" + filename = self._write_single_block( + 2000, b"\x01\x00\x00\x00\xcf\x0f") # REPEAT 1999 + + class FailingCollector: + def collect(self, stack_frames, timestamps_us): + raise ZeroDivisionError + + def failing_progress(current, total): + if current: + raise ZeroDivisionError + + with BinaryReader(filename) as reader: + with self.assertRaises(ZeroDivisionError): + reader.replay_samples(FailingCollector()) + with self.assertRaises(ZeroDivisionError): + reader.replay_samples(RawCollector(), failing_progress) + self.assertEqual(reader.replay_samples(RawCollector()), 2000) + class TestBinaryEncodings(BinaryFormatTestBase): """Tests specifically targeting different stack encodings.""" - def test_stack_full_encoding(self): - """First sample uses STACK_FULL encoding and roundtrips.""" + def test_stack_first_sample_pattern(self): + """First sample stores its complete stack and roundtrips.""" frames = [make_frame(f"f{i}.py", i + 1, f"func{i}") for i in range(5)] samples = [[make_interpreter(0, [make_thread(1, frames)])]] collector, count = self.roundtrip(samples) @@ -1393,8 +2094,8 @@ def test_stack_repeat_encoding(self): self.assertEqual(count, 1000) self.assert_samples_equal(samples, collector) - def test_stack_suffix_encoding(self): - """Samples sharing suffix use STACK_SUFFIX and roundtrip.""" + def test_stack_shared_outer_frames_pattern(self): + """Samples keeping outer frames roundtrip (STACK_REPLACE_SUFFIX).""" samples = [] for i in range(10): frames = [make_frame(f"new{i}.py", i + 1, f"new{i}")] @@ -1409,8 +2110,8 @@ def test_stack_suffix_encoding(self): self.assertEqual(count, 10) self.assert_samples_equal(samples, collector) - def test_stack_pop_push_encoding(self): - """Samples with pop+push pattern roundtrip.""" + def test_stack_pop_push_pattern(self): + """Samples that pop frames and push new ones roundtrip.""" samples = [] base_frames = [make_frame("base.py", 10, "base")] @@ -1549,15 +2250,29 @@ def export(self, filename): def test_rle_long_run_splits_batches(self): - # Construct a single repeat record larger than the writer's buffer. - num_samples = 8193 + # Construct one block whose repeat run is longer than a replay batch. + num_samples = 8194 filename = self.create_binary_file([], compression="none") data = bytearray(pathlib.Path(filename).read_bytes()) - record = (struct.pack("=QIB", 1, 0, 0) # STACK_REPEAT - + b"\x81\x40" # 8193 as a varint - + b"\x01\x00" * num_samples) # delta=1, status=0 + instructions = (b"\x01\x00\x00" # STACK_REPLACE_SUFFIX 0 0 + + b"\x00\x81\x40") # STACK_REPEAT 8193 + record = (b"TBLK" + + b"\x82\x40" # 8194 samples as a varint + + b"\x01" # base timestamp + + b"\x01\x00\x00" # frame width, status flags + + b"\x0c\x00" # stream table: 12 raw bytes + + b"\x82\x40\x02" # stream indices: constant + + b"\x82\x40\x02" # timestamp residuals: constant + + b"\x00\x00" # status bitmaps: empty + + bytes([len(instructions), 0]) # raw instructions + + b"\x00\x00" # frame IDs: empty + + struct.pack("=QI", 1, 0) + + b"\x00" # every sample is in stream 0 + + b"\x00" # every residual is zero + + instructions) data[64:64] = record struct.pack_into("=Q", data, 12, 0) # start timestamp + struct.pack_into("=Q", data, 20, 1) # sample interval struct.pack_into("=Q", data, 28, num_samples) struct.pack_into("=I", data, 36, 1) # thread count for offset in (40, 48): # string and frame table offsets @@ -1571,7 +2286,7 @@ def test_rle_long_run_splits_batches(self): count = reader.replay_samples(collector) batches = [call.args[1] for call in collector.collect.call_args_list] self.assertEqual(count, num_samples) - self.assertEqual([len(batch) for batch in batches], [8192, 1]) + self.assertEqual([len(batch) for batch in batches], [1, 8192, 1]) self.assertEqual([ts for batch in batches for ts in batch], list(range(1, num_samples + 1))) @@ -1701,7 +2416,7 @@ def test_encoding_transitions_stress(self): # Repeat previous (will use RLE if identical) frames = base_frames[: random.randint(1, 5)] elif choice == 2: - # Add frames on top (suffix encoding) + # Add frames on top (keeps the outer frames) extra = random.randint(1, 3) frames = [ make_frame(f"top{i}_{j}.py", j, f"top{j}") @@ -1709,7 +2424,7 @@ def test_encoding_transitions_stress(self): ] frames.extend(base_frames[: random.randint(2, 4)]) else: - # Pop and push (pop-push encoding) + # Pop some frames and push new ones keep = random.randint(1, 3) push = random.randint(0, 2) frames = [ @@ -1893,6 +2608,25 @@ def test_replay_includes_persisted_stats(self): self.assertIn('"error_rate": 2.5', content) self.assertIn('"missed_samples": 1.5', content) + @unittest.skipUnless(ZSTD_AVAILABLE, "zstd compression not available") + def test_measured_stats_with_zstd(self): + """Measured statistics are read back when zstd is enabled.""" + frames = [make_frame(f"pkg/module_{i:03}.py", i + 1, f"func_{i:03}") + for i in range(200)] + sample = [make_interpreter(0, [make_thread(1, frames)])] + filename = self.temp_filename() + collector = BinaryCollector(filename, 2000, compression="zstd") + collector.collect(sample, timestamp_us=1000) + collector.set_stats(2000, 1.25, 4.0, error_rate=2.5, + missed_samples=1.5) + collector.export(None) + + with BinaryReader(filename) as reader: + info = reader.get_info() + self.assertEqual(info["duration_sec"], 1.25) + self.assertEqual(info["missed_samples"], 1.5) + self.assertEqual(reader.replay_samples(RawCollector()), 1) + def test_legacy_binary_has_no_measured_stats(self): frame = make_frame("hot.py", 99, "hot_func") bin_path = self.create_binary_file([ diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py index e5dcd72c27ef5c2..789cc3408d5a856 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py @@ -954,7 +954,7 @@ def test_cli_replay_reader_errors_exit_cleanly(self): mock.patch("sys.argv", ["profiling.sampling.cli", "replay", profile]), mock.patch( "profiling.sampling.cli.BinaryReader", - side_effect=ValueError("Unsupported format version 2"), + side_effect=ValueError("Unsupported format version 3"), ), ): with self.assertRaises(SystemExit) as cm: @@ -962,7 +962,7 @@ def test_cli_replay_reader_errors_exit_cleanly(self): self.assertEqual( str(cm.exception), - "Error: Unsupported format version 2", + "Error: Unsupported format version 3", ) def test_cli_replay_propagates_recorded_mode(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index d0dcf163e380248..650838d68286c73 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3526,7 +3526,7 @@ MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h $(srcdir)/Modules/_openssl_mem.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h -MODULE__REMOTE_DEBUGGING_DEPS=$(srcdir)/Modules/_remote_debugging/_remote_debugging.h $(srcdir)/Modules/_remote_debugging/gc_stats.h +MODULE__REMOTE_DEBUGGING_DEPS=$(srcdir)/Modules/_remote_debugging/_remote_debugging.h $(srcdir)/Modules/_remote_debugging/binary_io.h $(srcdir)/Modules/_remote_debugging/gc_stats.h # HACL*-based cryptographic primitives MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) diff --git a/Misc/NEWS.d/next/Library/2026-10-20-12-49-32.gh-issue-159207.Lccezg.rst b/Misc/NEWS.d/next/Library/2026-10-20-12-49-32.gh-issue-159207.Lccezg.rst new file mode 100644 index 000000000000000..a96098fd211abc6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-20-12-49-32.gh-issue-159207.Lccezg.rst @@ -0,0 +1,3 @@ +The :mod:`profiling.sampling` binary output format now stores samples in +independent blocks. This is a breaking format change: Python 3.16 does not +support reading profiles written in the previous format. diff --git a/Modules/_remote_debugging/binary_io.h b/Modules/_remote_debugging/binary_io.h index 6a2c5b795823e15..4f740d953f5dcc3 100644 --- a/Modules/_remote_debugging/binary_io.h +++ b/Modules/_remote_debugging/binary_io.h @@ -2,7 +2,7 @@ * Python Remote Debugging Module - Binary I/O Header * * This header provides declarations for high-performance binary file I/O - * for profiling data with optional zstd streaming compression. + * for profiling data with optional zstd compression. ******************************************************************************/ #ifndef Py_BINARY_IO_H @@ -23,7 +23,7 @@ extern "C" { #define BINARY_FORMAT_MAGIC 0x54414348 /* "TACH" (Tachyon) in native byte order */ #define BINARY_FORMAT_MAGIC_SWAPPED 0x48434154 /* Byte-swapped magic for endianness detection */ -#define BINARY_FORMAT_VERSION 1 +#define BINARY_FORMAT_VERSION 2 /* Sentinel values for optional frame fields */ #define OPCODE_NONE 255 /* No opcode captured (u8 sentinel) */ @@ -67,17 +67,32 @@ extern "C" { static_assert(FILE_HEADER_SIZE <= FILE_HEADER_PLACEHOLDER_SIZE, "FILE_HEADER_SIZE exceeds FILE_HEADER_PLACEHOLDER_SIZE"); -/* Sample header field offsets and sizes */ -#define SMP_OFF_THREAD_ID 0 -#define SMP_SIZE_THREAD_ID sizeof(uint64_t) -#define SMP_OFF_INTERPRETER_ID (SMP_OFF_THREAD_ID + SMP_SIZE_THREAD_ID) -#define SMP_SIZE_INTERPRETER_ID sizeof(uint32_t) -#define SMP_OFF_ENCODING (SMP_OFF_INTERPRETER_ID + SMP_SIZE_INTERPRETER_ID) -#define SMP_SIZE_ENCODING sizeof(uint8_t) -#define SAMPLE_HEADER_FIXED_SIZE (SMP_OFF_ENCODING + SMP_SIZE_ENCODING) - -static_assert(SAMPLE_HEADER_FIXED_SIZE == 13, - "SAMPLE_HEADER_FIXED_SIZE must remain 13"); +/* Sample blocks */ +#define BLOCK_MAGIC "TBLK" +#define BLOCK_MAGIC_SIZE (sizeof(BLOCK_MAGIC) - 1) +#define MAX_BLOCK_SIZE (1024 * 1024) +#define MAX_BLOCK_STREAMS 4096 + +/* Block columns */ +enum { + COLUMN_STREAM_TABLE, + COLUMN_SAMPLE_STREAM_IDS, + COLUMN_TIMESTAMP_RESIDUALS, + COLUMN_STATUS_BITMAPS, + COLUMN_STACK_INSTRUCTIONS, + COLUMN_FRAME_IDS, + COLUMN_COUNT +}; + +/* Stream table entry offsets and sizes */ +#define STM_OFF_THREAD_ID 0 +#define STM_SIZE_THREAD_ID sizeof(uint64_t) +#define STM_OFF_INTERPRETER_ID (STM_OFF_THREAD_ID + STM_SIZE_THREAD_ID) +#define STM_SIZE_INTERPRETER_ID sizeof(uint32_t) +#define STREAM_ENTRY_SIZE (STM_OFF_INTERPRETER_ID + STM_SIZE_INTERPRETER_ID) + +static_assert(STREAM_ENTRY_SIZE == 12, + "STREAM_ENTRY_SIZE must remain 12"); /* Footer field offsets and sizes */ #define FTR_OFF_STRINGS 0 @@ -128,14 +143,10 @@ static_assert(PROFILE_STATS_SIZE == 56, #define MIN_STRING_ENTRY_SIZE 1 #define MIN_FRAME_ENTRY_SIZE 7 -/* Buffer sizes: 512KB balances syscall amortization against memory use, - * and aligns well with filesystem block sizes and zstd dictionary windows */ -#define WRITE_BUFFER_SIZE (512 * 1024) -#define COMPRESSED_BUFFER_SIZE (512 * 1024) - /* Compression types */ #define COMPRESSION_NONE 0 #define COMPRESSION_ZSTD 1 +#define COLUMN_CONSTANT 2 /* Column stored as one repeated byte */ /* Profiling configuration. The mode occupies the low three bits and is * stored plus one so zero remains compatible with files written before this @@ -152,17 +163,24 @@ static_assert(PROFILE_STATS_SIZE == 56, /* Stack encoding types for delta compression */ #define STACK_REPEAT 0x00 /* RLE: identical to previous, with count */ -#define STACK_FULL 0x01 /* Full stack (first sample or no match) */ -#define STACK_SUFFIX 0x02 /* Shares N frames from bottom */ -#define STACK_POP_PUSH 0x03 /* Remove M frames, add N frames */ +#define STACK_REPLACE_SUFFIX 0x01 /* Keep outer frames, append new ones */ +#define STACK_REUSE 0x02 /* Stack stored earlier in the block */ /* Maximum stack depth we'll buffer for delta encoding */ #define MAX_STACK_DEPTH 256 +/* Writer's stack cache size per block */ +#define STACK_CACHE_SIZE 4096 +#define STACK_CACHE_BUCKET_SIZE 2 +#define STACK_CACHE_BUCKET_COUNT \ + (STACK_CACHE_SIZE / STACK_CACHE_BUCKET_SIZE) + /* Initial capacities for dynamic arrays - sized to reduce reallocations */ #define INITIAL_STRING_CAPACITY 4096 #define INITIAL_FRAME_CAPACITY 4096 #define INITIAL_THREAD_CAPACITY 256 +#define INITIAL_BLOCK_CAPACITY 256 +#define INITIAL_BLOCK_STREAM_CAPACITY 16 /* ============================================================================ * STATISTICS STRUCTURES @@ -174,7 +192,7 @@ typedef struct { uint64_t repeat_samples; /* Total samples encoded via RLE */ uint64_t full_records; /* Number of full stack records */ uint64_t suffix_records; /* Number of suffix match records */ - uint64_t pop_push_records; /* Number of pop-push records */ + uint64_t reuse_records; /* Number of stack reuse records */ uint64_t total_frames_written;/* Total frame indices written */ uint64_t frames_saved; /* Frames avoided due to delta encoding */ uint64_t bytes_written; /* Total bytes written (before compression) */ @@ -186,7 +204,7 @@ typedef struct { uint64_t repeat_samples; /* Samples decoded from RLE */ uint64_t full_records; /* Full stack records decoded */ uint64_t suffix_records; /* Suffix match records decoded */ - uint64_t pop_push_records; /* Pop-push records decoded */ + uint64_t reuse_records; /* Stack reuse records decoded */ uint64_t total_samples; /* Total samples reconstructed */ uint64_t stack_reconstructions; /* Number of stack array reconstructions */ } BinaryReaderStats; @@ -245,8 +263,6 @@ typedef struct { #else void *cctx; /* Placeholder */ #endif - uint8_t *compressed_buffer; - size_t compressed_buffer_size; } ZstdCompressor; /* Frame entry - combines all frame data for better cache locality. @@ -273,22 +289,25 @@ typedef struct { uint8_t opcode; } FrameKey; -/* Thread entry - tracks per-thread state for delta encoding */ +/* One entry per (thread, interpreter) pair seen in the file */ typedef struct { uint64_t thread_id; - uint64_t prev_timestamp; uint32_t interpreter_id; + /* Stream index in the current block, or UINT32_MAX */ + uint32_t block_stream_index; +} ThreadEntry; + +/* Delta encoding state of a stream, reset with each block */ +typedef struct { + uint64_t prev_timestamp; + size_t thread_index; + uint32_t current_repeat; + /* Previous stack for delta encoding (frame indices, innermost first) */ - uint32_t *prev_stack; size_t prev_stack_depth; - size_t prev_stack_capacity; - - /* RLE pending buffer - samples waiting to be written as a repeat group */ - uint8_t *pending_rle; - size_t pending_rle_bytes; - size_t pending_rle_samples; -} ThreadEntry; + uint32_t prev_stack[MAX_STACK_DEPTH]; +} BlockStream; /* Limit errors occur before emitting an incomplete sample. Other write * failures may leave partial records and must prevent finalization. */ @@ -298,15 +317,65 @@ typedef enum { BINARY_WRITER_BROKEN, } BinaryWriterState; +/* Byte buffer for block data */ +typedef struct { + uint8_t *data; + size_t size; + size_t capacity; +} ByteBuffer; + +/* Column size and encoding */ +typedef struct { + uint32_t decoded_size; + uint32_t stored_size; + uint8_t encoding; +} ColumnDescriptor; + +/* Block header as stored in the file */ +typedef struct { + uint32_t sample_count; + uint64_t base_timestamp; + uint8_t frame_width; + uint8_t common_status_flags; + uint8_t status_bitmap_flags; + ColumnDescriptor columns[COLUMN_COUNT]; +} BlockHeader; + +/* Complete stack that later samples can REUSE */ +typedef struct { + Py_uhash_t hash; + uint32_t offset; + uint32_t depth; +} StackCacheEntry; + +/* REPEAT instruction position and count */ +typedef struct { + uint32_t offset; + uint32_t count; +} Repeat; + +/* Samples buffered until the next flush */ +typedef struct { + ByteBuffer columns[COLUMN_COUNT]; + ByteBuffer repeats; + ByteBuffer transformed; + ByteBuffer compressed; + ByteBuffer output; + StackCacheEntry cache[STACK_CACHE_BUCKET_COUNT][STACK_CACHE_BUCKET_SIZE]; + BlockStream *streams; + size_t stream_capacity; + uint64_t base_timestamp; + uint32_t sample_count; + uint32_t stream_count; + uint32_t max_frame; + size_t size; +} SampleBlock; + /* Main binary writer structure */ typedef struct { FILE *fp; BinaryWriterState state; - - /* Write buffer for batched I/O */ - uint8_t *write_buffer; - size_t buffer_pos; - size_t buffer_size; + SampleBlock *block; /* Compression */ int compression_type; @@ -353,17 +422,37 @@ typedef struct { * BINARY READER STRUCTURES * ============================================================================ */ -/* Per-thread state for stack reconstruction during replay */ +typedef struct { + uint32_t frame_index; + PyObject *frame_info; +} ReaderFrameCacheEntry; + +/* Decoded sample block */ +typedef struct { + BlockHeader header; + uint32_t stream_count; + uint32_t frame_count; + ByteBuffer columns[COLUMN_COUNT]; + const uint32_t *stream_ids; + const uint64_t *timestamps; + const uint8_t *statuses; + const uint32_t *frames; + size_t instruction_offset; + uint32_t frame_offset; +} DecodedBlock; + +/* Per-stream replay state */ typedef struct { uint64_t thread_id; uint32_t interpreter_id; uint64_t prev_timestamp; - - /* Reconstructed stack buffer (frame indices, innermost first) */ - uint32_t *current_stack; + uint32_t current_stack[MAX_STACK_DEPTH]; size_t current_stack_depth; - size_t current_stack_capacity; -} ReaderThreadState; + uint32_t pending_repeats; + int has_stack; + PyObject *batch; + uint8_t batch_status; +} ReaderBlockStream; /* Main binary reader structure */ typedef struct { @@ -378,9 +467,9 @@ typedef struct { /* Decompression state */ int compression_type; - /* Note: ZSTD_DCtx is not stored - created/freed during decompression */ - uint8_t *decompressed_data; - size_t decompressed_size; +#ifdef HAVE_ZSTD + ZSTD_DCtx *dctx; +#endif /* Header metadata */ uint8_t py_major; @@ -409,13 +498,24 @@ typedef struct { /* Parsed frame table: array of FrameEntry structures */ FrameEntry *frames; uint32_t frames_count; + ReaderFrameCacheEntry *frame_info_cache; + size_t frame_info_cache_size; /* Sample data region */ uint8_t *sample_data; size_t sample_data_size; - /* Per-thread state for stack reconstruction (used during replay) */ - ReaderThreadState *thread_states; + ByteBuffer block_data; + ByteBuffer block_stream_ids; + ByteBuffer block_timestamps; + ByteBuffer block_statuses; + ByteBuffer block_frames; + ReaderBlockStream *block_streams; + size_t block_stream_capacity; + int replaying; + + /* Threads seen during replay */ + ThreadEntry *thread_states; size_t thread_state_count; size_t thread_state_capacity; @@ -462,6 +562,14 @@ encode_varint_i32(uint8_t *buf, int32_t value) return encode_varint_u32(buf, zigzag); } +/* Encode signed 64-bit varint (zigzag encoding). Returns bytes written. */ +static inline size_t +encode_varint_i64(uint8_t *buf, int64_t value) +{ + uint64_t zigzag = ((uint64_t)value << 1) ^ (uint64_t)(value >> 63); + return encode_varint_u64(buf, zigzag); +} + /* Decode unsigned 64-bit varint (LEB128). Updates offset only on success. * On error (overflow or incomplete), offset is NOT updated, allowing callers * to detect errors via (offset == prev_offset) check. Sets PyErr on error. */ @@ -525,10 +633,64 @@ decode_varint_i32(const uint8_t *data, size_t *offset, size_t max_size) return (int32_t)((zigzag >> 1) ^ -(int32_t)(zigzag & 1)); } +/* Decode signed 64-bit varint (zigzag encoding). */ +static inline int64_t +decode_varint_i64(const uint8_t *data, size_t *offset, size_t max_size) +{ + size_t saved_offset = *offset; + uint64_t zigzag = decode_varint_u64(data, offset, max_size); + if (*offset == saved_offset) { + return 0; /* decode_varint_u64 already set PyErr */ + } + return (int64_t)((zigzag >> 1) ^ (0 - (zigzag & 1))); +} + /* ============================================================================ * SHARED UTILITY FUNCTIONS * ============================================================================ */ +/* Grow a byte buffer to hold at least size bytes */ +static inline int +buffer_reserve(ByteBuffer *buffer, size_t size) +{ + if (size <= buffer->capacity) { + return 0; + } + size_t capacity = buffer->capacity ? buffer->capacity + : INITIAL_BLOCK_CAPACITY; + while (capacity < size) { + if (capacity > SIZE_MAX / 2) { + PyErr_NoMemory(); + return -1; + } + capacity *= 2; + } + uint8_t *data = PyMem_Realloc(buffer->data, capacity); + if (data == NULL) { + PyErr_NoMemory(); + return -1; + } + buffer->data = data; + buffer->capacity = capacity; + return 0; +} + +static inline void +buffer_clear(ByteBuffer *buffer) +{ + PyMem_Free(buffer->data); + buffer->data = NULL; + buffer->size = 0; + buffer->capacity = 0; +} + +/* Bytes in one status bitmap */ +static inline size_t +status_bitmap_size(uint32_t sample_count) +{ + return ((size_t)sample_count + CHAR_BIT - 1) / CHAR_BIT; +} + /* Generic array growth - returns new pointer or NULL (sets PyErr_NoMemory) * Includes overflow checking for capacity doubling and allocation size. */ static inline void * @@ -716,6 +878,12 @@ void binary_reader_close(BinaryReader *reader); */ PyObject *binary_writer_get_stats(BinaryWriter *writer); +/* + * Decoded bytes of the samples buffered in the current block, not yet + * counted in the writer's bytes_written statistic. + */ +size_t binary_writer_buffered_size(BinaryWriter *writer); + /* * Get reader statistics as a Python dict. * diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index a51a2e475c9e06b..24079f26a137eb3 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -27,15 +27,14 @@ * CONSTANTS FOR BINARY FORMAT SIZES * ============================================================================ */ -/* File structure sizes */ -#define MIN_DECOMPRESS_BUFFER_SIZE (64 * 1024) /* Minimum decompression buffer */ - /* Progress callback frequency */ #define PROGRESS_CALLBACK_INTERVAL 1000 /* Cap per-batch RLE samples to bound the timestamp list (gh-151378) */ #define MAX_RLE_BATCH_SAMPLES 8192 +#define FRAME_INFO_CACHE_SIZE 1024 + /* ============================================================================ * BINARY READER IMPLEMENTATION * ============================================================================ */ @@ -230,119 +229,62 @@ reader_parse_profile_stats(BinaryReader *reader, const uint8_t *data, return 0; } -#ifdef HAVE_ZSTD -/* Maximum decompression buffer size to prevent memory exhaustion (1GB) */ -#define MAX_DECOMPRESS_SIZE (1ULL << 30) - -static inline int -reader_decompress_samples(BinaryReader *reader, const uint8_t *data) +static int +reader_decompress(BinaryReader *reader, const uint8_t *compressed_data, + size_t compressed_size, ByteBuffer *output, + size_t decoded_size) { - size_t compressed_size = reader->string_table_offset - FILE_HEADER_PLACEHOLDER_SIZE; - const uint8_t *compressed_data = data + FILE_HEADER_PLACEHOLDER_SIZE; - - /* Validate compressed data region */ - if (reader->string_table_offset < FILE_HEADER_PLACEHOLDER_SIZE) { - PyErr_SetString(PyExc_ValueError, "Invalid string table offset"); - return -1; - } - - ZSTD_DCtx *dctx = ZSTD_createDCtx(); - if (!dctx) { - PyErr_SetString(PyExc_MemoryError, "Failed to create zstd decompression context"); +#ifdef HAVE_ZSTD + /* The data must be exactly one zstd frame */ + size_t frame_size = ZSTD_findFrameCompressedSize(compressed_data, + compressed_size); + if (ZSTD_isError(frame_size) || frame_size != compressed_size) { + PyErr_SetString(PyExc_ValueError, "Corrupted zstd frame header"); return -1; } - /* Try to get exact decompressed size from frame header for optimal allocation */ + /* Check the decompressed size recorded in the frame header before + * allocating for it */ unsigned long long frame_content_size = ZSTD_getFrameContentSize(compressed_data, compressed_size); - size_t alloc_size; - if (frame_content_size == ZSTD_CONTENTSIZE_ERROR) { /* Corrupted frame header - fail early */ - ZSTD_freeDCtx(dctx); PyErr_SetString(PyExc_ValueError, "Corrupted zstd frame header"); return -1; - } else if (frame_content_size != ZSTD_CONTENTSIZE_UNKNOWN && - frame_content_size <= SIZE_MAX && - frame_content_size <= MAX_DECOMPRESS_SIZE) { - alloc_size = (size_t)frame_content_size; - } else { - alloc_size = ZSTD_DStreamOutSize() * 4; - if (alloc_size < MIN_DECOMPRESS_BUFFER_SIZE) { - alloc_size = MIN_DECOMPRESS_BUFFER_SIZE; - } } - - reader->decompressed_data = PyMem_Malloc(alloc_size); - if (!reader->decompressed_data) { - ZSTD_freeDCtx(dctx); - PyErr_NoMemory(); + if (frame_content_size != decoded_size) { + PyErr_SetString(PyExc_ValueError, + "zstd frame size does not match the decoded size"); return -1; } - ZSTD_inBuffer input = { compressed_data, compressed_size, 0 }; - size_t total_output = 0; - size_t last_result = 0; - - while (input.pos < input.size) { - if (total_output >= alloc_size) { - /* Check for overflow before doubling */ - if (alloc_size > SIZE_MAX / 2 || alloc_size * 2 > MAX_DECOMPRESS_SIZE) { - PyMem_Free(reader->decompressed_data); - reader->decompressed_data = NULL; - ZSTD_freeDCtx(dctx); - PyErr_SetString(PyExc_MemoryError, "Decompressed data exceeds maximum size"); - return -1; - } - size_t new_size = alloc_size * 2; - uint8_t *new_buf = PyMem_Realloc(reader->decompressed_data, new_size); - if (!new_buf) { - PyMem_Free(reader->decompressed_data); - reader->decompressed_data = NULL; - ZSTD_freeDCtx(dctx); - PyErr_NoMemory(); - return -1; - } - reader->decompressed_data = new_buf; - alloc_size = new_size; - } - - ZSTD_outBuffer output = { - reader->decompressed_data + total_output, - alloc_size - total_output, - 0 - }; - - last_result = ZSTD_decompressStream(dctx, &output, &input); - if (ZSTD_isError(last_result)) { - PyMem_Free(reader->decompressed_data); - reader->decompressed_data = NULL; - ZSTD_freeDCtx(dctx); - PyErr_Format(PyExc_ValueError, "zstd decompression error: %s", - ZSTD_getErrorName(last_result)); - return -1; - } - - _Py_MSAN_UNPOISON(output.dst, output.pos); - total_output += output.pos; + /* Append the decoded data to the output buffer */ + if (buffer_reserve(output, output->size + decoded_size) < 0) { + return -1; + } + uint8_t *decoded = output->data + output->size; + size_t result = ZSTD_decompressDCtx(reader->dctx, decoded, decoded_size, + compressed_data, compressed_size); + if (ZSTD_isError(result)) { + PyErr_Format(PyExc_ValueError, "zstd decompression error: %s", + ZSTD_getErrorName(result)); + return -1; } - /* Verify decompression is complete (last_result == 0 means frame is complete) */ - if (last_result != 0) { - PyMem_Free(reader->decompressed_data); - reader->decompressed_data = NULL; - ZSTD_freeDCtx(dctx); + /* Verify decompression is complete */ + if (result != decoded_size) { PyErr_SetString(PyExc_ValueError, "Incomplete zstd frame: data may be truncated"); return -1; } - ZSTD_freeDCtx(dctx); - reader->decompressed_size = total_output; - reader->sample_data = reader->decompressed_data; - reader->sample_data_size = reader->decompressed_size; - + _Py_MSAN_UNPOISON(decoded, decoded_size); + output->size += decoded_size; return 0; -} +#else + PyErr_SetString(PyExc_RuntimeError, + "File uses zstd compression but zstd support not compiled in"); + return -1; #endif +} /* Reject a table/run count whose entries cannot fit in the bytes still * available; a malicious file could otherwise drive a huge allocation. @@ -681,10 +623,11 @@ binary_reader_open(PyObject *path) goto error; } - /* Handle compressed data */ if (reader->compression_type == COMPRESSION_ZSTD) { #ifdef HAVE_ZSTD - if (reader_decompress_samples(reader, data) < 0) { + reader->dctx = ZSTD_createDCtx(); + if (!reader->dctx) { + PyErr_SetString(PyExc_MemoryError, "Failed to create zstd decompression context"); goto error; } #else @@ -692,10 +635,10 @@ binary_reader_open(PyObject *path) "File uses zstd compression but zstd support not compiled in"); goto error; #endif - } else { - reader->sample_data = data + FILE_HEADER_PLACEHOLDER_SIZE; - reader->sample_data_size = (size_t)reader->string_table_offset - FILE_HEADER_PLACEHOLDER_SIZE; } + reader->sample_data = data + FILE_HEADER_PLACEHOLDER_SIZE; + reader->sample_data_size = + (size_t)reader->string_table_offset - FILE_HEADER_PLACEHOLDER_SIZE; /* Parse string and frame tables */ if (reader_parse_string_table(reader, data, file_size) < 0) { @@ -713,7 +656,7 @@ binary_reader_open(PyObject *path) } /* Get or create reader thread state for stack reconstruction */ -static ReaderThreadState * +static ThreadEntry * reader_get_or_create_thread_state(BinaryReader *reader, uint64_t thread_id, uint32_t interpreter_id) { @@ -735,192 +678,104 @@ reader_get_or_create_thread_state(BinaryReader *reader, uint64_t thread_id, if (!reader->thread_states) { reader->thread_state_capacity = 16; - reader->thread_states = PyMem_Calloc(reader->thread_state_capacity, sizeof(ReaderThreadState)); + reader->thread_states = PyMem_Calloc(reader->thread_state_capacity, sizeof(ThreadEntry)); if (!reader->thread_states) { PyErr_NoMemory(); return NULL; } - } else if (reader->thread_state_count >= reader->thread_state_capacity) { - ReaderThreadState *new_states = grow_array(reader->thread_states, - &reader->thread_state_capacity, - sizeof(ReaderThreadState)); - if (!new_states) { - return NULL; - } - reader->thread_states = new_states; + } + else if (GROW_ARRAY(reader->thread_states, reader->thread_state_count, + reader->thread_state_capacity, ThreadEntry) < 0) { + return NULL; } - ReaderThreadState *ts = &reader->thread_states[reader->thread_state_count]; - memset(ts, 0, sizeof(ReaderThreadState)); + ThreadEntry *ts = &reader->thread_states[reader->thread_state_count]; + memset(ts, 0, sizeof(ThreadEntry)); ts->thread_id = thread_id; ts->interpreter_id = interpreter_id; - ts->prev_timestamp = reader->start_time_us; - ts->current_stack_capacity = MAX_STACK_DEPTH; - ts->current_stack = PyMem_Malloc(ts->current_stack_capacity * sizeof(uint32_t)); - if (!ts->current_stack) { - PyErr_NoMemory(); - return NULL; - } - // Increment count only after successful allocation to avoid - // leaving a half-initialized entry visible to future lookups reader->thread_state_count++; return ts; } -/* ============================================================================ - * STACK DECODING HELPERS - * ============================================================================ */ - -/* Validate that final_depth fits in the stack buffer. - * Uses uint64_t to prevent overflow on 32-bit platforms. */ -static inline int -validate_stack_depth(ReaderThreadState *ts, uint64_t final_depth) -{ - if (final_depth > ts->current_stack_capacity) { - PyErr_Format(PyExc_ValueError, - "Final stack depth %llu exceeds capacity %zu", - (unsigned long long)final_depth, ts->current_stack_capacity); - return -1; - } - return 0; -} - -/* Decode a full stack from sample data. - * Updates ts->current_stack and ts->current_stack_depth. - * Returns 0 on success, -1 on error (bounds violation). */ -static inline int -decode_stack_full(ReaderThreadState *ts, const uint8_t *data, - size_t *offset, size_t max_size) +static PyObject * +reader_get_frame_info(RemoteDebuggingState *state, BinaryReader *reader, + uint32_t frame_index) { - size_t prev_offset = *offset; - uint32_t depth = decode_varint_u32(data, offset, max_size); - if (*offset == prev_offset) { - return -1; - } - - /* Validate depth against capacity to prevent buffer overflow */ - if (depth > ts->current_stack_capacity) { - PyErr_Format(PyExc_ValueError, - "Stack depth %u exceeds capacity %zu", depth, ts->current_stack_capacity); - return -1; + if (frame_index >= reader->frames_count) { + PyErr_Format(PyExc_ValueError, "Invalid frame index: %u", frame_index); + return NULL; } - - ts->current_stack_depth = depth; - for (uint32_t i = 0; i < depth; i++) { - size_t prev_offset = *offset; - ts->current_stack[i] = decode_varint_u32(data, offset, max_size); - if (*offset == prev_offset) { - return -1; + if (reader->frame_info_cache == NULL) { + size_t cache_size = Py_MIN(reader->frames_count, + (uint32_t)FRAME_INFO_CACHE_SIZE); + reader->frame_info_cache = PyMem_Calloc( + cache_size, sizeof(*reader->frame_info_cache)); + if (reader->frame_info_cache == NULL) { + PyErr_NoMemory(); + return NULL; } + reader->frame_info_cache_size = cache_size; } - return 0; -} - -/* Decode a suffix-encoded stack from sample data. - * The suffix encoding shares frames from the bottom of the previous stack. - * Returns 0 on success, -1 on error (bounds violation). */ -static inline int -decode_stack_suffix(ReaderThreadState *ts, const uint8_t *data, - size_t *offset, size_t max_size) -{ - size_t prev_offset = *offset; - uint32_t shared = decode_varint_u32(data, offset, max_size); - if (*offset == prev_offset) { - return -1; - } - prev_offset = *offset; - uint32_t new_count = decode_varint_u32(data, offset, max_size); - if (*offset == prev_offset) { - return -1; - } - - /* Validate shared doesn't exceed current stack depth */ - if (shared > ts->current_stack_depth) { - PyErr_Format(PyExc_ValueError, - "Shared count %u exceeds current stack depth %zu", - shared, ts->current_stack_depth); - return -1; + ReaderFrameCacheEntry *entry = &reader->frame_info_cache[ + frame_index % reader->frame_info_cache_size]; + if (entry->frame_info != NULL && entry->frame_index == frame_index) { + return Py_NewRef(entry->frame_info); } - /* Use uint64_t to prevent overflow on 32-bit platforms */ - uint64_t final_depth = (uint64_t)shared + new_count; - if (validate_stack_depth(ts, final_depth) < 0) { - return -1; + FrameEntry *frame = &reader->frames[frame_index]; + if (frame->filename_idx >= reader->strings_count || + frame->funcname_idx >= reader->strings_count) { + PyErr_SetString(PyExc_ValueError, "Invalid string index in frame"); + return NULL; } - /* Move shared frames (from bottom of stack) to make room for new frames at the top */ - if (new_count > 0 && shared > 0) { - /* Defensive check: ensure subtraction won't underflow. - * This should already be guaranteed by the check above, but we add - * this assertion as defense-in-depth against stack corruption. */ - if (ts->current_stack_depth < shared) { - PyErr_SetString(PyExc_ValueError, - "Internal error: stack corruption detected in suffix decoding"); - return -1; - } - size_t prev_shared_start = ts->current_stack_depth - shared; - memmove(&ts->current_stack[new_count], - &ts->current_stack[prev_shared_start], - shared * sizeof(uint32_t)); + PyObject *frame_info = PyStructSequence_New(state->FrameInfo_Type); + if (!frame_info) { + return NULL; } - for (uint32_t i = 0; i < new_count; i++) { - size_t prev_offset = *offset; - ts->current_stack[i] = decode_varint_u32(data, offset, max_size); - if (*offset == prev_offset) { - return -1; + /* Build location tuple with full position info */ + PyObject *location; + if (frame->lineno != LOCATION_NOT_AVAILABLE) { + location = Py_BuildValue("(iiii)", + frame->lineno, + frame->end_lineno, + frame->column, + frame->end_column); + if (!location) { + Py_DECREF(frame_info); + return NULL; } } - ts->current_stack_depth = (size_t)final_depth; - return 0; -} - -/* Decode a pop-push encoded stack from sample data. - * Pops frames from the top and pushes new frames. - * Returns 0 on success, -1 on error (bounds violation). */ -static inline int -decode_stack_pop_push(ReaderThreadState *ts, const uint8_t *data, - size_t *offset, size_t max_size) -{ - size_t prev_offset = *offset; - uint32_t pop = decode_varint_u32(data, offset, max_size); - if (*offset == prev_offset) { - return -1; - } - prev_offset = *offset; - uint32_t push = decode_varint_u32(data, offset, max_size); - if (*offset == prev_offset) { - return -1; + else { + location = Py_NewRef(Py_None); } - size_t keep = (ts->current_stack_depth > pop) ? ts->current_stack_depth - pop : 0; - /* Use uint64_t to prevent overflow on 32-bit platforms */ - uint64_t final_depth = (uint64_t)keep + push; - if (validate_stack_depth(ts, final_depth) < 0) { - return -1; + /* Build opcode object */ + PyObject *opcode_obj; + if (frame->opcode != OPCODE_NONE) { + opcode_obj = PyLong_FromLong(frame->opcode); + if (!opcode_obj) { + Py_DECREF(location); + Py_DECREF(frame_info); + return NULL; + } } - - /* Move kept frames (from bottom of stack) to make room for new frames at the top. - * Even when push == 0, we need to move kept frames to index 0 if pop > 0. */ - if (keep > 0) { - memmove(&ts->current_stack[push], - &ts->current_stack[pop], - keep * sizeof(uint32_t)); + else { + opcode_obj = Py_NewRef(Py_None); } - for (uint32_t i = 0; i < push; i++) { - size_t prev_offset = *offset; - ts->current_stack[i] = decode_varint_u32(data, offset, max_size); - /* If offset didn't advance, varint decoding failed */ - if (*offset == prev_offset) { - return -1; - } - } - ts->current_stack_depth = (size_t)final_depth; - return 0; + PyStructSequence_SetItem(frame_info, 0, Py_NewRef(reader->strings[frame->filename_idx])); + PyStructSequence_SetItem(frame_info, 1, location); + PyStructSequence_SetItem(frame_info, 2, Py_NewRef(reader->strings[frame->funcname_idx])); + PyStructSequence_SetItem(frame_info, 3, opcode_obj); + Py_XSETREF(entry->frame_info, Py_NewRef(frame_info)); + entry->frame_index = frame_index; + return frame_info; } -/* Build a Python list of FrameInfo objects from frame indices */ +/* Build a Python list of FrameInfo objects from frame indices. Stacks are + * kept outermost first, but collectors expect innermost first. */ static PyObject * build_frame_list(RemoteDebuggingState *state, BinaryReader *reader, const uint32_t *frame_indices, size_t stack_depth) @@ -931,60 +786,11 @@ build_frame_list(RemoteDebuggingState *state, BinaryReader *reader, } for (size_t k = 0; k < stack_depth; k++) { - uint32_t frame_idx = frame_indices[k]; - if (frame_idx >= reader->frames_count) { - PyErr_Format(PyExc_ValueError, "Invalid frame index: %u", frame_idx); - goto error; - } - - FrameEntry *frame = &reader->frames[frame_idx]; - - if (frame->filename_idx >= reader->strings_count || - frame->funcname_idx >= reader->strings_count) { - PyErr_SetString(PyExc_ValueError, "Invalid string index in frame"); + PyObject *frame_info = reader_get_frame_info( + state, reader, frame_indices[stack_depth - 1 - k]); + if (frame_info == NULL) { goto error; } - - PyObject *frame_info = PyStructSequence_New(state->FrameInfo_Type); - if (!frame_info) { - goto error; - } - - /* Build location tuple with full position info */ - PyObject *location; - if (frame->lineno != LOCATION_NOT_AVAILABLE) { - location = Py_BuildValue("(iiii)", - frame->lineno, - frame->end_lineno, - frame->column, - frame->end_column); - if (!location) { - Py_DECREF(frame_info); - goto error; - } - } - else { - location = Py_NewRef(Py_None); - } - - /* Build opcode object */ - PyObject *opcode_obj; - if (frame->opcode != OPCODE_NONE) { - opcode_obj = PyLong_FromLong(frame->opcode); - if (!opcode_obj) { - Py_DECREF(location); - Py_DECREF(frame_info); - goto error; - } - } - else { - opcode_obj = Py_NewRef(Py_None); - } - - PyStructSequence_SetItem(frame_info, 0, Py_NewRef(reader->strings[frame->filename_idx])); - PyStructSequence_SetItem(frame_info, 1, location); - PyStructSequence_SetItem(frame_info, 2, Py_NewRef(reader->strings[frame->funcname_idx])); - PyStructSequence_SetItem(frame_info, 3, opcode_obj); PyList_SET_ITEM(frame_list, k, frame_info); } @@ -1101,252 +907,754 @@ invoke_progress_callback(PyObject *callback, Py_ssize_t current, uint64_t total) return 0; } -Py_ssize_t -binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progress_callback) +/* ============================================================================ + * SAMPLE BLOCKS + * ============================================================================ */ + +static int +block_error(const char *message) { - PyObject *collect_method; - int has_collect = PyObject_GetOptionalAttrString(collector, "collect", &collect_method); - if (has_collect < 0) { - return -1; + PyErr_Format(PyExc_ValueError, "Invalid sample block: %s", message); + return -1; +} + +static inline int +block_read_varint(const ByteBuffer *column, size_t *offset, uint64_t *value) +{ + size_t prev_offset = *offset; + *value = decode_varint_u64(column->data, offset, column->size); + return *offset == prev_offset ? -1 : 0; +} + +static int +block_read_varint_u32(const ByteBuffer *column, size_t *offset, + uint32_t *value) +{ + size_t prev_offset = *offset; + *value = decode_varint_u32(column->data, offset, column->size); + return *offset == prev_offset ? -1 : 0; +} + +static int +reader_parse_block_header(BinaryReader *reader, const ByteBuffer *section, + size_t *offset, BlockHeader *header) +{ + /* Block magic "TBLK" */ + if (section->size - *offset < BLOCK_MAGIC_SIZE) { + return block_error("truncated header"); } - if (has_collect == 0) { - PyErr_SetString(PyExc_TypeError, "Collector must have a collect() method"); - return -1; + if (memcmp(section->data + *offset, BLOCK_MAGIC, BLOCK_MAGIC_SIZE) != 0) { + return block_error("wrong magic"); } - Py_DECREF(collect_method); + *offset += BLOCK_MAGIC_SIZE; - /* Get module state for struct sequence types */ - PyObject *module = PyImport_ImportModule("_remote_debugging"); - if (!module) { + /* Sample count and base timestamp (varints) */ + if (block_read_varint_u32(section, offset, &header->sample_count) < 0 || + block_read_varint(section, offset, &header->base_timestamp) < 0) { return -1; } - RemoteDebuggingState *state = RemoteDebugging_GetState(module); - Py_DECREF(module); - if (!state) { - PyErr_SetString(PyExc_RuntimeError, "Failed to get module state"); - return -1; + /* Frame ID width and the two status flag masks (1 byte each) */ + if (section->size - *offset < 3) { + return block_error("truncated header"); } - - size_t offset = 0; - Py_ssize_t replayed = 0; - - /* Initial progress callback at 0% */ - if (invoke_progress_callback(progress_callback, 0, reader->sample_count) < 0) { - return -1; + header->frame_width = section->data[(*offset)++]; + header->common_status_flags = section->data[(*offset)++]; + header->status_bitmap_flags = section->data[(*offset)++]; + if (header->sample_count == 0) { + return block_error("no samples"); + } + if (header->frame_width < 1 || header->frame_width > sizeof(uint32_t)) { + return block_error("frame ID width out of range"); + } + if (header->common_status_flags & header->status_bitmap_flags) { + return block_error("common and varying status flags overlap"); } - while (offset < reader->sample_data_size) { - /* Read thread_id (8 bytes) + interpreter_id (4 bytes) + encoding byte */ - if (reader->sample_data_size - offset < SAMPLE_HEADER_FIXED_SIZE) { - PyErr_Format(PyExc_ValueError, - "Truncated sample data: %zu trailing bytes", - reader->sample_data_size - offset); + size_t decoded_total = 0; + for (unsigned column = 0; column < COLUMN_COUNT; column++) { + ColumnDescriptor *descriptor = &header->columns[column]; + /* Decoded size (varint) and encoding (1 byte) */ + if (block_read_varint_u32(section, offset, + &descriptor->decoded_size) < 0) { return -1; } + if (*offset >= section->size) { + return block_error("truncated column descriptor"); + } + descriptor->encoding = section->data[(*offset)++]; + + /* Stored size: implied for raw and constant, a varint for zstd */ + switch (descriptor->encoding) { + case COMPRESSION_NONE: + descriptor->stored_size = descriptor->decoded_size; + break; + case COLUMN_CONSTANT: + descriptor->stored_size = 1; + break; + case COMPRESSION_ZSTD: + if (reader->compression_type != COMPRESSION_ZSTD) { + return block_error("zstd column in an uncompressed file"); + } + if (block_read_varint_u32(section, offset, + &descriptor->stored_size) < 0) { + return -1; + } + break; + default: + return block_error("unknown column encoding"); + } - /* Use memcpy to avoid strict aliasing violations, then byte-swap if needed */ - uint64_t thread_id_raw; - uint32_t interpreter_id_raw; - memcpy(&thread_id_raw, &reader->sample_data[offset], SMP_SIZE_THREAD_ID); - offset += SMP_SIZE_THREAD_ID; + if (descriptor->encoding != COMPRESSION_NONE && + descriptor->decoded_size == 0) { + return block_error("encoded column is empty"); + } + if (descriptor->decoded_size > MAX_BLOCK_SIZE - decoded_total) { + return block_error("decoded columns exceed the block limit"); + } + decoded_total += descriptor->decoded_size; + } - memcpy(&interpreter_id_raw, &reader->sample_data[offset], SMP_SIZE_INTERPRETER_ID); - offset += SMP_SIZE_INTERPRETER_ID; + /* Each sample needs at least one byte of stream index and residual */ + if (header->sample_count > + header->columns[COLUMN_SAMPLE_STREAM_IDS].decoded_size || + header->sample_count > + header->columns[COLUMN_TIMESTAMP_RESIDUALS].decoded_size) { + return block_error("more samples than stored stream indices"); + } + return 0; +} - uint64_t thread_id = SWAP64_IF(reader->needs_swap, thread_id_raw); - uint32_t interpreter_id = SWAP32_IF(reader->needs_swap, interpreter_id_raw); +/* A constant stream index or residual column that holds the same one-byte + * varint for every sample is read from its stored byte, not expanded. */ +static int +constant_varint_column(const DecodedBlock *block, unsigned column, + uint8_t value) +{ + const ColumnDescriptor *descriptor = &block->header.columns[column]; + return (column == COLUMN_SAMPLE_STREAM_IDS || + column == COLUMN_TIMESTAMP_RESIDUALS) && + descriptor->encoding == COLUMN_CONSTANT && + descriptor->decoded_size == block->header.sample_count && + value < 0x80; +} - /* Get or create thread state for reconstruction */ - ReaderThreadState *ts = reader_get_or_create_thread_state(reader, thread_id, interpreter_id); - if (!ts) { - return -1; +static int +decode_block_columns(BinaryReader *reader, const ByteBuffer *section, + size_t *offset, DecodedBlock *block) +{ + const ColumnDescriptor *descriptors = block->header.columns; + /* Find where each column is stored and how much of the shared buffer + * the expanded and decompressed columns need. The buffer is reused + * across blocks and reserved up front so decoded columns never move. */ + uint8_t *stored[COLUMN_COUNT]; + size_t buffered_total = 0; + for (unsigned column = 0; column < COLUMN_COUNT; column++) { + const ColumnDescriptor *descriptor = &descriptors[column]; + if (descriptor->stored_size > section->size - *offset) { + return block_error("column extends past the sample section"); + } + stored[column] = section->data + *offset; + *offset += descriptor->stored_size; + if (descriptor->encoding == COMPRESSION_ZSTD || + (descriptor->encoding == COLUMN_CONSTANT && + !constant_varint_column(block, column, stored[column][0]))) { + buffered_total += descriptor->decoded_size; } + } + ByteBuffer *buffer = &reader->block_data; + if (buffer_reserve(buffer, buffered_total) < 0) { + return -1; + } - /* Read encoding byte */ - uint8_t encoding = reader->sample_data[offset++]; - - switch (encoding) { - case STACK_REPEAT: { - /* RLE repeat: [count: varint] [delta: varint, status: 1]... */ - size_t prev_offset = offset; - uint32_t count = decode_varint_u32(reader->sample_data, &offset, reader->sample_data_size); - /* Detect varint decode failure */ - if (offset == prev_offset) { - PyErr_SetString(PyExc_ValueError, "Malformed varint for RLE count"); - return -1; + buffer->size = 0; + for (unsigned column = 0; column < COLUMN_COUNT; column++) { + const ColumnDescriptor *descriptor = &descriptors[column]; + uint8_t *stored_data = stored[column]; + ByteBuffer *decoded = &block->columns[column]; + decoded->size = descriptor->decoded_size; + switch (descriptor->encoding) { + case COMPRESSION_NONE: + /* Raw columns point into the file data */ + decoded->data = stored_data; + break; + case COLUMN_CONSTANT: + if (constant_varint_column(block, column, stored_data[0])) { + decoded->data = stored_data; + decoded->size = 1; + break; } - - /* Reject a count larger than the remaining bytes can hold; each - * RLE sample needs at least 2 bytes (1-byte min varint + status). */ - if (reader_validate_count("RLE", count, - reader->sample_data_size - offset, 2) < 0) { + decoded->data = buffer->data + buffer->size; + memset(decoded->data, stored_data[0], decoded->size); + buffer->size += decoded->size; + break; + case COMPRESSION_ZSTD: + decoded->data = buffer->data + buffer->size; + if (reader_decompress(reader, stored_data, + descriptor->stored_size, buffer, + decoded->size) < 0) { return -1; } - if ((uint64_t)count > (uint64_t)PY_SSIZE_T_MAX - (uint64_t)replayed) { - PyErr_SetString(PyExc_OverflowError, - "Sample count exceeds Py_ssize_t maximum"); + break; + default: + return block_error("unknown column encoding"); + } + } + return 0; +} + +static int +validate_block_columns(DecodedBlock *block) +{ + size_t stream_table_size = block->columns[COLUMN_STREAM_TABLE].size; + if (stream_table_size % STREAM_ENTRY_SIZE != 0) { + return block_error("stream table has a partial entry"); + } + size_t stream_count = stream_table_size / STREAM_ENTRY_SIZE; + if (stream_count == 0 || stream_count > MAX_BLOCK_STREAMS) { + return block_error("stream count out of range"); + } + block->stream_count = (uint32_t)stream_count; + + size_t frame_ids_size = block->columns[COLUMN_FRAME_IDS].size; + if (frame_ids_size % block->header.frame_width != 0) { + return block_error("frame ID column has a partial value"); + } + block->frame_count = + (uint32_t)(frame_ids_size / block->header.frame_width); + + size_t plane_size = status_bitmap_size(block->header.sample_count); + size_t plane_count = + (size_t)_Py_popcount32(block->header.status_bitmap_flags); + const ByteBuffer *statuses = &block->columns[COLUMN_STATUS_BITMAPS]; + if (statuses->size != plane_size * plane_count) { + return block_error("status bitmaps have the wrong size"); + } + unsigned used_bits = block->header.sample_count % CHAR_BIT; + for (size_t plane = 0; used_bits && plane < plane_count; plane++) { + if (statuses->data[(plane + 1) * plane_size - 1] >> used_bits) { + return block_error("status bitmap padding is not zero"); + } + } + return 0; +} + +static int +reserve_block_streams(BinaryReader *reader, uint32_t stream_count) +{ + if (stream_count <= reader->block_stream_capacity) { + return 0; + } + ReaderBlockStream *streams = PyMem_Realloc( + reader->block_streams, stream_count * sizeof(ReaderBlockStream)); + if (streams == NULL) { + PyErr_NoMemory(); + return -1; + } + reader->block_streams = streams; + reader->block_stream_capacity = stream_count; + return 0; +} + +static int +reader_parse_stream_table(BinaryReader *reader, const DecodedBlock *block) +{ + const uint8_t *stream_table = block->columns[COLUMN_STREAM_TABLE].data; + uint64_t initial_prev_timestamp = + block->header.base_timestamp - reader->sample_interval_us; + memset(reader->block_streams, 0, + block->stream_count * sizeof(ReaderBlockStream)); + for (uint32_t i = 0; i < block->stream_count; i++) { + /* Read thread_id (8 bytes) + interpreter_id (4 bytes) */ + const uint8_t *entry = stream_table + (size_t)i * STREAM_ENTRY_SIZE; + uint64_t thread_id; + uint32_t interpreter_id; + memcpy(&thread_id, entry + STM_OFF_THREAD_ID, STM_SIZE_THREAD_ID); + memcpy(&interpreter_id, entry + STM_OFF_INTERPRETER_ID, + STM_SIZE_INTERPRETER_ID); + ThreadEntry *thread = reader_get_or_create_thread_state( + reader, SWAP64_IF(reader->needs_swap, thread_id), + SWAP32_IF(reader->needs_swap, interpreter_id)); + if (thread == NULL) { + return -1; + } + /* Each thread may appear only once in a block. An index left over + * from an earlier block points at a different thread here. */ + uint32_t previous_index = thread->block_stream_index; + if (previous_index < i && + reader->block_streams[previous_index].thread_id == + thread->thread_id && + reader->block_streams[previous_index].interpreter_id == + thread->interpreter_id) { + return block_error("stream table repeats a thread"); + } + thread->block_stream_index = i; + + /* Each stream's first expected timestamp is the base timestamp */ + ReaderBlockStream *stream = &reader->block_streams[i]; + stream->thread_id = thread->thread_id; + stream->interpreter_id = thread->interpreter_id; + stream->prev_timestamp = initial_prev_timestamp; + } + return 0; +} + +static int +decode_samples(BinaryReader *reader, DecodedBlock *block) +{ + size_t sample_count = block->header.sample_count; + if (buffer_reserve(&reader->block_stream_ids, + sample_count * sizeof(uint32_t)) < 0 || + buffer_reserve(&reader->block_timestamps, + sample_count * sizeof(uint64_t)) < 0) { + return -1; + } + uint32_t *stream_ids = (uint32_t *)reader->block_stream_ids.data; + uint64_t *timestamps = (uint64_t *)reader->block_timestamps.data; + const ByteBuffer *stream_id_column = + &block->columns[COLUMN_SAMPLE_STREAM_IDS]; + const ByteBuffer *residual_column = + &block->columns[COLUMN_TIMESTAMP_RESIDUALS]; + int constant_stream_ids = constant_varint_column( + block, COLUMN_SAMPLE_STREAM_IDS, stream_id_column->data[0]); + int constant_residuals = constant_varint_column( + block, COLUMN_TIMESTAMP_RESIDUALS, residual_column->data[0]); + size_t stream_id_offset = 0; + size_t residual_offset = 0; + int64_t constant_residual = 0; + if (constant_residuals) { + constant_residual = decode_varint_i64(residual_column->data, + &residual_offset, 1); + } + for (uint32_t sample = 0; sample < block->header.sample_count; sample++) { + uint64_t stream_id; + if (constant_stream_ids) { + stream_id = stream_id_column->data[0]; + } + else if (block_read_varint(stream_id_column, &stream_id_offset, + &stream_id) < 0) { + return -1; + } + if (stream_id >= block->stream_count) { + return block_error("sample refers to an unknown stream"); + } + int64_t residual = constant_residual; + if (!constant_residuals) { + size_t prev_offset = residual_offset; + residual = decode_varint_i64(residual_column->data, + &residual_offset, + residual_column->size); + if (residual_offset == prev_offset) { return -1; } + } - reader->stats.repeat_records++; - reader->stats.repeat_samples += count; - - /* Process RLE samples, batching by status */ - PyObject *timestamps_list = NULL; - uint8_t batch_status = 0; - Py_ssize_t batch_idx = 0; - - for (uint32_t i = 0; i < count; i++) { - size_t delta_prev_offset = offset; - uint64_t delta = decode_varint_u64(reader->sample_data, &offset, reader->sample_data_size); - if (offset == delta_prev_offset) { - Py_XDECREF(timestamps_list); - PyErr_SetString(PyExc_ValueError, "Malformed varint in RLE sample data"); - return -1; - } - if (offset >= reader->sample_data_size) { - Py_XDECREF(timestamps_list); - PyErr_SetString(PyExc_ValueError, "Unexpected end of sample data in RLE"); - return -1; - } - uint8_t status = reader->sample_data[offset++]; - ts->prev_timestamp += delta; - - /* Start new batch on first sample or status change */ - if (i == 0 || status != batch_status - || batch_idx >= MAX_RLE_BATCH_SAMPLES) { - if (timestamps_list) { - int rc = emit_sample(state, collector, thread_id, interpreter_id, - batch_status, ts->current_stack, ts->current_stack_depth, - reader, timestamps_list); - Py_DECREF(timestamps_list); - if (rc < 0) { - return -1; - } - } - timestamps_list = PyList_New(0); - if (!timestamps_list) { - return -1; - } - batch_status = status; - batch_idx = 0; - } - - PyObject *ts_obj = PyLong_FromUnsignedLongLong(ts->prev_timestamp); - if (!ts_obj) { - Py_DECREF(timestamps_list); - return -1; - } - int append_rc = PyList_Append(timestamps_list, ts_obj); - Py_DECREF(ts_obj); - if (append_rc < 0) { - Py_DECREF(timestamps_list); - return -1; - } - batch_idx++; - } + /* Timestamp is the expected time plus the stored residual */ + ReaderBlockStream *stream = &reader->block_streams[stream_id]; + stream->prev_timestamp += + reader->sample_interval_us + (uint64_t)residual; - /* Emit final batch */ - if (timestamps_list) { - int rc = emit_sample(state, collector, thread_id, interpreter_id, - batch_status, ts->current_stack, ts->current_stack_depth, - reader, timestamps_list); - Py_DECREF(timestamps_list); - if (rc < 0) { - return -1; - } - } + stream_ids[sample] = (uint32_t)stream_id; + timestamps[sample] = stream->prev_timestamp; + } + if (!constant_stream_ids && stream_id_offset != stream_id_column->size) { + return block_error("stream index column has trailing data"); + } + if (residual_offset != residual_column->size) { + return block_error("timestamp column has trailing data"); + } + block->stream_ids = stream_ids; + block->timestamps = timestamps; + return 0; +} - replayed += count; - reader->stats.total_samples += count; +static int +decode_statuses(BinaryReader *reader, DecodedBlock *block) +{ + if (buffer_reserve(&reader->block_statuses, + block->header.sample_count) < 0) { + return -1; + } + /* Start from the common flags, then add each varying flag's bitmap */ + uint8_t *statuses = reader->block_statuses.data; + memset(statuses, block->header.common_status_flags, + block->header.sample_count); + const uint8_t *plane = block->columns[COLUMN_STATUS_BITMAPS].data; + size_t plane_size = status_bitmap_size(block->header.sample_count); + for (unsigned bit = 0; bit < CHAR_BIT; bit++) { + if (!(block->header.status_bitmap_flags & (1U << bit))) { + continue; + } + for (uint32_t i = 0; i < block->header.sample_count; i++) { + statuses[i] |= ((plane[i / CHAR_BIT] >> (i % CHAR_BIT)) & 1U) + << bit; + } + plane += plane_size; + } + block->statuses = statuses; + return 0; +} - /* Progress callback after batch */ - if ((uint64_t)replayed % PROGRESS_CALLBACK_INTERVAL < (uint64_t)count) { - if (invoke_progress_callback(progress_callback, replayed, reader->sample_count) < 0) { - return -1; - } - } - break; +static int +decode_frame_ids(BinaryReader *reader, DecodedBlock *block) +{ + size_t frame_count = block->frame_count; + size_t frames_size = frame_count * sizeof(uint32_t); + if (buffer_reserve(&reader->block_frames, frames_size) < 0) { + return -1; + } + uint32_t *frames = (uint32_t *)reader->block_frames.data; + block->frames = frames; + if (frame_count == 0) { + return 0; + } + + /* Combine byte b of every frame index from plane b, low byte first */ + memset(frames, 0, frames_size); + const uint8_t *plane = block->columns[COLUMN_FRAME_IDS].data; + for (unsigned byte = 0; byte < block->header.frame_width; byte++) { + for (size_t i = 0; i < frame_count; i++) { + frames[i] |= (uint32_t)plane[i] << (byte * CHAR_BIT); + } + plane += frame_count; + } + for (size_t i = 0; i < frame_count; i++) { + if (frames[i] >= reader->frames_count) { + return block_error("frame index out of range"); } + } + return 0; +} - case STACK_FULL: - case STACK_SUFFIX: - case STACK_POP_PUSH: { - /* All three encodings share: [delta: varint] [status: 1] ... */ - size_t prev_offset = offset; - uint64_t delta = decode_varint_u64(reader->sample_data, &offset, reader->sample_data_size); - /* Detect varint decode failure: offset unchanged means error */ - if (offset == prev_offset) { - PyErr_SetString(PyExc_ValueError, "Malformed varint in sample data"); - return -1; - } - if (offset >= reader->sample_data_size) { - PyErr_SetString(PyExc_ValueError, "Unexpected end of sample data"); - return -1; - } - uint8_t status = reader->sample_data[offset++]; - ts->prev_timestamp += delta; - - if (encoding == STACK_FULL) { - if (decode_stack_full(ts, reader->sample_data, &offset, reader->sample_data_size) < 0) { - return -1; - } - reader->stats.full_records++; - } else if (encoding == STACK_SUFFIX) { - if (decode_stack_suffix(ts, reader->sample_data, &offset, reader->sample_data_size) < 0) { - return -1; - } - reader->stats.suffix_records++; - } else { /* STACK_POP_PUSH */ - if (decode_stack_pop_push(ts, reader->sample_data, &offset, reader->sample_data_size) < 0) { - return -1; - } - reader->stats.pop_push_records++; - } - reader->stats.stack_reconstructions++; +static int +decode_block(BinaryReader *reader, const ByteBuffer *section, + size_t *offset, DecodedBlock *block) +{ + block->instruction_offset = 0; + block->frame_offset = 0; + if (reader_parse_block_header(reader, section, offset, + &block->header) < 0 || + decode_block_columns(reader, section, offset, block) < 0 || + validate_block_columns(block) < 0 || + reserve_block_streams(reader, block->stream_count) < 0 || + reader_parse_stream_table(reader, block) < 0 || + decode_samples(reader, block) < 0 || + decode_statuses(reader, block) < 0 || + decode_frame_ids(reader, block) < 0) { + return -1; + } + return 0; +} - /* Build single-element timestamp list */ - PyObject *ts_obj = PyLong_FromUnsignedLongLong(ts->prev_timestamp); - if (!ts_obj) { - return -1; - } - PyObject *timestamps_list = PyList_New(1); - if (!timestamps_list) { - Py_DECREF(ts_obj); - return -1; - } - PyList_SET_ITEM(timestamps_list, 0, ts_obj); +static int +flush_stream_batch(BinaryReader *reader, RemoteDebuggingState *state, + PyObject *collector, ReaderBlockStream *stream) +{ + if (stream->batch == NULL) { + return 0; + } + int result = emit_sample(state, collector, stream->thread_id, + stream->interpreter_id, stream->batch_status, + stream->current_stack, + stream->current_stack_depth, reader, + stream->batch); + Py_CLEAR(stream->batch); + return result; +} - if (emit_sample(state, collector, thread_id, interpreter_id, status, - ts->current_stack, ts->current_stack_depth, reader, - timestamps_list) < 0) { - Py_DECREF(timestamps_list); - return -1; - } - Py_DECREF(timestamps_list); - if (replayed == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "Sample count exceeds Py_ssize_t maximum"); - return -1; - } - replayed++; - reader->stats.total_samples++; +static int +decode_stack_repeat(BinaryReader *reader, DecodedBlock *block, + ReaderBlockStream *stream) +{ + const ByteBuffer *instructions = + &block->columns[COLUMN_STACK_INSTRUCTIONS]; + size_t *offset = &block->instruction_offset; + uint64_t count; + if (block_read_varint(instructions, offset, &count) < 0) { + return -1; + } + if (!stream->has_stack) { + return block_error("first stack of a stream cannot repeat"); + } + if (count == 0 || count > block->header.sample_count) { + return block_error("repeat count out of range"); + } + stream->pending_repeats = (uint32_t)(count - 1); + reader->stats.repeat_records++; + reader->stats.repeat_samples++; + return 0; +} + +static int +decode_stack_replace_suffix(BinaryReader *reader, DecodedBlock *block, + ReaderBlockStream *stream) +{ + const ByteBuffer *instructions = + &block->columns[COLUMN_STACK_INSTRUCTIONS]; + size_t *offset = &block->instruction_offset; + uint64_t keep, count; + if (block_read_varint(instructions, offset, &keep) < 0 || + block_read_varint(instructions, offset, &count) < 0) { + return -1; + } + if (keep > stream->current_stack_depth || count > MAX_STACK_DEPTH - keep) { + return block_error("stack prefix out of range"); + } + if (count > block->frame_count - block->frame_offset) { + return block_error("stack uses more frame IDs than stored"); + } + if (count > 0) { + memcpy(&stream->current_stack[keep], + &block->frames[block->frame_offset], count * sizeof(uint32_t)); + } + block->frame_offset += (uint32_t)count; + stream->current_stack_depth = (size_t)(keep + count); + if (keep == 0) { + reader->stats.full_records++; + } + else { + reader->stats.suffix_records++; + } + return 0; +} + +static int +decode_stack_reuse(BinaryReader *reader, DecodedBlock *block, + ReaderBlockStream *stream) +{ + const ByteBuffer *instructions = + &block->columns[COLUMN_STACK_INSTRUCTIONS]; + size_t *offset = &block->instruction_offset; + uint64_t start, count; + if (block_read_varint(instructions, offset, &start) < 0 || + block_read_varint(instructions, offset, &count) < 0) { + return -1; + } + if (count > MAX_STACK_DEPTH) { + return block_error("stack is too deep"); + } + if (start > block->frame_offset || count > block->frame_offset - start) { + return block_error("reused stack is not stored earlier"); + } + if (count > 0) { + memcpy(stream->current_stack, &block->frames[start], + count * sizeof(uint32_t)); + } + stream->current_stack_depth = (size_t)count; + reader->stats.reuse_records++; + return 0; +} + +/* Return 1 if a new stack was decoded, 0 if the stack repeats, -1 on error. */ +static int +decode_stack(BinaryReader *reader, RemoteDebuggingState *state, + PyObject *collector, DecodedBlock *block, + ReaderBlockStream *stream) +{ + /* An earlier REPEAT still covers this sample */ + if (stream->pending_repeats > 0) { + stream->pending_repeats--; + reader->stats.repeat_samples++; + return 0; + } + uint64_t encoding; + if (block_read_varint(&block->columns[COLUMN_STACK_INSTRUCTIONS], + &block->instruction_offset, &encoding) < 0) { + return -1; + } + if (encoding == STACK_REPEAT) { + return decode_stack_repeat(reader, block, stream); + } + + /* The pending batch was collected with the stack being replaced. */ + if (flush_stream_batch(reader, state, collector, stream) < 0) { + return -1; + } + int result; + switch (encoding) { + case STACK_REPLACE_SUFFIX: + result = decode_stack_replace_suffix(reader, block, stream); + break; + case STACK_REUSE: + result = decode_stack_reuse(reader, block, stream); + break; + default: + return block_error("unknown stack instruction"); + } + if (result < 0) { + return -1; + } + stream->has_stack = 1; + reader->stats.stack_reconstructions++; + return 1; +} + +static int +replay_sample(BinaryReader *reader, RemoteDebuggingState *state, + PyObject *collector, DecodedBlock *block, uint32_t sample) +{ + ReaderBlockStream *stream = + &reader->block_streams[block->stream_ids[sample]]; + uint8_t status = block->statuses[sample]; + int new_stack = decode_stack(reader, state, collector, block, stream); + if (new_stack < 0) { + return -1; + } + + /* Start a new batch when the status changes or the batch is full */ + if (stream->batch == NULL || status != stream->batch_status || + PyList_GET_SIZE(stream->batch) >= MAX_RLE_BATCH_SAMPLES) { + if (flush_stream_batch(reader, state, collector, stream) < 0) { + return -1; + } + stream->batch = PyList_New(0); + if (stream->batch == NULL) { + return -1; + } + stream->batch_status = status; + } + PyObject *timestamp = + PyLong_FromUnsignedLongLong(block->timestamps[sample]); + if (timestamp == NULL) { + return -1; + } + int result = PyList_Append(stream->batch, timestamp); + Py_DECREF(timestamp); + if (result < 0) { + return -1; + } + /* Send a new stack's sample immediately so a stream's first sample is + * not held back behind samples from other streams. */ + if (new_stack) { + return flush_stream_batch(reader, state, collector, stream); + } + return 0; +} + +static int +validate_block_consumed(BinaryReader *reader, const DecodedBlock *block) +{ + if (block->instruction_offset != + block->columns[COLUMN_STACK_INSTRUCTIONS].size) { + return block_error("stack instruction column has trailing data"); + } + if (block->frame_offset != block->frame_count) { + return block_error("frame ID column has unused values"); + } + for (uint32_t i = 0; i < block->stream_count; i++) { + if (!reader->block_streams[i].has_stack) { + return block_error("stream table has an unused entry"); + } + if (reader->block_streams[i].pending_repeats != 0) { + return block_error("repeat count extends past the block"); + } + } + return 0; +} + +static int +replay_block(BinaryReader *reader, RemoteDebuggingState *state, + PyObject *collector, PyObject *progress_callback, + DecodedBlock *block, Py_ssize_t *replayed) +{ + if ((uint64_t)block->header.sample_count > + reader->sample_count - (uint64_t)*replayed) { + return block_error("more samples than the file header declares"); + } + int result = 0; + for (uint32_t sample = 0; sample < block->header.sample_count; sample++) { + if (replay_sample(reader, state, collector, block, sample) < 0) { + result = -1; break; } + (*replayed)++; + reader->stats.total_samples++; + if (*replayed % PROGRESS_CALLBACK_INTERVAL == 0 && + invoke_progress_callback(progress_callback, *replayed, + reader->sample_count) < 0) { + result = -1; + break; + } + } + if (result == 0) { + result = validate_block_consumed(reader, block); + } - default: - PyErr_Format(PyExc_ValueError, "Unknown stack encoding: %u", encoding); - return -1; + /* Send the remaining batches, or drop them after an error. */ + for (uint32_t i = 0; i < block->stream_count; i++) { + ReaderBlockStream *stream = &reader->block_streams[i]; + if (result == 0 && + flush_stream_batch(reader, state, collector, stream) < 0) { + result = -1; } + Py_CLEAR(stream->batch); + } - /* Progress callback */ - if (replayed % PROGRESS_CALLBACK_INTERVAL == 0) { - if (invoke_progress_callback(progress_callback, replayed, reader->sample_count) < 0) { - return -1; - } + return result; +} + +static int +replay_blocks(BinaryReader *reader, RemoteDebuggingState *state, + PyObject *collector, PyObject *progress_callback, + Py_ssize_t *replayed) +{ + ByteBuffer section = {reader->sample_data, reader->sample_data_size, 0}; + size_t offset = 0; + while (offset < section.size) { + DecodedBlock block; + if (decode_block(reader, §ion, &offset, &block) < 0 || + replay_block(reader, state, collector, progress_callback, + &block, replayed) < 0) { + return -1; } } + return 0; +} + +static Py_ssize_t +replay_all_samples(BinaryReader *reader, PyObject *collector, + PyObject *progress_callback) +{ + PyObject *collect_method; + int has_collect = PyObject_GetOptionalAttrString(collector, "collect", &collect_method); + if (has_collect < 0) { + return -1; + } + if (has_collect == 0) { + PyErr_SetString(PyExc_TypeError, "Collector must have a collect() method"); + return -1; + } + Py_DECREF(collect_method); + + /* Get module state for struct sequence types */ + PyObject *module = PyImport_ImportModule("_remote_debugging"); + if (!module) { + return -1; + } + RemoteDebuggingState *state = RemoteDebugging_GetState(module); + Py_DECREF(module); + + if (!state) { + PyErr_SetString(PyExc_RuntimeError, "Failed to get module state"); + return -1; + } + if (reader->sample_count > (uint64_t)PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "Sample count exceeds Py_ssize_t maximum"); + return -1; + } + + /* Initial progress callback at 0% */ + if (invoke_progress_callback(progress_callback, 0, reader->sample_count) < 0) { + return -1; + } + + memset(&reader->stats, 0, sizeof(reader->stats)); + Py_ssize_t replayed = 0; + if (replay_blocks(reader, state, collector, progress_callback, + &replayed) < 0) { + return -1; + } if ((uint64_t)replayed != reader->sample_count) { PyErr_Format(PyExc_ValueError, @@ -1354,8 +1662,25 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre (unsigned long long)reader->sample_count, replayed); return -1; } + return replayed; +} - /* Final progress callback at 100% */ +Py_ssize_t +binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progress_callback) +{ + if (reader->replaying) { + PyErr_SetString(PyExc_RuntimeError, "Reader is already replaying"); + return -1; + } + reader->replaying = 1; + Py_ssize_t replayed = replay_all_samples(reader, collector, + progress_callback); + reader->replaying = 0; + if (replayed < 0) { + return -1; + } + + /* Final progress callback at 100%; it may close the reader */ if (invoke_progress_callback(progress_callback, replayed, reader->sample_count) < 0) { return -1; } @@ -1453,8 +1778,9 @@ binary_writer_get_stats(BinaryWriter *writer) /* Calculate derived stats */ uint64_t total_records = s->repeat_records + s->full_records + - s->suffix_records + s->pop_push_records; + s->suffix_records + s->reuse_records; uint64_t total_samples = writer->total_samples; + uint64_t bytes_written = s->bytes_written + binary_writer_buffered_size(writer); uint64_t potential_frames = s->total_frames_written + s->frames_saved; double compression_ratio = (potential_frames > 0) ? (double)s->frames_saved / potential_frames * 100.0 : 0.0; @@ -1465,12 +1791,12 @@ binary_writer_get_stats(BinaryWriter *writer) "repeat_samples", s->repeat_samples, "full_records", s->full_records, "suffix_records", s->suffix_records, - "pop_push_records", s->pop_push_records, + "reuse_records", s->reuse_records, "total_records", total_records, "total_samples", total_samples, "total_frames_written", s->total_frames_written, "frames_saved", s->frames_saved, - "bytes_written", s->bytes_written, + "bytes_written", bytes_written, "frame_compression_pct", compression_ratio ); } @@ -1481,7 +1807,7 @@ binary_reader_get_stats(BinaryReader *reader) BinaryReaderStats *s = &reader->stats; uint64_t total_records = s->repeat_records + s->full_records + - s->suffix_records + s->pop_push_records; + s->suffix_records + s->reuse_records; return Py_BuildValue( "{s:K, s:K, s:K, s:K, s:K, s:K, s:K, s:K}", @@ -1489,7 +1815,7 @@ binary_reader_get_stats(BinaryReader *reader) "repeat_samples", s->repeat_samples, "full_records", s->full_records, "suffix_records", s->suffix_records, - "pop_push_records", s->pop_push_records, + "reuse_records", s->reuse_records, "total_records", total_records, "total_samples", s->total_samples, "stack_reconstructions", s->stack_reconstructions @@ -1524,7 +1850,22 @@ binary_reader_close(BinaryReader *reader) } #endif - PyMem_Free(reader->decompressed_data); +#ifdef HAVE_ZSTD + ZSTD_freeDCtx(reader->dctx); +#endif + buffer_clear(&reader->block_data); + buffer_clear(&reader->block_stream_ids); + buffer_clear(&reader->block_timestamps); + buffer_clear(&reader->block_statuses); + buffer_clear(&reader->block_frames); + PyMem_Free(reader->block_streams); + + if (reader->frame_info_cache != NULL) { + for (size_t i = 0; i < reader->frame_info_cache_size; i++) { + Py_XDECREF(reader->frame_info_cache[i].frame_info); + } + PyMem_Free(reader->frame_info_cache); + } if (reader->strings) { for (uint32_t i = 0; i < reader->strings_count; i++) { @@ -1535,12 +1876,7 @@ binary_reader_close(BinaryReader *reader) PyMem_Free(reader->frames); - if (reader->thread_states) { - for (size_t i = 0; i < reader->thread_state_count; i++) { - PyMem_Free(reader->thread_states[i].current_stack); - } - PyMem_Free(reader->thread_states); - } + PyMem_Free(reader->thread_states); PyMem_Free(reader); } diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index 1794017053da503..01331c3722484f7 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -2,7 +2,7 @@ * Python Remote Debugging Module - Binary Writer Implementation * * High-performance binary file writer for profiling data with optional zstd - * streaming compression. + * compression. ******************************************************************************/ #ifndef Py_BUILD_CORE_MODULE @@ -11,6 +11,7 @@ #include "binary_io.h" #include "_remote_debugging.h" +#include "pycore_bitutils.h" // _Py_bit_length(), _Py_popcount32() #include "pycore_opcode_utils.h" // MAX_REAL_OPCODE #include #include @@ -27,16 +28,69 @@ * CONSTANTS FOR BINARY FORMAT SIZES * ============================================================================ */ -/* Sample header sizes */ -#define SAMPLE_HEADER_MAX_SIZE 26 /* fixed + max_varint(10) + status(1) + margin */ #define MAX_VARINT_SIZE 10 /* Maximum bytes for a varint64 */ #define MAX_VARINT_SIZE_U32 5 /* Maximum bytes for a varint32 */ -/* Frame buffer: depth varint (max 2 bytes for 256) + 256 frames * 5 bytes/varint + margin */ -#define MAX_FRAME_BUFFER_SIZE ((MAX_STACK_DEPTH * MAX_VARINT_SIZE_U32) + MAX_VARINT_SIZE_U32 + 16) -/* RLE pending buffer (per thread): one entry is a u64 delta varint + status byte */ -#define MAX_RLE_BUF_SIZE (16 * 1024) -#define MAX_RLE_ENTRY_SIZE (MAX_VARINT_SIZE + sizeof(uint8_t)) +#define MAX_BLOCK_HEADER_SIZE \ + (BLOCK_MAGIC_SIZE + MAX_VARINT_SIZE_U32 + MAX_VARINT_SIZE + \ + 3 * sizeof(uint8_t) + \ + COLUMN_COUNT * (2 * MAX_VARINT_SIZE_U32 + sizeof(uint8_t))) + +#define ZSTD_MIN_SIZE 32 +#define STACK_REUSE_MIN_FRAMES 3 + +#define MAX_SAMPLE_SIZE \ + (STREAM_ENTRY_SIZE + 5 * MAX_VARINT_SIZE + sizeof(uint8_t) + \ + MAX_STACK_DEPTH * sizeof(uint32_t) + sizeof(Repeat)) +#define MAX_BITMAP_PADDING (CHAR_BIT - 1) +#define BLOCK_FLUSH_SIZE \ + (MAX_BLOCK_SIZE - MAX_SAMPLE_SIZE - MAX_BITMAP_PADDING) + +static int +buffer_append(ByteBuffer *buffer, const void *data, size_t size) +{ + if (buffer->size > MAX_BLOCK_SIZE || + size > MAX_BLOCK_SIZE - buffer->size) { + PyErr_SetString(PyExc_RuntimeError, + "sample block buffer exceeds the writer safety limit"); + return -1; + } + if (buffer_reserve(buffer, buffer->size + size) < 0) { + return -1; + } + if (size != 0) { + memcpy(buffer->data + buffer->size, data, size); + } + buffer->size += size; + return 0; +} + +static int +buffer_append_varint(ByteBuffer *buffer, uint64_t value) +{ + uint8_t bytes[MAX_VARINT_SIZE]; + size_t size = encode_varint_u64(bytes, value); + return buffer_append(buffer, bytes, size); +} + +static int +column_write(SampleBlock *block, unsigned column, const void *data, + size_t size) +{ + if (buffer_append(&block->columns[column], data, size) < 0) { + return -1; + } + block->size += size; + return 0; +} + +static int +column_write_varint(SampleBlock *block, unsigned column, uint64_t value) +{ + uint8_t bytes[MAX_VARINT_SIZE]; + size_t size = encode_varint_u64(bytes, value); + return column_write(block, column, bytes, size); +} /* Helper macro: convert PyLong to int32, using default_val if conversion fails */ #define PYLONG_TO_INT32_OR_DEFAULT(obj, var, default_val) \ @@ -131,19 +185,6 @@ fwrite_checked_allow_threads(const void *data, size_t size, FILE *fp) return 0; } -/* Forward declaration for writer_write_bytes */ -static inline int writer_write_bytes(BinaryWriter *writer, const void *data, size_t size); - -/* Encode and write a varint u32 - returns 0 on success, -1 on error */ -static inline int -writer_write_varint_u32(BinaryWriter *writer, uint32_t value) -{ - uint8_t buf[MAX_VARINT_SIZE]; - size_t len = encode_varint_u32(buf, value); - return writer_write_bytes(writer, buf, len); -} - - /* ============================================================================ * UTILITY FUNCTIONS * ============================================================================ */ @@ -182,9 +223,9 @@ writer_init_zstd(BinaryWriter *writer) return -1; } - /* Compression level 5: better ratio for repetitive profiling data */ + /* Compression level 3: good ratio without slowing down each block flush */ size_t result = ZSTD_CCtx_setParameter(writer->zstd.cctx, - ZSTD_c_compressionLevel, 5); + ZSTD_c_compressionLevel, 3); if (ZSTD_isError(result)) { PyErr_Format(PyExc_RuntimeError, "Failed to set zstd compression level: %s", ZSTD_getErrorName(result)); @@ -193,16 +234,6 @@ writer_init_zstd(BinaryWriter *writer) return -1; } - /* Use large buffer (512KB) for fewer I/O syscalls */ - writer->zstd.compressed_buffer = PyMem_Malloc(COMPRESSED_BUFFER_SIZE); - if (!writer->zstd.compressed_buffer) { - ZSTD_freeCCtx(writer->zstd.cctx); - writer->zstd.cctx = NULL; - PyErr_NoMemory(); - return -1; - } - writer->zstd.compressed_buffer_size = COMPRESSED_BUFFER_SIZE; - return 0; #else PyErr_SetString(PyExc_RuntimeError, @@ -211,79 +242,6 @@ writer_init_zstd(BinaryWriter *writer) #endif } -static int -writer_flush_buffer(BinaryWriter *writer) -{ - if (writer->buffer_pos == 0) { - return 0; - } - -#ifdef HAVE_ZSTD - if (writer->compression_type == COMPRESSION_ZSTD) { - ZSTD_inBuffer input = { writer->write_buffer, writer->buffer_pos, 0 }; - - while (input.pos < input.size) { - ZSTD_outBuffer output = { - writer->zstd.compressed_buffer, - writer->zstd.compressed_buffer_size, - 0 - }; - - size_t result = ZSTD_compressStream2( - writer->zstd.cctx, &output, &input, ZSTD_e_continue - ); - - if (ZSTD_isError(result)) { - PyErr_Format(PyExc_IOError, "zstd compression error: %s", - ZSTD_getErrorName(result)); - return -1; - } - - _Py_MSAN_UNPOISON(writer->zstd.compressed_buffer, output.pos); - if (output.pos > 0) { - if (fwrite_checked_allow_threads(writer->zstd.compressed_buffer, output.pos, writer->fp) < 0) { - return -1; - } - } - } - } else -#endif - { - if (fwrite_checked_allow_threads(writer->write_buffer, writer->buffer_pos, writer->fp) < 0) { - return -1; - } - } - - writer->buffer_pos = 0; - return 0; -} - -static inline int -writer_write_bytes(BinaryWriter *writer, const void *data, size_t size) -{ - const uint8_t *src = (const uint8_t *)data; - size_t original_size = size; - - while (size > 0) { - size_t space = writer->buffer_size - writer->buffer_pos; - size_t to_copy = (size < space) ? size : space; - - memcpy(writer->write_buffer + writer->buffer_pos, src, to_copy); - writer->buffer_pos += to_copy; - src += to_copy; - size -= to_copy; - - if (writer->buffer_pos == writer->buffer_size) { - if (writer_flush_buffer(writer) < 0) { - return -1; - } - } - } - - writer->stats.bytes_written += original_size; - return 0; -} - /* ============================================================================ * HASH TABLE SUPPORT FUNCTIONS (using _Py_hashtable) * ============================================================================ */ @@ -481,21 +439,16 @@ writer_intern_frame(BinaryWriter *writer, const FrameEntry *entry, uint32_t *ind return 0; } -/* Get or create a thread entry for the given thread_id. - * Returns pointer to ThreadEntry, or NULL on allocation failure. - * If is_new is non-NULL, sets it to 1 if this is a new thread, 0 otherwise. */ +/* Get or create the entry for a (thread, interpreter) pair. */ static ThreadEntry * writer_get_or_create_thread_entry(BinaryWriter *writer, uint64_t thread_id, - uint32_t interpreter_id, int *is_new) + uint32_t interpreter_id) { /* Linear search is OK for small number of threads. * Key is (thread_id, interpreter_id) since same thread_id can exist in different interpreters. */ for (size_t i = 0; i < writer->thread_count; i++) { if (writer->thread_entries[i].thread_id == thread_id && writer->thread_entries[i].interpreter_id == interpreter_id) { - if (is_new) { - *is_new = 0; - } return &writer->thread_entries[i]; } } @@ -506,249 +459,620 @@ writer_get_or_create_thread_entry(BinaryWriter *writer, uint64_t thread_id, "too many threads for binary format"); return NULL; } - if (writer->thread_count >= writer->thread_capacity) { - ThreadEntry *new_entries = grow_array(writer->thread_entries, - &writer->thread_capacity, - sizeof(ThreadEntry)); - if (!new_entries) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - writer->state = BINARY_WRITER_LIMIT_REACHED; - } - return NULL; + if (GROW_ARRAY(writer->thread_entries, writer->thread_count, + writer->thread_capacity, ThreadEntry) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + writer->state = BINARY_WRITER_LIMIT_REACHED; } - writer->thread_entries = new_entries; + return NULL; } ThreadEntry *entry = &writer->thread_entries[writer->thread_count]; memset(entry, 0, sizeof(ThreadEntry)); entry->thread_id = thread_id; entry->interpreter_id = interpreter_id; - entry->prev_timestamp = writer->start_time_us; - entry->prev_stack_capacity = MAX_STACK_DEPTH; - - entry->prev_stack = PyMem_Calloc(entry->prev_stack_capacity, sizeof(uint32_t)); - if (!entry->prev_stack) { - PyErr_NoMemory(); - return NULL; - } + entry->block_stream_index = UINT32_MAX; writer->thread_count++; - if (is_new) { - *is_new = 1; - } return entry; } -/* Compare two stacks and return the encoding type and parameters. - * Sets: - * - shared_count: number of frames matching from bottom of stack - * - pop_count: frames to remove from prev stack - * - push_count: new frames to add - * - * Returns the best encoding type to use. */ static int -compare_stacks(const uint32_t *prev_stack, size_t prev_depth, - const uint32_t *curr_stack, size_t curr_depth, - size_t *shared_count, size_t *pop_count, size_t *push_count) +encode_column(BinaryWriter *writer, const ByteBuffer *decoded, + ColumnDescriptor *descriptor) { - if (prev_depth == curr_depth) { - int identical = 1; - for (size_t i = 0; i < prev_depth; i++) { - if (prev_stack[i] != curr_stack[i]) { - identical = 0; - break; - } + SampleBlock *block = writer->block; + const uint8_t *data = decoded->data; + size_t size = decoded->size; + uint8_t encoding = COMPRESSION_NONE; + if (size > 1 && memcmp(data, data + 1, size - 1) == 0) { + encoding = COLUMN_CONSTANT; + size = 1; + } +#ifdef HAVE_ZSTD + if (encoding == COMPRESSION_NONE && size >= ZSTD_MIN_SIZE && + writer->compression_type == COMPRESSION_ZSTD) { + size_t compress_bound = ZSTD_compressBound(size); + if (buffer_reserve(&block->compressed, compress_bound) < 0) { + return -1; } - if (identical) { - *shared_count = prev_depth; - *pop_count = 0; - *push_count = 0; - return STACK_REPEAT; + size_t compressed_size = ZSTD_compress2( + writer->zstd.cctx, block->compressed.data, compress_bound, + data, size); + if (ZSTD_isError(compressed_size)) { + PyErr_Format(PyExc_IOError, "zstd compression error: %s", + ZSTD_getErrorName(compressed_size)); + return -1; + } + _Py_MSAN_UNPOISON(block->compressed.data, compressed_size); + uint8_t stored_size[MAX_VARINT_SIZE]; + if (compressed_size + encode_varint_u64(stored_size, compressed_size) + < size) { + data = block->compressed.data; + size = compressed_size; + encoding = COMPRESSION_ZSTD; } } +#endif + descriptor->decoded_size = (uint32_t)decoded->size; + descriptor->stored_size = (uint32_t)size; + descriptor->encoding = encoding; + if (buffer_reserve(&block->output, block->output.size + size) < 0) { + return -1; + } + if (size != 0) { + memcpy(block->output.data + block->output.size, data, size); + } + block->output.size += size; + return 0; +} - /* Find longest common suffix (frames at the bottom/outer part of stack). - * Stacks are stored innermost-first, so suffix is at the end. */ - size_t suffix_len = 0; - size_t min_depth = (prev_depth < curr_depth) ? prev_depth : curr_depth; - - for (size_t i = 0; i < min_depth; i++) { - size_t prev_idx = prev_depth - 1 - i; - size_t curr_idx = curr_depth - 1 - i; - if (prev_stack[prev_idx] == curr_stack[curr_idx]) { - suffix_len++; - } else { - break; +static ByteBuffer * +transform_statuses(SampleBlock *block, uint8_t bitmap_flags) +{ + size_t plane_size = status_bitmap_size(block->sample_count); + unsigned plane_count = (unsigned)_Py_popcount32(bitmap_flags); + if (buffer_reserve(&block->transformed, + plane_size * plane_count) < 0) { + return NULL; + } + const uint8_t *statuses = + block->columns[COLUMN_STATUS_BITMAPS].data; + block->transformed.size = 0; + for (unsigned bit = 0; bit < CHAR_BIT; bit++) { + if (!(bitmap_flags & (1U << bit))) { + continue; + } + uint8_t *plane = block->transformed.data + block->transformed.size; + memset(plane, 0, plane_size); + for (uint32_t sample = 0; sample < block->sample_count; sample++) { + plane[sample / CHAR_BIT] |= ((statuses[sample] >> bit) & 1U) + << (sample % CHAR_BIT); } + block->transformed.size += plane_size; } + return &block->transformed; +} - *shared_count = suffix_len; - *pop_count = prev_depth - suffix_len; - *push_count = curr_depth - suffix_len; - - /* Choose best encoding based on byte cost */ - /* STACK_FULL: 1 (type) + 1-2 (depth) + sum(frame varints) */ - /* STACK_SUFFIX: 1 (type) + 1-2 (shared) + 1-2 (new_count) + sum(new frame varints) */ - /* STACK_POP_PUSH: 1 (type) + 1-2 (pop) + 1-2 (push) + sum(new frame varints) */ - - /* If no common suffix, use full stack */ - if (suffix_len == 0) { - return STACK_FULL; +static ByteBuffer * +transform_frame_ids(SampleBlock *block, unsigned width) +{ + ByteBuffer *frames = &block->columns[COLUMN_FRAME_IDS]; + size_t frame_count = frames->size / sizeof(uint32_t); + if (buffer_reserve(&block->transformed, + frame_count * width) < 0) { + return NULL; } - - /* If only adding frames (suffix == prev_depth), use SUFFIX */ - if (*pop_count == 0 && *push_count > 0) { - return STACK_SUFFIX; + const uint32_t *frame_ids = (const uint32_t *)frames->data; + for (unsigned byte = 0; frame_count && byte < width; byte++) { + uint8_t *plane = + block->transformed.data + (size_t)byte * frame_count; + for (uint32_t frame = 0; frame < frame_count; frame++) { + plane[frame] = + (uint8_t)(frame_ids[frame] >> (byte * CHAR_BIT)); + } } + block->transformed.size = frame_count * width; + return &block->transformed; +} - /* If popping and/or pushing, use POP_PUSH if it saves bytes */ - /* Heuristic: POP_PUSH is better when we're modifying top frames */ - if (*pop_count > 0 || *push_count > 0) { - /* Use full stack if sharing less than half the frames */ - if (suffix_len < curr_depth / 2) { - return STACK_FULL; +static ByteBuffer * +transform_stack_instructions(SampleBlock *block) +{ + ByteBuffer *instructions = + &block->columns[COLUMN_STACK_INSTRUCTIONS]; + size_t repeat_count = block->repeats.size / sizeof(Repeat); + if (repeat_count == 0) { + return instructions; + } + const Repeat *repeats = (const Repeat *)block->repeats.data; + size_t copied_offset = 0; + block->transformed.size = 0; + for (size_t i = 0; i < repeat_count; i++) { + if (buffer_append(&block->transformed, + instructions->data + copied_offset, + repeats[i].offset - copied_offset) < 0 || + buffer_append_varint( + &block->transformed, STACK_REPEAT) < 0 || + buffer_append_varint( + &block->transformed, repeats[i].count) < 0) { + return NULL; } - return STACK_POP_PUSH; + copied_offset = repeats[i].offset; + } + if (buffer_append(&block->transformed, + instructions->data + copied_offset, + instructions->size - copied_offset) < 0) { + return NULL; } + return &block->transformed; +} - return STACK_FULL; +static void +block_reset(BinaryWriter *writer) +{ + SampleBlock *block = writer->block; + /* Only threads with a stream in this block need resetting */ + for (uint32_t i = 0; i < block->stream_count; i++) { + size_t thread = block->streams[i].thread_index; + writer->thread_entries[thread].block_stream_index = UINT32_MAX; + } + for (unsigned column = 0; column < COLUMN_COUNT; column++) { + block->columns[column].size = 0; + } + block->sample_count = 0; + block->stream_count = 0; + block->repeats.size = 0; + block->max_frame = 0; + block->size = 0; + memset(block->cache, 0, sizeof(block->cache)); } -/* Write common sample header: thread_id(8) + interpreter_id(4) + encoding(1). - * Returns 0 on success, -1 on failure. */ -static inline int -write_sample_header(BinaryWriter *writer, ThreadEntry *entry, uint8_t encoding) +static unsigned +block_frame_width(const SampleBlock *block) +{ + if (block->max_frame == 0) { + return 1; + } + return (unsigned)(_Py_bit_length(block->max_frame) + CHAR_BIT - 1) + / CHAR_BIT; +} + +static void +block_status_flags(const SampleBlock *block, uint8_t *common_flags, + uint8_t *bitmap_flags) +{ + uint8_t common_status = UINT8_MAX; + uint8_t any_status = 0; + const uint8_t *statuses = block->columns[COLUMN_STATUS_BITMAPS].data; + for (uint32_t sample = 0; sample < block->sample_count; sample++) { + common_status &= statuses[sample]; + any_status |= statuses[sample]; + } + *common_flags = common_status; + *bitmap_flags = any_status ^ common_status; +} + +static size_t +write_block_header(const BlockHeader *header, uint8_t *bytes) { - uint8_t header[SAMPLE_HEADER_FIXED_SIZE]; - memcpy(header + SMP_OFF_THREAD_ID, &entry->thread_id, SMP_SIZE_THREAD_ID); - memcpy(header + SMP_OFF_INTERPRETER_ID, &entry->interpreter_id, SMP_SIZE_INTERPRETER_ID); - header[SMP_OFF_ENCODING] = encoding; - return writer_write_bytes(writer, header, SAMPLE_HEADER_FIXED_SIZE); + /* Magic "TBLK", sample count and base timestamp (varints), then frame + * ID width and the two status flag masks (1 byte each) */ + size_t size = BLOCK_MAGIC_SIZE; + memcpy(bytes, BLOCK_MAGIC, BLOCK_MAGIC_SIZE); + size += encode_varint_u32(bytes + size, header->sample_count); + size += encode_varint_u64(bytes + size, header->base_timestamp); + bytes[size++] = header->frame_width; + bytes[size++] = header->common_status_flags; + bytes[size++] = header->status_bitmap_flags; + + /* Each column: decoded size (varint), encoding (1 byte), and the stored + * size (varint) for zstd only */ + for (unsigned column = 0; column < COLUMN_COUNT; column++) { + const ColumnDescriptor *descriptor = &header->columns[column]; + size += encode_varint_u32(bytes + size, descriptor->decoded_size); + bytes[size++] = descriptor->encoding; + if (descriptor->encoding == COMPRESSION_ZSTD) { + size += encode_varint_u32(bytes + size, descriptor->stored_size); + } + } + return size; } -/* Flush pending RLE samples for a thread. - * Writes the RLE record to the output buffer. - * Returns 0 on success, -1 on failure. */ static int -flush_pending_rle(BinaryWriter *writer, ThreadEntry *entry) +block_flush(BinaryWriter *writer) { - if (entry->pending_rle_samples == 0) { + SampleBlock *block = writer->block; + if (block->sample_count == 0) { return 0; } + BlockHeader header = {0}; + header.sample_count = block->sample_count; + header.base_timestamp = block->base_timestamp; + header.frame_width = (uint8_t)block_frame_width(block); + block_status_flags(block, &header.common_status_flags, + &header.status_bitmap_flags); - /* Write RLE record: - * [thread_id: 8] [interpreter_id: 4] [STACK_REPEAT: 1] [count: varint] - * [timestamp_delta_1: varint] [status_1: 1] ... [timestamp_delta_N: varint] [status_N: 1] - */ - - if (write_sample_header(writer, entry, STACK_REPEAT) < 0) { + /* Convert each column to its file form, then encode it */ + /* Leave room for the header, which is written once its size is known */ + if (buffer_reserve(&block->output, MAX_BLOCK_HEADER_SIZE) < 0) { return -1; } - - if (writer_write_varint_u32(writer, (uint32_t)entry->pending_rle_samples) < 0) { + block->output.size = MAX_BLOCK_HEADER_SIZE; + size_t decoded_payload_size = 0; + for (unsigned column = 0; column < COLUMN_COUNT; column++) { + ByteBuffer *decoded = &block->columns[column]; + switch (column) { + case COLUMN_STATUS_BITMAPS: + decoded = transform_statuses(block, header.status_bitmap_flags); + break; + case COLUMN_STACK_INSTRUCTIONS: + decoded = transform_stack_instructions(block); + break; + case COLUMN_FRAME_IDS: + decoded = transform_frame_ids(block, header.frame_width); + break; + } + if (decoded == NULL) { + return -1; + } + if (decoded->size > + MAX_BLOCK_SIZE - decoded_payload_size) { + PyErr_SetString(PyExc_RuntimeError, + "sample block exceeds the decoded size limit"); + return -1; + } + decoded_payload_size += decoded->size; + if (encode_column(writer, decoded, &header.columns[column]) < 0) { + return -1; + } + } + /* Put the header right before the columns and write the block */ + uint8_t header_bytes[MAX_BLOCK_HEADER_SIZE]; + size_t header_size = write_block_header(&header, header_bytes); + size_t unused = MAX_BLOCK_HEADER_SIZE - header_size; + uint8_t *header_start = block->output.data + unused; + memcpy(header_start, header_bytes, header_size); + if (fwrite_checked_allow_threads(header_start, + block->output.size - unused, + writer->fp) < 0) { return -1; } + writer->stats.bytes_written += decoded_payload_size; + block_reset(writer); + return 0; +} - if (writer_write_bytes(writer, entry->pending_rle, entry->pending_rle_bytes) < 0) { - return -1; +/* FNV-1a over the frame indices, the same in every process */ +static Py_uhash_t +stack_hash(const uint32_t *stack, size_t depth) +{ + uint64_t hash = 0xcbf29ce484222325; + for (size_t i = 0; i < depth; i++) { + hash = (hash ^ stack[i]) * 0x100000001b3; + } + /* Mix the high bits into the low bits that select the cache bucket */ + hash ^= hash >> 33; + hash *= 0xff51afd7ed558ccd; + hash ^= hash >> 33; + return (Py_uhash_t)hash; +} + +static size_t +common_outer_frames(const BlockStream *stream, const uint32_t *stack, + size_t depth) +{ + size_t shared = 0; + while (shared < depth && shared < stream->prev_stack_depth && + stack[depth - 1 - shared] == + stream->prev_stack[stream->prev_stack_depth - 1 - shared]) { + shared++; } + return shared; +} - writer->stats.repeat_records++; - writer->stats.repeat_samples += entry->pending_rle_samples; - /* Each RLE sample saves writing the entire stack */ - writer->stats.frames_saved += entry->pending_rle_samples * entry->prev_stack_depth; +static int +cached_stack_matches(const StackCacheEntry *cached, + const ByteBuffer *frames, Py_uhash_t hash, + const uint32_t *stack, size_t depth) +{ + if (cached->hash != hash || cached->depth != depth) { + return 0; + } + const uint32_t *frame_ids = (const uint32_t *)frames->data; + for (size_t frame = 0; frame < depth; frame++) { + if (stack[frame] != + frame_ids[cached->offset + depth - 1 - frame]) { + return 0; + } + } + return 1; +} + +static StackCacheEntry * +stack_cache_lookup(StackCacheEntry *bucket, const ByteBuffer *frames, + Py_uhash_t hash, const uint32_t *stack, size_t depth) +{ + for (size_t i = 0; i < STACK_CACHE_BUCKET_SIZE; i++) { + if (cached_stack_matches(&bucket[i], frames, hash, stack, depth)) { + return &bucket[i]; + } + } + return NULL; +} - entry->pending_rle_bytes = 0; - entry->pending_rle_samples = 0; +static void +stack_cache_insert(StackCacheEntry *bucket, StackCacheEntry entry) +{ + memmove(&bucket[1], &bucket[0], + (STACK_CACHE_BUCKET_SIZE - 1) * sizeof(*bucket)); + bucket[0] = entry; +} +static int +write_stack_repeat(BinaryWriter *writer, BlockStream *stream, size_t depth) +{ + SampleBlock *block = writer->block; + if (stream->current_repeat == UINT32_MAX) { + Repeat repeat = { + (uint32_t)block->columns[COLUMN_STACK_INSTRUCTIONS].size, 1}; + if (buffer_append(&block->repeats, + &repeat, sizeof(repeat)) < 0) { + return -1; + } + block->size += sizeof(repeat); + stream->current_repeat = + (uint32_t)(block->repeats.size / sizeof(repeat) - 1); + writer->stats.repeat_records++; + } + else { + Repeat *repeats = (Repeat *)block->repeats.data; + repeats[stream->current_repeat].count++; + } + writer->stats.repeat_samples++; + writer->stats.frames_saved += depth; return 0; } -/* Write a single sample with the specified encoding. - * Returns 0 on success, -1 on failure. */ static int -write_sample_with_encoding(BinaryWriter *writer, ThreadEntry *entry, - uint64_t timestamp_delta, uint8_t status, - int encoding_type, - const uint32_t *frame_indices, size_t stack_depth, - size_t shared_count, size_t pop_count, size_t push_count) +write_stack_reuse(BinaryWriter *writer, + const StackCacheEntry *cached, size_t depth) { - /* Header: thread_id(8) + interpreter_id(4) + encoding(1) + delta(varint) + status(1) */ - uint8_t header_buf[SAMPLE_HEADER_MAX_SIZE]; - memcpy(header_buf + SMP_OFF_THREAD_ID, &entry->thread_id, SMP_SIZE_THREAD_ID); - memcpy(header_buf + SMP_OFF_INTERPRETER_ID, &entry->interpreter_id, SMP_SIZE_INTERPRETER_ID); - header_buf[SMP_OFF_ENCODING] = (uint8_t)encoding_type; - size_t varint_len = encode_varint_u64( - header_buf + SAMPLE_HEADER_FIXED_SIZE, - timestamp_delta); - header_buf[SAMPLE_HEADER_FIXED_SIZE + varint_len] = status; - - if (writer_write_bytes(writer, header_buf, - SAMPLE_HEADER_FIXED_SIZE + varint_len + 1) < 0) { + SampleBlock *block = writer->block; + if (column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, + STACK_REUSE) < 0 || + column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, + cached->offset) < 0 || + column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, depth) < 0) { return -1; } + writer->stats.reuse_records++; + writer->stats.frames_saved += depth; + return 0; +} - uint8_t frame_buf[MAX_FRAME_BUFFER_SIZE]; - size_t frame_buf_pos = 0; - size_t frames_written = 0; +static int +write_stack_replace_suffix(BinaryWriter *writer, StackCacheEntry *bucket, + Py_uhash_t hash, const uint32_t *stack, + size_t depth, size_t shared) +{ + SampleBlock *block = writer->block; + ByteBuffer *frames = &block->columns[COLUMN_FRAME_IDS]; + size_t new_frames = depth - shared; + if (column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, + STACK_REPLACE_SUFFIX) < 0 || + column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, shared) < 0 || + column_write_varint(block, COLUMN_STACK_INSTRUCTIONS, + new_frames) < 0) { + return -1; + } - switch (encoding_type) { - case STACK_FULL: - /* [depth: varint] [frame_idx: varint]... */ - frame_buf_pos += encode_varint_u32(frame_buf, (uint32_t)stack_depth); - for (size_t i = 0; i < stack_depth; i++) { - frame_buf_pos += encode_varint_u32(frame_buf + frame_buf_pos, frame_indices[i]); + /* Append the new inner frames, outermost first */ + size_t stack_offset = frames->size / sizeof(uint32_t); + if (new_frames != 0) { + if (buffer_reserve( + frames, frames->size + new_frames * sizeof(uint32_t)) < 0) { + return -1; } - frames_written = stack_depth; - writer->stats.full_records++; - break; - - case STACK_SUFFIX: - /* [shared_count: varint] [new_count: varint] [new_frame_idx: varint]... */ - frame_buf_pos += encode_varint_u32(frame_buf, (uint32_t)shared_count); - frame_buf_pos += encode_varint_u32(frame_buf + frame_buf_pos, (uint32_t)push_count); - /* New frames are at the top (beginning) of current stack */ - for (size_t i = 0; i < push_count; i++) { - frame_buf_pos += encode_varint_u32(frame_buf + frame_buf_pos, frame_indices[i]); + uint32_t *new_frame_ids = (uint32_t *)(frames->data + frames->size); + for (size_t i = 0; i < new_frames; i++) { + uint32_t frame = stack[new_frames - 1 - i]; + new_frame_ids[i] = frame; + if (frame > block->max_frame) { + block->max_frame = frame; + } } - frames_written = push_count; - writer->stats.suffix_records++; - /* Saved writing shared_count frames */ - writer->stats.frames_saved += shared_count; - break; - - case STACK_POP_PUSH: - /* [pop_count: varint] [push_count: varint] [new_frame_idx: varint]... */ - frame_buf_pos += encode_varint_u32(frame_buf, (uint32_t)pop_count); - frame_buf_pos += encode_varint_u32(frame_buf + frame_buf_pos, (uint32_t)push_count); - /* New frames are at the top (beginning) of current stack */ - for (size_t i = 0; i < push_count; i++) { - frame_buf_pos += encode_varint_u32(frame_buf + frame_buf_pos, frame_indices[i]); + frames->size += new_frames * sizeof(uint32_t); + block->size += new_frames * sizeof(uint32_t); + } + /* Complete stacks are cached so later samples can REUSE them */ + if (shared == 0) { + if (bucket != NULL) { + stack_cache_insert(bucket, (StackCacheEntry){ + hash, (uint32_t)stack_offset, (uint32_t)depth}); } - frames_written = push_count; - writer->stats.pop_push_records++; - /* Saved writing shared_count frames (stack_depth - push_count if we had written full) */ - writer->stats.frames_saved += shared_count; - break; - - default: - PyErr_SetString(PyExc_RuntimeError, "Invalid stack encoding type"); + writer->stats.full_records++; + } + else { + writer->stats.suffix_records++; + } + writer->stats.total_frames_written += new_frames; + writer->stats.frames_saved += shared; + return 0; +} + +static int +write_stack(BinaryWriter *writer, BlockStream *stream, int new_stream, + const uint32_t *stack, size_t depth) +{ + SampleBlock *block = writer->block; + size_t shared = common_outer_frames(stream, stack, depth); + /* Same stack as this stream's previous sample: extend its REPEAT */ + if (!new_stream && shared == depth && + shared == stream->prev_stack_depth) { + return write_stack_repeat(writer, stream, depth); + } + + /* Enough new frames to be worth looking up an earlier complete stack */ + Py_uhash_t hash = 0; + StackCacheEntry *bucket = NULL; + StackCacheEntry *cached = NULL; + if (depth - shared >= STACK_REUSE_MIN_FRAMES) { + hash = stack_hash(stack, depth); + bucket = block->cache[hash % STACK_CACHE_BUCKET_COUNT]; + cached = stack_cache_lookup(bucket, &block->columns[COLUMN_FRAME_IDS], + hash, stack, depth); + } + + int result; + if (cached != NULL) { + /* Stack already stored in this block: point to its frames */ + result = write_stack_reuse(writer, cached, depth); + } + else { + /* Keep the shared outer frames and write only the new inner ones */ + result = write_stack_replace_suffix(writer, bucket, hash, stack, + depth, shared); + } + if (result < 0) { return -1; } + stream->current_repeat = UINT32_MAX; + memcpy(stream->prev_stack, stack, depth * sizeof(uint32_t)); + stream->prev_stack_depth = depth; + return 0; +} + +size_t +binary_writer_buffered_size(BinaryWriter *writer) +{ + SampleBlock *block = writer->block; + if (block == NULL || block->sample_count == 0) { + return 0; + } + size_t size = block->columns[COLUMN_STREAM_TABLE].size + + block->columns[COLUMN_SAMPLE_STREAM_IDS].size + + block->columns[COLUMN_TIMESTAMP_RESIDUALS].size + + block->columns[COLUMN_STACK_INSTRUCTIONS].size; + const Repeat *repeats = (const Repeat *)block->repeats.data; + size_t repeat_count = block->repeats.size / sizeof(Repeat); + uint8_t bytes[MAX_VARINT_SIZE]; + for (size_t i = 0; i < repeat_count; i++) { + size += encode_varint_u64(bytes, STACK_REPEAT) + + encode_varint_u64(bytes, repeats[i].count); + } + uint8_t common_flags, bitmap_flags; + block_status_flags(block, &common_flags, &bitmap_flags); + size += (size_t)_Py_popcount32(bitmap_flags) + * status_bitmap_size(block->sample_count); + size += block->columns[COLUMN_FRAME_IDS].size / sizeof(uint32_t) + * block_frame_width(block); + return size; +} - if (writer_write_bytes(writer, frame_buf, frame_buf_pos) < 0) { +static int +block_add_stream(BinaryWriter *writer, ThreadEntry *entry) +{ + SampleBlock *block = writer->block; + if (GROW_ARRAY(block->streams, block->stream_count, + block->stream_capacity, BlockStream) < 0) { return -1; } + /* Stream table entry: thread_id (8 bytes) + interpreter_id (4 bytes) */ + uint8_t stream_entry[STREAM_ENTRY_SIZE]; + memcpy(stream_entry + STM_OFF_THREAD_ID, + &entry->thread_id, sizeof(entry->thread_id)); + memcpy(stream_entry + STM_OFF_INTERPRETER_ID, + &entry->interpreter_id, sizeof(entry->interpreter_id)); + if (column_write(block, COLUMN_STREAM_TABLE, + stream_entry, sizeof(stream_entry)) < 0) { + return -1; + } + BlockStream *stream = &block->streams[block->stream_count]; + stream->thread_index = (size_t)(entry - writer->thread_entries); + stream->prev_timestamp = block->base_timestamp - writer->sample_interval_us; + stream->current_repeat = UINT32_MAX; + stream->prev_stack_depth = 0; + entry->block_stream_index = block->stream_count++; + return 0; +} - writer->stats.total_frames_written += frames_written; +static int +block_add_sample(BinaryWriter *writer, ThreadEntry *entry, + uint64_t timestamp, uint8_t status, + const uint32_t *stack, size_t depth) +{ + static_assert(BLOCK_FLUSH_SIZE + MAX_SAMPLE_SIZE + MAX_BITMAP_PADDING == + MAX_BLOCK_SIZE, + "block flush size needs headroom for one complete sample"); + + SampleBlock *block = writer->block; + /* Start a new block if this sample might not fit */ + if (block->size >= BLOCK_FLUSH_SIZE || + (entry->block_stream_index == UINT32_MAX && + block->stream_count == MAX_BLOCK_STREAMS)) { + if (block_flush(writer) < 0) { + return -1; + } + } + if (block->sample_count == 0) { + block->base_timestamp = timestamp; + } + int new_stream = entry->block_stream_index == UINT32_MAX; + if (new_stream && block_add_stream(writer, entry) < 0) { + return -1; + } + BlockStream *stream = &block->streams[entry->block_stream_index]; + /* Store the difference from the expected timestamp */ + uint64_t residual = + timestamp - stream->prev_timestamp - writer->sample_interval_us; + uint8_t residual_bytes[MAX_VARINT_SIZE]; + size_t residual_size = + encode_varint_i64(residual_bytes, (int64_t)residual); + if (column_write_varint(block, COLUMN_SAMPLE_STREAM_IDS, + entry->block_stream_index) < 0 || + column_write(block, COLUMN_TIMESTAMP_RESIDUALS, + residual_bytes, residual_size) < 0 || + column_write(block, COLUMN_STATUS_BITMAPS, + &status, sizeof(status)) < 0) { + return -1; + } + if (write_stack(writer, stream, new_stream, stack, depth) < 0) { + return -1; + } + stream->prev_timestamp = timestamp; + block->sample_count++; + writer->total_samples++; return 0; } +static void +block_destroy(SampleBlock *block) +{ + if (block == NULL) { + return; + } + for (unsigned i = 0; i < COLUMN_COUNT; i++) { + buffer_clear(&block->columns[i]); + } + buffer_clear(&block->repeats); + buffer_clear(&block->transformed); + buffer_clear(&block->compressed); + buffer_clear(&block->output); + PyMem_Free(block->streams); + PyMem_Free(block); +} + BinaryWriter * binary_writer_create(PyObject *path, uint64_t sample_interval_us, int compression_type, uint64_t start_time_us, int profiling_mode, int capture_features) { + if (compression_type != COMPRESSION_NONE && + compression_type != COMPRESSION_ZSTD) { + PyErr_SetString(PyExc_ValueError, "invalid compression type"); + return NULL; + } BinaryWriter *writer = PyMem_Calloc(1, sizeof(BinaryWriter)); if (!writer) { PyErr_NoMemory(); @@ -761,12 +1085,18 @@ binary_writer_create(PyObject *path, uint64_t sample_interval_us, int compressio writer->profiling_mode = profiling_mode; writer->capture_features = capture_features; - writer->write_buffer = PyMem_Malloc(WRITE_BUFFER_SIZE); - if (!writer->write_buffer) { + writer->block = PyMem_Calloc(1, sizeof(SampleBlock)); + if (writer->block == NULL) { + PyErr_NoMemory(); + goto error; + } + writer->block->streams = PyMem_Malloc( + INITIAL_BLOCK_STREAM_CAPACITY * sizeof(BlockStream)); + if (writer->block->streams == NULL) { PyErr_NoMemory(); goto error; } - writer->buffer_size = WRITE_BUFFER_SIZE; + writer->block->stream_capacity = INITIAL_BLOCK_STREAM_CAPACITY; writer->string_hash = _Py_hashtable_new_full( string_hash_func, @@ -964,74 +1294,29 @@ process_thread_sample(BinaryWriter *writer, PyObject *thread_info, if (status_long == -1 && PyErr_Occurred()) { return -1; } + if (status_long < 0 || status_long > UINT8_MAX) { + writer->state = BINARY_WRITER_LIMIT_REACHED; + PyErr_Format(PyExc_OverflowError, + "thread status %ld exceeds the binary format range", + status_long); + return -1; + } uint8_t status = (uint8_t)status_long; ThreadEntry *entry = writer_get_or_create_thread_entry( - writer, thread_id, interpreter_id, NULL); + writer, thread_id, interpreter_id); if (!entry) { return -1; } - /* Calculate timestamp delta */ - uint64_t delta = timestamp_us - entry->prev_timestamp; - /* Process frames and build current stack */ uint32_t curr_stack[MAX_STACK_DEPTH]; size_t curr_depth; if (build_frame_stack(writer, frame_list, curr_stack, &curr_depth) < 0) { return -1; } - - /* Compare with previous stack to determine encoding */ - size_t shared_count, pop_count, push_count; - int encoding = compare_stacks( - entry->prev_stack, entry->prev_stack_depth, - curr_stack, curr_depth, - &shared_count, &pop_count, &push_count); - - if (encoding == STACK_REPEAT) { - /* Buffer this sample for RLE. - * - * STACK_REPEAT also covers the "first sample for a fresh thread, - * empty stack" case: a new ThreadEntry has prev_stack_depth == 0 - * and a zero-initialized prev_stack, so compare_stacks() returns - * STACK_REPEAT against an empty curr_stack (depth 0). Buffering - * it here is correct; the RLE flush path emits it as a normal - * STACK_REPEAT record. */ - if (entry->pending_rle == NULL) { - entry->pending_rle = PyMem_Malloc(MAX_RLE_BUF_SIZE); - if (!entry->pending_rle) { - PyErr_NoMemory(); - return -1; - } - } - if (entry->pending_rle_bytes + MAX_RLE_ENTRY_SIZE > MAX_RLE_BUF_SIZE - && flush_pending_rle(writer, entry) < 0) { - return -1; - } - entry->pending_rle_bytes += encode_varint_u64( - entry->pending_rle + entry->pending_rle_bytes, delta); - entry->pending_rle[entry->pending_rle_bytes++] = status; - entry->pending_rle_samples++; - } else { - /* Stack changed - flush any pending RLE first */ - if (flush_pending_rle(writer, entry) < 0) { - return -1; - } - - if (write_sample_with_encoding(writer, entry, delta, status, encoding, - curr_stack, curr_depth, - shared_count, pop_count, push_count) < 0) { - return -1; - } - - memcpy(entry->prev_stack, curr_stack, curr_depth * sizeof(uint32_t)); - entry->prev_stack_depth = curr_depth; - } - - entry->prev_timestamp = timestamp_us; - writer->total_samples++; - return 0; + return block_add_sample( + writer, entry, timestamp_us, status, curr_stack, curr_depth); } int @@ -1078,47 +1363,10 @@ binary_writer_write_sample(BinaryWriter *writer, PyObject *stack_frames, uint64_ int binary_writer_finalize(BinaryWriter *writer) { - for (size_t i = 0; i < writer->thread_count; i++) { - if (flush_pending_rle(writer, &writer->thread_entries[i]) < 0) { - return -1; - } - } - - if (writer_flush_buffer(writer) < 0) { + if (block_flush(writer) < 0) { return -1; } -#ifdef HAVE_ZSTD - /* Finalize compression stream */ - if (writer->compression_type == COMPRESSION_ZSTD && writer->zstd.cctx) { - ZSTD_inBuffer input = { NULL, 0, 0 }; - size_t remaining; - - do { - ZSTD_outBuffer output = { - writer->zstd.compressed_buffer, - writer->zstd.compressed_buffer_size, - 0 - }; - - remaining = ZSTD_compressStream2(writer->zstd.cctx, &output, &input, ZSTD_e_end); - - if (ZSTD_isError(remaining)) { - PyErr_Format(PyExc_IOError, "zstd finalization error: %s", - ZSTD_getErrorName(remaining)); - return -1; - } - - _Py_MSAN_UNPOISON(writer->zstd.compressed_buffer, output.pos); - if (output.pos > 0) { - if (fwrite_checked_allow_threads(writer->zstd.compressed_buffer, output.pos, writer->fp) < 0) { - return -1; - } - } - } while (remaining > 0); - } -#endif - /* Use 64-bit file position for >2GB files */ file_offset_t string_table_offset = FTELL64(writer->fp); if (string_table_offset < 0) { @@ -1314,13 +1562,12 @@ binary_writer_destroy(BinaryWriter *writer) Py_fclose(writer->fp); } - PyMem_Free(writer->write_buffer); + block_destroy(writer->block); #ifdef HAVE_ZSTD if (writer->zstd.cctx) { ZSTD_freeCCtx(writer->zstd.cctx); } - PyMem_Free(writer->zstd.compressed_buffer); #endif if (writer->string_hash) { @@ -1339,13 +1586,7 @@ binary_writer_destroy(BinaryWriter *writer) } PyMem_Free(writer->frame_entries); - if (writer->thread_entries) { - for (size_t i = 0; i < writer->thread_count; i++) { - PyMem_Free(writer->thread_entries[i].prev_stack); - PyMem_Free(writer->thread_entries[i].pending_rle); - } - PyMem_Free(writer->thread_entries); - } + PyMem_Free(writer->thread_entries); PyMem_Free(writer); } diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index 9895286cdb80196..28e5a3ca32ac803 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -1168,7 +1168,7 @@ PyDoc_STRVAR(_remote_debugging_BinaryWriter_get_stats__doc__, "Get encoding statistics for the writer.\n" "\n" "Returns a dict with encoding statistics including\n" -"repeat/full/suffix/pop-push record counts, frames written/saved, and\n" +"repeat/full/suffix/reuse record counts, frames written/saved, and\n" "compression ratio."); #define _REMOTE_DEBUGGING_BINARYWRITER_GET_STATS_METHODDEF \ @@ -1740,4 +1740,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=2667550106e045bd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ef0d1eaac8262c2c input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 303af4a4505fea7..c77ff9ae0abd5b5 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -1842,8 +1842,8 @@ _remote_debugging_BinaryWriter_set_stats_impl(BinaryWriterObject *self, /* Finalize the writer, cache total_samples, and destroy it. * * The cache assignment must happen AFTER binary_writer_finalize(): finalize - * flushes pending RLE records and updates the file header before the writer is - * destroyed. Caching before finalize would preserve a count for a file that + * flushes the open sample block and updates the file header before the writer + * is destroyed. Caching before finalize would preserve a count for a file that * might still fail to finish. */ static int binary_writer_finalize_and_cache(BinaryWriterObject *self) @@ -1957,13 +1957,13 @@ _remote_debugging.BinaryWriter.get_stats Get encoding statistics for the writer. Returns a dict with encoding statistics including -repeat/full/suffix/pop-push record counts, frames written/saved, and +repeat/full/suffix/reuse record counts, frames written/saved, and compression ratio. [clinic start generated code]*/ static PyObject * _remote_debugging_BinaryWriter_get_stats_impl(BinaryWriterObject *self) -/*[clinic end generated code: output=06522cd52544df89 input=a8bb8c8682ccd34b]*/ +/*[clinic end generated code: output=06522cd52544df89 input=09c317b6289ae311]*/ { if (!self->writer) { PyErr_SetString(PyExc_ValueError, "Writer is closed"); @@ -2044,6 +2044,22 @@ static PyType_Spec BinaryWriter_spec = { #define BinaryReader_CAST(op) ((BinaryReaderObject *)(op)) +static int +binary_reader_close_checked(BinaryReaderObject *self) +{ + if (self->reader && self->reader->replaying) { + PyErr_SetString(PyExc_RuntimeError, + "Cannot close or reinitialize the reader " + "while it is replaying"); + return -1; + } + if (self->reader) { + binary_reader_close(self->reader); + self->reader = NULL; + } + return 0; +} + /*[clinic input] class _remote_debugging.BinaryReader "BinaryReaderObject *" "&PyBinaryReader_Type" [clinic start generated code]*/ @@ -2066,10 +2082,9 @@ _remote_debugging_BinaryReader___init___impl(BinaryReaderObject *self, PyObject *filename) /*[clinic end generated code: output=f04b33ee5c5e6dbf input=9d7cbe8b4f1a97c9]*/ { - if (self->reader) { - binary_reader_close(self->reader); + if (binary_reader_close_checked(self) < 0) { + return -1; } - self->reader = binary_reader_open(filename); if (!self->reader) { return -1; @@ -2163,9 +2178,8 @@ static PyObject * _remote_debugging_BinaryReader_close_impl(BinaryReaderObject *self) /*[clinic end generated code: output=ad0238cf5240b4f8 input=b919a66c737712d5]*/ { - if (self->reader) { - binary_reader_close(self->reader); - self->reader = NULL; + if (binary_reader_close_checked(self) < 0) { + return NULL; } Py_RETURN_NONE; } @@ -2200,9 +2214,8 @@ _remote_debugging_BinaryReader___exit___impl(BinaryReaderObject *self, PyObject *exc_tb) /*[clinic end generated code: output=2acdd36cfdc14e4a input=87284243d7935835]*/ { - if (self->reader) { - binary_reader_close(self->reader); - self->reader = NULL; + if (binary_reader_close_checked(self) < 0) { + return NULL; } Py_RETURN_FALSE; } From 4b8e53431f0bd02aeeb13c2e2a7b7fe51576774c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Nov 2026 13:29:12 +0100 Subject: [PATCH 182/229] gh-157695: Rename HAVE_GETDENTS64 to _Py_HAVE_GETDENTS64 (#162184) In configure script, use PY_CHECK_FUNC_PRIVATE() to detect functions: * getdents64() of : _Py_HAVE_GETDENTS64 * gettid() of : _Py_HAVE_GETTID * if_indextoname() of : _Py_HAVE_IF_INDEXTONAME * if_nametoindex() of : _Py_HAVE_IF_NAMETOINDEX * pidfd_getfd() of : _Py_HAVE_PIDFD_GETFD * pidfd_open() of : _Py_HAVE_PIDFD_OPEN * pidfd_send_signal() of : _Py_HAVE_PIDFD_SEND_SIGNAL Add shebang to Tools/build/check_capi_macros.py and make the script executable. --- Modules/_posixsubprocess.c | 8 +- Modules/clinic/posixmodule.c.h | 10 +- Modules/clinic/signalmodule.c.h | 6 +- Modules/clinic/socketmodule.c.h | 10 +- Modules/posixmodule.c | 8 +- Modules/signalmodule.c | 4 +- Modules/socketmodule.c | 12 +- Python/perf_jit_trampoline.c | 2 +- Python/thread_pthread.h | 4 +- Tools/build/check_capi_macros.py | 1 + Tools/build/check_capi_macros_ignored.txt | 7 - configure | 335 +++++++++++++++++++--- configure.ac | 16 +- pyconfig.h.in | 42 +-- 14 files changed, 358 insertions(+), 107 deletions(-) mode change 100644 => 100755 Tools/build/check_capi_macros.py diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 415e0afbe88125e..8ad01414ad80755 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -373,10 +373,10 @@ _close_range_except(int start_fd, return 0; } -#if defined(HAVE_GETDENTS64) \ +#if defined(_Py_HAVE_GETDENTS64) \ || (defined(__linux__) && defined(HAVE_SYS_SYSCALL_H)) -#ifdef HAVE_GETDENTS64 +#ifdef _Py_HAVE_GETDENTS64 # define py_dirent64 dirent64 #else /* It doesn't matter if d_name has room for NAME_MAX chars; we're using this @@ -392,7 +392,7 @@ struct py_dirent64 { unsigned char d_type; char d_name[256]; /* Filename (null-terminated) */ }; -#endif // !HAVE_GETDENTS64 +#endif // !_Py_HAVE_GETDENTS64 static int _brute_force_closer(int first, int last) @@ -435,7 +435,7 @@ _close_open_fds_safe(int start_fd, int *fds_to_keep, Py_ssize_t fds_to_keep_len) char buffer[sizeof(struct py_dirent64)]; Py_ssize_t bytes; while (1) { -#ifdef HAVE_GETDENTS64 +#ifdef _Py_HAVE_GETDENTS64 /* musl <1.2.4 headers expose `getdents64` as an alias for `getdents`, whose buffer parameter * is `struct dirent *`. We use `void *` to support both that interface and the glibc declaration * without an incompatible-pointer diagnostic. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index e992a4d1b8f7971..c2f183f70378367 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -6322,7 +6322,7 @@ os_wait(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_WAIT) */ -#if (defined(HAVE_PIDFD_OPEN) || (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) +#if (defined(_Py_HAVE_PIDFD_OPEN) || (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) PyDoc_STRVAR(os_pidfd_open__doc__, "pidfd_open($module, /, pid, flags=0)\n" @@ -6397,9 +6397,9 @@ os_pidfd_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -#endif /* (defined(HAVE_PIDFD_OPEN) || (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) */ +#endif /* (defined(_Py_HAVE_PIDFD_OPEN) || (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) */ -#if (defined(HAVE_PIDFD_GETFD) || (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) +#if (defined(_Py_HAVE_PIDFD_GETFD) || (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) PyDoc_STRVAR(os_pidfd_getfd__doc__, "pidfd_getfd($module, /, pidfd, targetfd, *, flags=0)\n" @@ -6484,7 +6484,7 @@ os_pidfd_getfd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje return return_value; } -#endif /* (defined(HAVE_PIDFD_GETFD) || (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) */ +#endif /* (defined(_Py_HAVE_PIDFD_GETFD) || (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) */ #if defined(HAVE_SETNS) @@ -13763,4 +13763,4 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=c92854d21c8f6290 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=077c57e5f93bd790 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 6125f253ac0380d..26139ff0fc0cd39 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -689,7 +689,7 @@ signal_pthread_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_PTHREAD_KILL) */ -#if (defined(HAVE_PIDFD_SEND_SIGNAL) || (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) +#if (defined(_Py_HAVE_PIDFD_SEND_SIGNAL) || (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) PyDoc_STRVAR(signal_pidfd_send_signal__doc__, "pidfd_send_signal($module, pidfd, signalnum, siginfo=None, flags=0, /)\n" @@ -754,7 +754,7 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } -#endif /* (defined(HAVE_PIDFD_SEND_SIGNAL) || (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) */ +#endif /* (defined(_Py_HAVE_PIDFD_SEND_SIGNAL) || (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31))) */ #ifndef SIGNAL_ALARM_METHODDEF #define SIGNAL_ALARM_METHODDEF @@ -807,4 +807,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=2a04ec31f49b1c93 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=48d30418971f312a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 09041d92a305793..5a1a128b304f028 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -2152,7 +2152,7 @@ _socket_if_nameindex(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* (defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS_DESKTOP)) */ -#if (defined(HAVE_IF_NAMETOINDEX) || defined(MS_WINDOWS_DESKTOP)) +#if (defined(_Py_HAVE_IF_NAMETOINDEX) || defined(MS_WINDOWS_DESKTOP)) PyDoc_STRVAR(_socket_if_nametoindex__doc__, "if_nametoindex($module, oname, /)\n" @@ -2184,9 +2184,9 @@ _socket_if_nametoindex(PyObject *module, PyObject *arg) return return_value; } -#endif /* (defined(HAVE_IF_NAMETOINDEX) || defined(MS_WINDOWS_DESKTOP)) */ +#endif /* (defined(_Py_HAVE_IF_NAMETOINDEX) || defined(MS_WINDOWS_DESKTOP)) */ -#if (defined(HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP)) +#if (defined(_Py_HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP)) PyDoc_STRVAR(_socket_if_indextoname__doc__, "if_indextoname($module, if_index, /)\n" @@ -2215,7 +2215,7 @@ _socket_if_indextoname(PyObject *module, PyObject *arg) return return_value; } -#endif /* (defined(HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP)) */ +#endif /* (defined(_Py_HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP)) */ #if defined(CMSG_LEN) @@ -2449,4 +2449,4 @@ _socket_CMSG_SPACE(PyObject *module, PyObject *arg) #ifndef _SOCKET_CMSG_SPACE_METHODDEF #define _SOCKET_CMSG_SPACE_METHODDEF #endif /* !defined(_SOCKET_CMSG_SPACE_METHODDEF) */ -/*[clinic end generated code: output=bca43f33969d67ad input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e6e677d3de0c63e4 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c6f89fc2454e88a..619154309b40e31 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10870,7 +10870,7 @@ os_wait_impl(PyObject *module) // This system call always crashes on older Android versions. -#if defined(HAVE_PIDFD_OPEN) \ +#if defined(_Py_HAVE_PIDFD_OPEN) \ || (defined(__linux__) && defined(__NR_pidfd_open) \ && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) /*[clinic input] @@ -10888,7 +10888,7 @@ static PyObject * os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags) /*[clinic end generated code: output=5c7252698947dc41 input=03058b32c389f874]*/ { -#ifdef HAVE_PIDFD_OPEN +#ifdef _Py_HAVE_PIDFD_OPEN int fd = pidfd_open(pid, flags); #else int fd = syscall(__NR_pidfd_open, pid, flags); @@ -10901,7 +10901,7 @@ os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags) #endif -#if defined(HAVE_PIDFD_GETFD) \ +#if defined(_Py_HAVE_PIDFD_GETFD) \ || (defined(__linux__) && defined(__NR_pidfd_getfd) \ && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) /*[clinic input] @@ -10922,7 +10922,7 @@ os_pidfd_getfd_impl(PyObject *module, int pidfd, int targetfd, unsigned int flags) /*[clinic end generated code: output=e1a1415a13c7137f input=ef6417fb10deb1cc]*/ { -#ifdef HAVE_PIDFD_GETFD +#ifdef _Py_HAVE_PIDFD_GETFD int fd = pidfd_getfd(pidfd, targetfd, flags); #else int fd = syscall(__NR_pidfd_getfd, pidfd, targetfd, flags); diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index addf6b456b1f95c..85ed113a17e5e81 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1304,7 +1304,7 @@ signal_pthread_kill_impl(PyObject *module, unsigned long thread_id, // This system call always crashes on older Android versions. -#if defined(HAVE_PIDFD_SEND_SIGNAL) \ +#if defined(_Py_HAVE_PIDFD_SEND_SIGNAL) \ || (defined(__linux__) && defined(__NR_pidfd_send_signal) \ && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) /*[clinic input] @@ -1329,7 +1329,7 @@ signal_pidfd_send_signal_impl(PyObject *module, int pidfd, int signalnum, PyErr_SetString(PyExc_TypeError, "siginfo must be None"); return NULL; } -#ifdef HAVE_PIDFD_SEND_SIGNAL +#ifdef _Py_HAVE_PIDFD_SEND_SIGNAL int res = pidfd_send_signal(pidfd, signalnum, NULL, flags); #else int res = syscall(__NR_pidfd_send_signal, pidfd, signalnum, NULL, flags); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 4df03284cbdd24f..7a76882e1633e10 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -606,14 +606,14 @@ _PyLong_##NAME##_Converter(PyObject *obj, void *ptr) \ return 1; \ } -#if defined(HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP) +#if defined(_Py_HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP) # ifdef MS_WINDOWS UNSIGNED_INT_CONVERTER(NetIfindex, NET_IFINDEX) # else # define _PyLong_NetIfindex_Converter _PyLong_UnsignedInt_Converter # define NET_IFINDEX unsigned int # endif -#endif // defined(HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP) +#endif // defined(_Py_HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP) /*[python input] class NET_IFINDEX_converter(CConverter): @@ -7403,7 +7403,7 @@ _socket_if_nameindex_impl(PyObject *module) #endif // defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS_DESKTOP) -#if defined(HAVE_IF_NAMETOINDEX) || defined(MS_WINDOWS_DESKTOP) +#if defined(_Py_HAVE_IF_NAMETOINDEX) || defined(MS_WINDOWS_DESKTOP) /*[clinic input] _socket.if_nametoindex @@ -7433,9 +7433,9 @@ _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) return PyLong_FromUnsignedLong(index); } -#endif // defined(HAVE_IF_NAMETOINDEX) || defined(MS_WINDOWS_DESKTOP) +#endif // defined(_Py_HAVE_IF_NAMETOINDEX) || defined(MS_WINDOWS_DESKTOP) -#if defined(HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP) +#if defined(_Py_HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP) /*[clinic input] @permit_long_summary @@ -7460,7 +7460,7 @@ _socket_if_indextoname_impl(PyObject *module, NET_IFINDEX index) return PyUnicode_DecodeFSDefault(name); } -#endif // defined(HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP) +#endif // defined(_Py_HAVE_IF_INDEXTONAME) || defined(MS_WINDOWS_DESKTOP) #ifdef CMSG_LEN diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 07aa8e769bf2d7d..da6a7e69633db52 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -672,7 +672,7 @@ static void perf_map_jit_write_entry_with_name( uint64_t thread_id = 0; pthread_threadid_np(NULL, &thread_id); ev.thread_id = (uint32_t)thread_id; -#elif defined(HAVE_GETTID) +#elif defined(_Py_HAVE_GETTID) ev.thread_id = gettid(); #else ev.thread_id = syscall(SYS_gettid); // Get thread ID via system call diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index e660bca84068604..c7f245acddb290c 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -18,7 +18,7 @@ #include #include /* pause(), also getthrid() on OpenBSD */ -#ifdef HAVE_GETTID +#ifdef _Py_HAVE_GETTID # include // gettid() #elif defined(__linux__) # include /* syscall(SYS_gettid) */ @@ -384,7 +384,7 @@ PyThread_get_thread_native_id(void) #ifdef __APPLE__ uint64_t native_id; (void) pthread_threadid_np(NULL, &native_id); -#elif defined(HAVE_GETTID) +#elif defined(_Py_HAVE_GETTID) pid_t native_id; native_id = gettid(); #elif defined(__linux__) diff --git a/Tools/build/check_capi_macros.py b/Tools/build/check_capi_macros.py old mode 100644 new mode 100755 index 62ae1fd0bb1c941..86c433a11683487 --- a/Tools/build/check_capi_macros.py +++ b/Tools/build/check_capi_macros.py @@ -1,3 +1,4 @@ +#! /usr/bin/env python3 """ Check that all macros defined by the Python C API have a name starting with "Py". Ignore names listed by check_capi_macros_ignored.txt: macros with an diff --git a/Tools/build/check_capi_macros_ignored.txt b/Tools/build/check_capi_macros_ignored.txt index 1800a2c4ec8946c..73d4f7be0766b9f 100644 --- a/Tools/build/check_capi_macros_ignored.txt +++ b/Tools/build/check_capi_macros_ignored.txt @@ -235,7 +235,6 @@ HAVE_GDBM_H HAVE_GDBM_NDBM_H HAVE_GETADDRINFO HAVE_GETC_UNLOCKED -HAVE_GETDENTS64 HAVE_GETEGID HAVE_GETENTROPY HAVE_GETEUID @@ -279,16 +278,13 @@ HAVE_GETSERVBYNAME HAVE_GETSERVBYPORT HAVE_GETSID HAVE_GETSOCKNAME -HAVE_GETTID HAVE_GETUID HAVE_GRANTPT HAVE_GRP_H HAVE_HSTRERROR HAVE_HTOLE64 HAVE_ICONV_H -HAVE_IF_INDEXTONAME HAVE_IF_NAMEINDEX -HAVE_IF_NAMETOINDEX HAVE_INET_ATON HAVE_INET_NTOA HAVE_INET_PTON @@ -389,9 +385,6 @@ HAVE_PANELW HAVE_PANEL_H HAVE_PATHCONF HAVE_PAUSE -HAVE_PIDFD_GETFD -HAVE_PIDFD_OPEN -HAVE_PIDFD_SEND_SIGNAL HAVE_PIPE HAVE_PIPE2 HAVE_PLOCK diff --git a/configure b/configure index e033b0217aefad5..a2e9c1711adb794 100755 --- a/configure +++ b/configure @@ -21009,12 +21009,6 @@ if test "x$ac_cv_func_gai_strerror" = xyes then : printf "%s\n" "#define HAVE_GAI_STRERROR 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "getdents64" "ac_cv_func_getdents64" -if test "x$ac_cv_func_getdents64" = xyes -then : - printf "%s\n" "#define HAVE_GETDENTS64 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "getegid" "ac_cv_func_getegid" if test "x$ac_cv_func_getegid" = xyes @@ -21177,12 +21171,6 @@ if test "x$ac_cv_func_getsid" = xyes then : printf "%s\n" "#define HAVE_GETSID 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "gettid" "ac_cv_func_gettid" -if test "x$ac_cv_func_gettid" = xyes -then : - printf "%s\n" "#define HAVE_GETTID 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "getuid" "ac_cv_func_getuid" if test "x$ac_cv_func_getuid" = xyes @@ -21195,24 +21183,12 @@ if test "x$ac_cv_func_grantpt" = xyes then : printf "%s\n" "#define HAVE_GRANTPT 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "if_indextoname" "ac_cv_func_if_indextoname" -if test "x$ac_cv_func_if_indextoname" = xyes -then : - printf "%s\n" "#define HAVE_IF_INDEXTONAME 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "if_nameindex" "ac_cv_func_if_nameindex" if test "x$ac_cv_func_if_nameindex" = xyes then : printf "%s\n" "#define HAVE_IF_NAMEINDEX 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "if_nametoindex" "ac_cv_func_if_nametoindex" -if test "x$ac_cv_func_if_nametoindex" = xyes -then : - printf "%s\n" "#define HAVE_IF_NAMETOINDEX 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "initgroups" "ac_cv_func_initgroups" if test "x$ac_cv_func_initgroups" = xyes @@ -21357,24 +21333,6 @@ if test "x$ac_cv_func_pause" = xyes then : printf "%s\n" "#define HAVE_PAUSE 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "pidfd_open" "ac_cv_func_pidfd_open" -if test "x$ac_cv_func_pidfd_open" = xyes -then : - printf "%s\n" "#define HAVE_PIDFD_OPEN 1" >>confdefs.h - -fi -ac_fn_c_check_func "$LINENO" "pidfd_getfd" "ac_cv_func_pidfd_getfd" -if test "x$ac_cv_func_pidfd_getfd" = xyes -then : - printf "%s\n" "#define HAVE_PIDFD_GETFD 1" >>confdefs.h - -fi -ac_fn_c_check_func "$LINENO" "pidfd_send_signal" "ac_cv_func_pidfd_send_signal" -if test "x$ac_cv_func_pidfd_send_signal" = xyes -then : - printf "%s\n" "#define HAVE_PIDFD_SEND_SIGNAL 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "pipe" "ac_cv_func_pipe" if test "x$ac_cv_func_pipe" = xyes @@ -22101,6 +22059,299 @@ fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for getdents64" >&5 +printf %s "checking for getdents64... " >&6; } +if test ${ac_cv_func_getdents64+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main (void) +{ +(void)getdents64 + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_func_getdents64=yes +else case e in #( + e) ac_cv_func_getdents64=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_getdents64" >&5 +printf "%s\n" "$ac_cv_func_getdents64" >&6; } + if test "x$ac_cv_func_getdents64" = xyes +then : + +printf "%s\n" "#define _Py_HAVE_GETDENTS64 1" >>confdefs.h + +fi + + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gettid" >&5 +printf %s "checking for gettid... " >&6; } +if test ${ac_cv_func_gettid+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main (void) +{ +(void)gettid + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_func_gettid=yes +else case e in #( + e) ac_cv_func_gettid=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_gettid" >&5 +printf "%s\n" "$ac_cv_func_gettid" >&6; } + if test "x$ac_cv_func_gettid" = xyes +then : + +printf "%s\n" "#define _Py_HAVE_GETTID 1" >>confdefs.h + +fi + + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for if_indextoname" >&5 +printf %s "checking for if_indextoname... " >&6; } +if test ${ac_cv_func_if_indextoname+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main (void) +{ +(void)if_indextoname + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_func_if_indextoname=yes +else case e in #( + e) ac_cv_func_if_indextoname=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_if_indextoname" >&5 +printf "%s\n" "$ac_cv_func_if_indextoname" >&6; } + if test "x$ac_cv_func_if_indextoname" = xyes +then : + +printf "%s\n" "#define _Py_HAVE_IF_INDEXTONAME 1" >>confdefs.h + +fi + + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for if_nametoindex" >&5 +printf %s "checking for if_nametoindex... " >&6; } +if test ${ac_cv_func_if_nametoindex+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main (void) +{ +(void)if_nametoindex + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_func_if_nametoindex=yes +else case e in #( + e) ac_cv_func_if_nametoindex=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_if_nametoindex" >&5 +printf "%s\n" "$ac_cv_func_if_nametoindex" >&6; } + if test "x$ac_cv_func_if_nametoindex" = xyes +then : + +printf "%s\n" "#define _Py_HAVE_IF_NAMETOINDEX 1" >>confdefs.h + +fi + + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for pidfd_open" >&5 +printf %s "checking for pidfd_open... " >&6; } +if test ${ac_cv_func_pidfd_open+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main (void) +{ +(void)pidfd_open + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_func_pidfd_open=yes +else case e in #( + e) ac_cv_func_pidfd_open=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_pidfd_open" >&5 +printf "%s\n" "$ac_cv_func_pidfd_open" >&6; } + if test "x$ac_cv_func_pidfd_open" = xyes +then : + +printf "%s\n" "#define _Py_HAVE_PIDFD_OPEN 1" >>confdefs.h + +fi + + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for pidfd_getfd" >&5 +printf %s "checking for pidfd_getfd... " >&6; } +if test ${ac_cv_func_pidfd_getfd+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main (void) +{ +(void)pidfd_getfd + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_func_pidfd_getfd=yes +else case e in #( + e) ac_cv_func_pidfd_getfd=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_pidfd_getfd" >&5 +printf "%s\n" "$ac_cv_func_pidfd_getfd" >&6; } + if test "x$ac_cv_func_pidfd_getfd" = xyes +then : + +printf "%s\n" "#define _Py_HAVE_PIDFD_GETFD 1" >>confdefs.h + +fi + + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for pidfd_send_signal" >&5 +printf %s "checking for pidfd_send_signal... " >&6; } +if test ${ac_cv_func_pidfd_send_signal+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main (void) +{ +(void)pidfd_send_signal + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_func_pidfd_send_signal=yes +else case e in #( + e) ac_cv_func_pidfd_send_signal=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_func_pidfd_send_signal" >&5 +printf "%s\n" "$ac_cv_func_pidfd_send_signal" >&6; } + if test "x$ac_cv_func_pidfd_send_signal" = xyes +then : + +printf "%s\n" "#define _Py_HAVE_PIDFD_SEND_SIGNAL 1" >>confdefs.h + +fi + + + + + # os.statx uses Linux's statx function. AIX also has a function named statx, # but it's unrelated. Check only on Linux (including Android). case $ac_sys_system in #( diff --git a/configure.ac b/configure.ac index d654f680643041c..2be6d852c6e977a 100644 --- a/configure.ac +++ b/configure.ac @@ -5757,15 +5757,15 @@ AC_CHECK_FUNCS([ \ copy_file_range ctermid dladdr dup explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fpathconf fstatat ftruncate futimens futimes futimesat \ - gai_strerror getdents64 getegid geteuid getgid getgrent getgrgid getgrgid_r \ + gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin getlogin_r \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid \ - gettid getuid grantpt if_indextoname if_nameindex \ - if_nametoindex initgroups kill killpg lchown linkat \ + getuid grantpt if_nameindex \ + initgroups kill killpg lchown linkat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause \ - pidfd_open pidfd_getfd pidfd_send_signal pipe \ + pipe \ plock poll ppoll posix_fadvise posix_fallocate posix_openpt \ pread preadv preadv2 process_vm_readv \ pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ @@ -5788,7 +5788,13 @@ PY_CHECK_FUNC_PRIVATE([freadlink], [@%:@include ]) PY_CHECK_FUNC_PRIVATE([strerror_r], [@%:@include ]) # FreeBSD pthread_attr_get_np() PY_CHECK_FUNC_PRIVATE([pthread_attr_get_np], [@%:@include ]) - +PY_CHECK_FUNC_PRIVATE([getdents64], [@%:@include ]) +PY_CHECK_FUNC_PRIVATE([gettid], [@%:@include ]) +PY_CHECK_FUNC_PRIVATE([if_indextoname], [@%:@include ]) +PY_CHECK_FUNC_PRIVATE([if_nametoindex], [@%:@include ]) +PY_CHECK_FUNC_PRIVATE([pidfd_open], [@%:@include ]) +PY_CHECK_FUNC_PRIVATE([pidfd_getfd], [@%:@include ]) +PY_CHECK_FUNC_PRIVATE([pidfd_send_signal], [@%:@include ]) # os.statx uses Linux's statx function. AIX also has a function named statx, # but it's unrelated. Check only on Linux (including Android). diff --git a/pyconfig.h.in b/pyconfig.h.in index dba7d193bcb3d19..18534d7e80e3893 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -552,9 +552,6 @@ /* Define this if you have flockfile(), getc_unlocked(), and funlockfile() */ #undef HAVE_GETC_UNLOCKED -/* Define to 1 if you have the 'getdents64' function. */ -#undef HAVE_GETDENTS64 - /* Define to 1 if you have the 'getegid' function. */ #undef HAVE_GETEGID @@ -684,9 +681,6 @@ /* Define if you have the 'getsockname' function. */ #undef HAVE_GETSOCKNAME -/* Define to 1 if you have the 'gettid' function. */ -#undef HAVE_GETTID - /* Define to 1 if you have the 'getuid' function. */ #undef HAVE_GETUID @@ -705,15 +699,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_ICONV_H -/* Define to 1 if you have the 'if_indextoname' function. */ -#undef HAVE_IF_INDEXTONAME - /* Define to 1 if you have the 'if_nameindex' function. */ #undef HAVE_IF_NAMEINDEX -/* Define to 1 if you have the 'if_nametoindex' function. */ -#undef HAVE_IF_NAMETOINDEX - /* Define if you have the 'inet_aton' function. */ #undef HAVE_INET_ATON @@ -1009,15 +997,6 @@ /* Define to 1 if you have the 'pause' function. */ #undef HAVE_PAUSE -/* Define to 1 if you have the 'pidfd_getfd' function. */ -#undef HAVE_PIDFD_GETFD - -/* Define to 1 if you have the 'pidfd_open' function. */ -#undef HAVE_PIDFD_OPEN - -/* Define to 1 if you have the 'pidfd_send_signal' function. */ -#undef HAVE_PIDFD_SEND_SIGNAL - /* Define to 1 if you have the 'pipe' function. */ #undef HAVE_PIPE @@ -2151,13 +2130,34 @@ /* Define if you have the 'freadlink' function. */ #undef _Py_HAVE_FREADLINK +/* Define if you have the 'getdents64' function. */ +#undef _Py_HAVE_GETDENTS64 + +/* Define if you have the 'gettid' function. */ +#undef _Py_HAVE_GETTID + /* Define if you have a working iconv() function. */ #undef _Py_HAVE_ICONV +/* Define if you have the 'if_indextoname' function. */ +#undef _Py_HAVE_IF_INDEXTONAME + +/* Define if you have the 'if_nametoindex' function. */ +#undef _Py_HAVE_IF_NAMETOINDEX + /* Define to 1 if libgcc __register_frame and __deregister_frame are linkable. */ #undef _Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION +/* Define if you have the 'pidfd_getfd' function. */ +#undef _Py_HAVE_PIDFD_GETFD + +/* Define if you have the 'pidfd_open' function. */ +#undef _Py_HAVE_PIDFD_OPEN + +/* Define if you have the 'pidfd_send_signal' function. */ +#undef _Py_HAVE_PIDFD_SEND_SIGNAL + /* Define if you have the 'PR_SET_VMA_ANON_NAME' constant. */ #undef _Py_HAVE_PR_SET_VMA_ANON_NAME From aa17c688d2c160969f2c8963f4454cdd139808e7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 18 Nov 2026 13:45:32 +0100 Subject: [PATCH 183/229] gh-157579: Fix TemporaryDirectory cleanup fallbacks with bytes paths (GH-162116) Co-authored-by: Stan Ulbrych --- Lib/tempfile.py | 7 +- Lib/test/test_shutil.py | 57 ++++++++ Lib/test/test_tempfile.py | 122 +++++++++++++++++- ...-11-16-16-10-42.gh-issue-157579.6choQ_.rst | 3 + 4 files changed, 180 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-16-16-10-42.gh-issue-157579.6choQ_.rst diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 82c1ee353117e94..53fea2873d2bae8 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -987,10 +987,13 @@ def __init__(self, suffix=None, prefix=None, dir=None, @classmethod def _rmtree(cls, name, ignore_errors=False, repeated=False, dir_fd=None, fullname=None): - if fullname is None: - fullname = name + + # work with str paths throughout + name = _os.fsdecode(name) + fullname = name if fullname is None else _os.fsdecode(fullname) def onexc(func, path, exc, direntry=None, dir_fd=None): + path = _os.fsdecode(path) # On DragonFly BSD, UF_NOUNLINK removal fails with EISDIR, not EPERM. if isinstance(exc, (PermissionError, IsADirectoryError)): if repeated and path == name: diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 1104552215a44f1..47221d1bf46fa18 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -566,6 +566,63 @@ def onexc(*args): self.assertTrue(onexc_called) self.assertFalse(onerror_called) + @unittest.skipUnless(shutil.rmtree.avoids_symlink_attacks, + 'requires the fd based implementation of rmtree()') + def test_onexc_kwargs_dir_fd(self): + os.mkdir(TESTFN) + self.addCleanup(os_helper.rmtree, TESTFN) + top_name = os.path.join(TESTFN, 'top') + child_name = os.path.join(TESTFN, 'top', 'child') + os.mkdir(top_name) + os.mkdir(child_name) + parent_fd = os.open(TESTFN, os.O_RDONLY) + self.addCleanup(os.close, parent_fd) + + errors = [] + def onexc(func, path, err, direntry=None, dir_fd=None): + errors.append((func, path.replace(os.path.sep, '/'))) + name = path if direntry is None else direntry.name + self.assertEqual(name, os.path.basename(path)) + self.assertTrue(os.path.samestat( + os.stat(os.path.join(TESTFN, path)), + os.stat(name, dir_fd=dir_fd))) + + def failing_rmdir(path, *, dir_fd=None): + raise PermissionError(errno.EACCES, 'simulated') + with unittest.mock.patch('os.rmdir', failing_rmdir): + shutil.rmtree('top', dir_fd=parent_fd, onexc=onexc, + _onexc_kwargs=True) + self.assertEqual( + errors, + [(failing_rmdir, 'top/child'), (failing_rmdir, 'top')]) + + child_stat = os.stat(child_name) + scandir = os.scandir + + # Fail for the top-level directory + errors.clear() + def failing_scandir(fd): + raise PermissionError(errno.EACCES, 'simulated') + with unittest.mock.patch('os.scandir', failing_scandir): + shutil.rmtree('top', dir_fd=parent_fd, onexc=onexc, + _onexc_kwargs=True) + self.assertEqual( + errors, + [(failing_scandir, 'top'), (os.rmdir, 'top')]) + self.assertTrue(os.path.exists(top_name)) + + # Fail for the child (which is empty, so the rmdir succeeds) + errors.clear() + def failing_scandir_2(fd): + if not os.path.samestat(os.fstat(fd), child_stat): + return scandir(fd) + raise PermissionError(errno.EACCES, 'simulated') + with unittest.mock.patch('os.scandir', failing_scandir_2): + shutil.rmtree('top', dir_fd=parent_fd, onexc=onexc, + _onexc_kwargs=True) + self.assertEqual(errors, [(failing_scandir_2, 'top/child')]) + self.assertFalse(os.path.exists(top_name)) + def test_rmtree_does_not_choke_on_failing_lstat(self): try: orig_lstat = os.lstat diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 2f7b8798daa8c95..3e87ccfc436d121 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1653,10 +1653,14 @@ def __exit__(self, *exc_info): class TestTemporaryDirectory(BaseTestCase): """Test TemporaryDirectory().""" - def do_create(self, dir=None, pre="", suf="", recurse=1, dirs=1, files=1, + def do_create(self, dir=None, pre=None, suf=None, recurse=1, dirs=1, files=1, ignore_cleanup_errors=False): if dir is None: dir = tempfile.gettempdir() + if pre is None: + pre = type(dir)() + if suf is None: + suf = type(dir)() tmp = tempfile.TemporaryDirectory( dir=dir, prefix=pre, suffix=suf, ignore_cleanup_errors=ignore_cleanup_errors) @@ -1666,13 +1670,17 @@ def do_create(self, dir=None, pre="", suf="", recurse=1, dirs=1, files=1, def do_create2(self, path, recurse=1, dirs=1, files=1): # Create subdirectories and some files + def join(name): + if isinstance(path, bytes): + name = os.fsencode(name) + return os.path.join(path, name) if recurse: for i in range(dirs): - name = os.path.join(path, "dir%d" % i) + name = join("dir%d" % i) os.mkdir(name) self.do_create2(name, recurse-1, dirs, files) for i in range(files): - with open(os.path.join(path, "test%d.txt" % i), "wb") as f: + with open(join("test%d.txt" % i), "wb") as f: f.write(b"Hello world!") def test_mkdtemp_failure(self): @@ -1698,6 +1706,15 @@ def test_explicit_cleanup(self): finally: os.rmdir(dir) + def test_explicit_cleanup_bytes(self): + # A TemporaryDirectory with a bytes name is deleted when cleaned up + with tempfile.TemporaryDirectory() as working_dir: + d = self.do_create(dir=os.fsencode(working_dir), recurse=2) + self.assertIsInstance(d.name, bytes) + self.assertTrue(os.path.exists(d.name)) + d.cleanup() + self.assertFalse(os.path.exists(d.name)) + def test_explicit_cleanup_ignore_errors(self): """Test that cleanup doesn't return an error when ignoring them.""" with tempfile.TemporaryDirectory() as working_dir: @@ -1910,6 +1927,87 @@ def hook(path, *, dir_fd=None): self.assertTrue(os.path.exists(target_file)) self.assertEqual(os.stat(target_file).st_mode, target_mode) + def _make_unopenable_subdir(self): + # Create the following structure: + # work/ - top-level + # / - tempdir to clean up (random name) + # sub/ - an unopenable dir + # subsub/ + # file + # Return: + # - `work` name (str) + # - `d` (TemporaryDirectory) + # - `subsub` stat result + work = self.enterContext(os_helper.temp_dir()) + d = tempfile.TemporaryDirectory(dir=work) + sub = os.path.join(d.name, 'sub') + subsub = os.path.join(sub, 'subsub') + os.mkdir(sub) + os.mkdir(subsub) + subsub_stat = os.lstat(subsub) + with open(os.path.join(subsub, 'file'), 'w') as f: + f.write("rm me pls") + os.chmod(sub, 0) + # If the test fails, make sure everything can still be removed. + self.addCleanup(d.cleanup) + @self.addCleanup + def _clean(): + if os.path.isdir(sub): + os.chmod(sub, 0o700) + return work, d, subsub_stat + + @os_helper.skip_unless_working_chmod + @support.requires_non_root_user + @unittest.skipUnless(tempfile._rmtree_use_dir_fd + and shutil.rmtree.avoids_symlink_attacks, + 'requires the fd based cleanup') + def test_cleanup_nested_toplevel_error(self): + # rmtree() reports errors relative to the correct parent directory + work, d, subsub_stat = self._make_unopenable_subdir() + # An unrelated directory with the same name; should not be touched + unrelated = os.path.join(work, 'sub') + os.mkdir(unrelated) + + rmdir = os.rmdir + failed_path = None + def failing_rmdir(path, *, dir_fd=None): + nonlocal failed_path + if path == 'sub' and dir_fd is not None and failed_path is None: + failed_path = path + raise PermissionError(errno.EPERM, 'simulated', path) + return rmdir(path, dir_fd=dir_fd) + with os_helper.change_cwd(work), mock.patch('os.rmdir', failing_rmdir): + d.cleanup() + self.assertEqual(failed_path, 'sub') + self.assertFalse(os.path.exists(d.name)) + self.assertTrue(os.path.exists(unrelated)) + + @os_helper.skip_unless_working_chmod + @support.requires_non_root_user + @unittest.skipUnless(tempfile._rmtree_use_dir_fd + and shutil.rmtree.avoids_symlink_attacks, + 'requires the fd based cleanup') + def test_cleanup_nested_scandir_error(self): + # A scandir() error is reported relative to the parent directory. + work, d, subsub_stat = self._make_unopenable_subdir() + + scandir = os.scandir + failed = False + def failing_scandir(path='.'): + nonlocal failed + if ( + not failed + and isinstance(path, int) + and os.path.samestat(os.fstat(path), subsub_stat) + ): + failed = True + raise PermissionError(errno.EACCES, 'simulated') + return scandir(path) + with os_helper.change_cwd(work), mock.patch('os.scandir', failing_scandir): + d.cleanup() + self.assertTrue(failed) + self.assertFalse(os.path.exists(d.name)) + @support.cpython_only def test_del_on_collection(self): # A TemporaryDirectory is deleted when garbage collected @@ -2068,11 +2166,17 @@ def test_context_manager(self): self.assertEqual(name, d.name) self.assertFalse(os.path.exists(name)) - def test_modes(self): + @support.subTests('bytes_name', (False, True)) + def test_modes(self, bytes_name): for mode in range(8): mode <<= 6 with self.subTest(mode=format(mode, '03o')): - d = self.do_create(recurse=3, dirs=2, files=2) + if bytes_name: + d = self.do_create(recurse=3, dirs=2, files=2, + dir=tempfile.gettempdirb()) + else: + d = self.do_create(recurse=3, dirs=2, files=2) + self.assertIsInstance(d.name, bytes if bytes_name else str) with d: # Change files and directories mode recursively. for root, dirs, files in os.walk(d.name, topdown=False): @@ -2083,13 +2187,17 @@ def test_modes(self): self.assertFalse(os.path.exists(d.name)) @support.subTests('ignore_errors', (True, False)) - def test_parent_mode_preserved(self, ignore_errors): + @support.subTests('bytes_name', (False, True)) + def test_parent_mode_preserved(self, ignore_errors, bytes_name): # Test that cleanup does not touch the parent directory, # even if that prevents removal. for mode in range(8): mode <<= 6 with self.subTest(mode=format(mode, '03o')): - outer = self.do_create() + if bytes_name: + outer = self.do_create(dir=tempfile.gettempdirb()) + else: + outer = self.do_create() with outer: d = self.do_create(dir=outer.name, dirs=2, files=2, ignore_cleanup_errors=ignore_errors) diff --git a/Misc/NEWS.d/next/Library/2026-11-16-16-10-42.gh-issue-157579.6choQ_.rst b/Misc/NEWS.d/next/Library/2026-11-16-16-10-42.gh-issue-157579.6choQ_.rst new file mode 100644 index 000000000000000..1bd89a77b5995c1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-16-16-10-42.gh-issue-157579.6choQ_.rst @@ -0,0 +1,3 @@ +The error handler :class:`~tempfile.TemporaryDirectory` uses to clean up +unreadable and inaccessible files no longer fails for temporary directories +created using :class:`bytes` paths. From b107dc253f2e788693d922e76c74501c91f7d42e Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 18 Nov 2026 16:46:32 +0300 Subject: [PATCH 184/229] gh-161660: Fix `test_json` with `PYTHON_LAZY_IMPORTS=all` (#161866) --- Lib/test/lazy_imports_all_exclude.txt | 2 -- Lib/test/support/import_helper.py | 8 +++++++- Lib/test/test_type_annotations.py | 6 +++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Lib/test/lazy_imports_all_exclude.txt b/Lib/test/lazy_imports_all_exclude.txt index e5519323e88e38a..f85ea5e4f301c04 100644 --- a/Lib/test/lazy_imports_all_exclude.txt +++ b/Lib/test/lazy_imports_all_exclude.txt @@ -20,10 +20,8 @@ test.test_inspect.test_inspect test___all__ test__interpreters test_crossinterp -test_datetime test_import test_importlib -test_json test_pkg test_profile test_profiling diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 962f65d8dcd14a2..26790c6995e7810 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -156,6 +156,7 @@ def multi_interp_extensions_check(enabled=True): def import_fresh_module(name, fresh=(), blocked=(), *, deprecated=False, usefrozen=False, + eager_imports=True, ): """Import and return a module, deliberately bypassing sys.modules. @@ -187,7 +188,12 @@ def import_fresh_module(name, fresh=(), blocked=(), *, """ # NOTE: test_heapq, test_json and test_warnings include extra sanity checks # to make sure that this utility function is working as expected - with _ignore_deprecated_imports(deprecated): + eager_imports_context = ( + force_eager_imports + if eager_imports + else contextlib.nullcontext + ) + with eager_imports_context(), _ignore_deprecated_imports(deprecated): # Keep track of modules saved for later restoration as well # as those which just need a blocking entry removed fresh = list(fresh) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 879a571d06b4a03..02f3e8b9caddb4c 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -153,7 +153,11 @@ class D(metaclass=C): self.assertEqual(D.__annotations__, {}) def test_partially_executed_module(self): - partialexe = import_helper.import_fresh_module("test.typinganndata.partialexecution") + # Results are different with and without lazy imports, we need both: + partialexe = import_helper.import_fresh_module( + "test.typinganndata.partialexecution", + eager_imports=False, + ) self.assertEqual( partialexe.a.__annotations__, {"v1": int, "v2": int}, From 355486e62b8c6aa06ad5858bede24f464220a097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Wed, 18 Nov 2026 16:01:39 +0100 Subject: [PATCH 185/229] gh-149584: Read remote stack chunks without the page cache (GH-160888) Read stack chunks directly in the sampling profiler. Avoid the page cache retry that can read a replacement chunk after the target process reuses the same address. Add a NEWS entry for the change. --- .../Library/2026-11-18-12-18-53.gh-issue-149584.rDchNk.rst | 3 +++ Modules/_remote_debugging/frames.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-18-12-18-53.gh-issue-149584.rDchNk.rst diff --git a/Misc/NEWS.d/next/Library/2026-11-18-12-18-53.gh-issue-149584.rDchNk.rst b/Misc/NEWS.d/next/Library/2026-11-18-12-18-53.gh-issue-149584.rDchNk.rst new file mode 100644 index 000000000000000..2d0e9d203cf6b3e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-18-12-18-53.gh-issue-149584.rDchNk.rst @@ -0,0 +1,3 @@ +Read stack chunks directly in :mod:`profiling.sampling`, without the page +cache. This removes a redundant retry on macOS that can read a replacement +chunk after the target process reuses the same address. diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index 5df309109fcfe35..da2d9f865e536f1 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -36,7 +36,7 @@ process_single_stack_chunk( return -1; } - if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, chunk_addr, current_size, this_chunk) < 0) { + if (_Py_RemoteDebug_ReadRemoteMemory(&unwinder->handle, chunk_addr, current_size, this_chunk) < 0) { PyMem_RawFree(this_chunk); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read stack chunk"); return -1; @@ -65,7 +65,7 @@ process_single_stack_chunk( } this_chunk = tmp; - if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, chunk_addr, actual_size, this_chunk) < 0) { + if (_Py_RemoteDebug_ReadRemoteMemory(&unwinder->handle, chunk_addr, actual_size, this_chunk) < 0) { PyMem_RawFree(this_chunk); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to reread stack chunk with correct size"); return -1; From e59260b1c0165567438211082a181fb7b74b385a Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Wed, 18 Nov 2026 16:56:11 +0100 Subject: [PATCH 186/229] gh-100687: Reduce frequency of overallocation in `x_add()` (#158002) * gh-100687: Reduce frequency of overallocation in x_add() x_add() allocated a digit for the carry out of the top digit on every call. In most cases that digit is zero and long_normalize() drops it from the digit count, but the allocation keeps its size, so the result uses more memory than its value needs for as long as it lives. Allocate the carry digit only when the top digits of the operands sum to at least PyLong_MASK, the only case in which a carry out of the top digit is possible. When no carry digit was allocated the top digit of the result is at least the top digit of the larger operand, so the result is already normalized and is returned directly. This applies to same-sign additions and opposite-sign subtractions, which go through x_add(); x_sub() is unchanged. Extends PR 100688 by Mark Dickinson, which contributed the size test and the tests. Co-authored-by: Mark Dickinson Co-Authored-By: Claude Fable 5.1 * Apply batched suggestions from code review Co-authored-by: Stan Ulbrych --------- Co-authored-by: Mark Dickinson Co-authored-by: Claude Fable 5.1 Co-authored-by: Stan Ulbrych --- Lib/test/test_long.py | 14 ++++++++++++++ ...23-01-02-14-54-12.gh-issue-100687.yenIag.rst | 2 ++ Objects/longobject.c | 17 ++++++++++++++--- 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2023-01-02-14-54-12.gh-issue-100687.yenIag.rst diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index c16bf1b1fe020e9..ce72c256774d77a 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -1734,6 +1734,20 @@ class MyInt(int): # GH-117195 -- This shouldn't crash object.__sizeof__(1) + def test_long_add_overallocate(self): + # see gh-100687 + x = (MASK//2) * (MASK+1) + x2 = (MASK//2 + 1) * (MASK+1) + self.assertEqual(x + x2, MASK * (MASK + 1)) + + def test_karatsuba_single_digit_parts(self): + # gh-100687: k_mul() adds the halves of its operands with x_add(), + # and those halves can be single digits. + a = 1 + (1 << (SHIFT * 70)) + b = 1 << (SHIFT * 139) + self.assertEqual(a * b, (1 << (SHIFT * 139)) + (1 << (SHIFT * 209))) + self.assertEqual(b * a, a * b) + def test_hash(self): # gh-136599 self.assertEqual(hash(-1), -2) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2023-01-02-14-54-12.gh-issue-100687.yenIag.rst b/Misc/NEWS.d/next/Core_and_Builtins/2023-01-02-14-54-12.gh-issue-100687.yenIag.rst new file mode 100644 index 000000000000000..f4f63584b1659ea --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2023-01-02-14-54-12.gh-issue-100687.yenIag.rst @@ -0,0 +1,2 @@ +Reduce the memory used by the result of adding two large :class:`int` +values. Such additions are also slightly faster for most sizes. diff --git a/Objects/longobject.c b/Objects/longobject.c index 09bfe4ad4641281..4a3117c43de5e12 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3755,7 +3755,13 @@ x_add(PyLongObject *a, PyLongObject *b) size_a = size_b; size_b = size_temp; } } - z = long_alloc(size_a+1); + assert(size_a >= 1); + /* A carry out of the top digit is only possible if the top digits sum + to at least PyLong_MASK; only then allocate a digit for it. */ + digit top_sum = a->long_value.ob_digit[size_a - 1] + + (size_b == size_a ? b->long_value.ob_digit[size_b - 1] : 0); + int extra_digit = top_sum >= PyLong_MASK; + z = long_alloc(size_a + extra_digit); if (z == NULL) return NULL; for (i = 0; i < size_b; ++i) { @@ -3768,8 +3774,13 @@ x_add(PyLongObject *a, PyLongObject *b) z->long_value.ob_digit[i] = carry & PyLong_MASK; carry >>= PyLong_SHIFT; } - z->long_value.ob_digit[i] = carry; - return long_normalize(z); + if (extra_digit) { + z->long_value.ob_digit[i] = carry; + return long_normalize(z); + } + assert(carry == 0); + assert(z->long_value.ob_digit[i - 1] != 0); + return z; } /* Subtract the absolute values of two integers. */ From ddd88c74b08f253bd9d2a05528050965c75350fa Mon Sep 17 00:00:00 2001 From: Faris Basha TM <72191505+farisbasha@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:52:03 +0530 Subject: [PATCH 187/229] Replace stale signature lines in `getopt` docstrings with summaries (#162202) Co-authored-by: Stan Ulbrych --- Lib/getopt.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Lib/getopt.py b/Lib/getopt.py index 25f3e2439b3e352..272a4bf7d0c2820 100644 --- a/Lib/getopt.py +++ b/Lib/getopt.py @@ -47,11 +47,10 @@ def __str__(self): error = GetoptError # backward compatibility def getopt(args, shortopts, longopts = []): - """getopt(args, options[, long_options]) -> opts, args + """Parse command line options and parameter list. - Parses command line options and parameter list. args is the - argument list to be parsed, without the leading reference to the - running program. Typically, this means "sys.argv[1:]". shortopts + args is the argument list to be parsed, without the leading reference + to the running program. Typically, this means "sys.argv[1:]". shortopts is the string of option letters that the script wants to recognize, with options that require an argument followed by a colon and options that accept an optional argument followed by @@ -72,7 +71,6 @@ def getopt(args, shortopts, longopts = []): string if the option has no argument. The options occur in the list in the same order in which they were found, thus allowing multiple occurrences. Long and short options may be mixed. - """ opts = [] @@ -92,7 +90,7 @@ def getopt(args, shortopts, longopts = []): return opts, args def gnu_getopt(args, shortopts, longopts = []): - """getopt(args, options[, long_options]) -> opts, args + """Parse command line options using GNU style scanning. This function works like getopt(), except that GNU style scanning mode is used by default. This means that option and non-option @@ -103,7 +101,6 @@ def gnu_getopt(args, shortopts, longopts = []): If the first character of the option string is '+', or if the environment variable POSIXLY_CORRECT is set, then option processing stops as soon as a non-option argument is encountered. - """ opts = [] From 59a7a46544184935a1b547e341244d9397d276de Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Nov 2026 22:01:23 +0100 Subject: [PATCH 188/229] gh-162214: Test Py_GIL_DISABLED macro using #ifdef (#162223) Replace "#if Py_GIL_DISABLED" with "#ifdef Py_GIL_DISABLED" to fix compiler warnings when building Python with -Wundef. Example of fixed warning: Objects/unicodeobject.c:14740:5: warning: 'Py_GIL_DISABLED' is not defined, evaluates to '0' [-Wundef] 14740 | #if Py_GIL_DISABLED | ^~~~~~~~~~~~~~~ Simplify also Py_GIL_DISABLED == 0 test in PC/pyconfig.h. --- Modules/_testinternalcapi/test_cases.c.h | 4 ++-- Objects/unicodeobject.c | 2 +- PC/pyconfig.h | 4 +--- Python/bytecodes.c | 4 ++-- Python/executor_cases.c.h | 4 ++-- Python/generated_cases.c.h | 4 ++-- Python/lock.c | 2 +- Python/modsupport.c | 2 +- 8 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 8e7ba1f69553dda..cfa8faa15f4faea 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -10120,7 +10120,7 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - #if Py_GIL_DISABLED + #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); @@ -10187,7 +10187,7 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - #if Py_GIL_DISABLED + #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0fe765c36848f25..5b9e8985d1bea8e 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14737,7 +14737,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, /* Statically allocated strings must be already interned. */ assert(!_PyUnicode_STATE(s).statically_allocated); -#if Py_GIL_DISABLED +#ifdef Py_GIL_DISABLED /* In the free-threaded build, all interned strings are immortal */ immortalize = 1; #endif diff --git a/PC/pyconfig.h b/PC/pyconfig.h index f969525c1db5c7c..9de3786a0c769b4 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -98,17 +98,15 @@ WIN32 is still required for the locale module. #endif /* Define to 1 when compiling for experimental free-threaded builds */ -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) && Py_GIL_DISABLED == 0 /* We undefine if it was set to zero because all later checks are #ifdef. * Note that non-Windows builds do not do this, and so every effort should * be made to avoid defining the variable at all when not desired. However, * sysconfig.get_config_var always returns a 1 or a 0, and so it seems likely * that a build backend will define it with the value. */ -#if Py_GIL_DISABLED == 0 #undef Py_GIL_DISABLED #endif -#endif /* Compiler specific defines */ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 42ad83daa1ea461..bd5debcd01e9bfd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2278,7 +2278,7 @@ dummy_func( assert(index < DK_SIZE(keys)); PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); DEOPT_IF(res_o == NULL); - #if Py_GIL_DISABLED + #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); DEOPT_IF(!increfed); #else @@ -2297,7 +2297,7 @@ dummy_func( PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); DEOPT_IF(res_o == NULL); - #if Py_GIL_DISABLED + #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); DEOPT_IF(!increfed); #else diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bc85ccd9ddb4fc1..58473deddcebe3c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -10440,7 +10440,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - #if Py_GIL_DISABLED + #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UOP_STAT_INC(uopcode, miss); @@ -10485,7 +10485,7 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } - #if Py_GIL_DISABLED + #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UOP_STAT_INC(uopcode, miss); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ee9b64107bb9f24..e5b3548b66412cd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -10118,7 +10118,7 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - #if Py_GIL_DISABLED + #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); @@ -10185,7 +10185,7 @@ assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); JUMP_TO_PREDICTED(LOAD_GLOBAL); } - #if Py_GIL_DISABLED + #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); diff --git a/Python/lock.c b/Python/lock.c index a2d47eea64a006b..59dfe772a0365f1 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -25,7 +25,7 @@ static const PyTime_t TIME_TO_BE_FAIR_NS = 1000*1000; // Spin for a bit before parking the thread. This is only enabled for // `--disable-gil` builds because it is unlikely to be helpful if the GIL is // enabled. -#if Py_GIL_DISABLED +#ifdef Py_GIL_DISABLED static const int MAX_SPIN_COUNT = 40; #else static const int MAX_SPIN_COUNT = 0; diff --git a/Python/modsupport.c b/Python/modsupport.c index bab21d1b2be5b51..f944b1c05ba9889 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -767,7 +767,7 @@ int PyABIInfo_Check(PyABIInfo *info, const char *module_name) /* Free-threading/GIL */ uint16_t gilflags = info->flags & (PyABIInfo_GIL | PyABIInfo_FREETHREADED); -#if Py_GIL_DISABLED +#ifdef Py_GIL_DISABLED if (gilflags == PyABIInfo_GIL) { return _abiinfo_raise(module_name, "incompatible with free-threaded CPython"); From fce90ca6d1a90b40c94694991bb200fb26355bdd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Nov 2026 22:08:34 +0100 Subject: [PATCH 189/229] gh-162214: Test Py_STATS macro using #ifdef (#162225) Replace "#if Py_STATS" with "#ifdef Py_STATS". All other C files already check for "#ifdef Py_STATS". Example of fixed compiler warning when building Python with -Wundef: Python/specialize.c:33:5: warning: 'Py_STATS' is not defined, evaluates to '0' [-Wundef] 33 | #if Py_STATS | ^~~~~~~~ On Windows, the Py_STATS macro is now defined to 1 when it's defined. --- PCbuild/pyproject.props | 2 +- Python/pystate.c | 2 +- Python/specialize.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index ca8330c004cccea..67296c22806c3e4 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -52,7 +52,7 @@ <_DebugPreprocessorDefinition>NDEBUG; <_DebugPreprocessorDefinition Condition="$(Configuration) == 'Debug'">_DEBUG; <_PyStatsPreprocessorDefinition>PyStats; - <_PyStatsPreprocessorDefinition Condition="$(PySTATS) != ''">Py_STATS; + <_PyStatsPreprocessorDefinition Condition="$(PySTATS) != ''">Py_STATS=1; <_PlatformPreprocessorDefinition>_WIN32; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition) diff --git a/Python/pystate.c b/Python/pystate.c index ce7197c4e054283..4c95669f9fa1b09 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2294,7 +2294,7 @@ tstate_deactivate(PyThreadState *tstate) assert(tstate_is_bound(tstate)); assert(tstate->_status.active); -#if Py_STATS +#ifdef Py_STATS _PyStats_Detach((_PyThreadStateImpl *)tstate); #endif diff --git a/Python/specialize.c b/Python/specialize.c index 067bae59f6073b7..b5cf4ebc31142ab 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -30,7 +30,7 @@ * InternalDocs/interpreter.md `Specialization` section. */ -#if Py_STATS +#ifdef Py_STATS #define SPECIALIZATION_FAIL(opcode, kind) \ do { \ PyStats *s = _PyStats_GET(); \ @@ -42,7 +42,7 @@ do { \ } while (0) #else # define SPECIALIZATION_FAIL(opcode, kind) ((void)0) -#endif // Py_STATS +#endif // !Py_STATS static void fixup_getiter(_Py_CODEUNIT *instruction, int flags) From d7d41e27086a34fa552e48c36c48317ed7bdb371 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Nov 2026 22:12:39 +0100 Subject: [PATCH 190/229] gh-162214: Use #ifdef to test USING_APPLE_OS_LIBFFI macro in ctypes.h (#162221) The macro is already tested by "#if defined(USING_APPLE_OS_LIBFFI)" in other ctypes C files. --- Modules/_ctypes/ctypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 248559aa364a198..98d8b660850b0d9 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -24,7 +24,7 @@ // Do we support C99 complex types in ffi? // For Apple's libffi, this must be determined at runtime (see gh-128156). #if defined(_Py_FFI_SUPPORT_C_COMPLEX) -# if USING_APPLE_OS_LIBFFI && defined(__has_builtin) +# if defined(USING_APPLE_OS_LIBFFI) && defined(__has_builtin) # if __has_builtin(__builtin_available) # define Py_FFI_COMPLEX_AVAILABLE __builtin_available(macOS 10.15, *) # else From fb9330bbacb39c847bd3a2d5437836a04e10ad14 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Wed, 18 Nov 2026 13:13:50 -0800 Subject: [PATCH 191/229] gh-136267: Use the target configuration in build-details.json (#161529) With --with-build-python, generate-build-details.py runs on a different interpreter than the one being built. abi.flags, suffixes.extensions, abi.stable_abi_suffix and supports_isolated_interpreters came from sys and importlib, so they described the build interpreter. Read them from sysconfig instead. The extension suffix list is rebuilt the same way Python/dynload_shlib.c builds it; a test checks that the result matches importlib.machinery.EXTENSION_SUFFIXES on native builds. Also set abi.extension_suffix and abi.stable_abi_suffix for static libpython builds. They were only set with a shared libpython, but extension modules work either way and PEP 739 requires them. Co-authored-by: Claude Fable 5.1 --- Lib/test/test_build_details.py | 88 ++++++++++++++++--- ...-11-06-12-00-01.gh-issue-136267.Qv7mRs.rst | 8 ++ Tools/build/generate-build-details.py | 59 +++++++++++-- 3 files changed, 137 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-11-06-12-00-01.gh-issue-136267.Qv7mRs.rst diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index 12b7664d83aae16..552c7252c32d049 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -142,6 +142,17 @@ def contents(self): def test_location(self): self.assertTrue(os.path.isfile(self.location)) + def test_abi(self): + self.assertEqual(self.key('abi.flags'), list(sys.abiflags)) + self.assertEqual(self.key('abi.extension_suffix'), + sysconfig.get_config_var('EXT_SUFFIX')) + self.assertIn(self.key('abi.stable_abi_suffix'), + importlib.machinery.EXTENSION_SUFFIXES) + + def test_extension_suffixes(self): + self.assertEqual(self.key('suffixes.extensions'), + importlib.machinery.EXTENSION_SUFFIXES) + # Override generic format tests with tests for our specific implemenation. @needs_installed_python @@ -289,6 +300,62 @@ def check_relative_paths(self, data_abs_path, data_rel_path): self.assertEqual(data_abs_path[part][key], abs_rel_path) +def generate_data(**config_vars): + """Generate build-details data with overridden config vars.""" + get_config_var = sysconfig.get_config_var + def patched_get_config_var(name): + if name in config_vars: + return config_vars[name] + return get_config_var(name) + with mock.patch('sysconfig.get_config_var', patched_get_config_var): + data = generate_build_details.generate_data(schema_version='1.0') + return json.loads(json.dumps(data)) + + +@unittest.skipIf( + generate_build_details is None, + "Failed to import generate-build-details" +) +@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX') +class BuildDetailsTargetTests(unittest.TestCase): + """Values that must come from the target configuration (gh-136267).""" + + def test_native_extension_suffixes(self): + self.assertEqual(generate_build_details.get_extension_suffixes(), + importlib.machinery.EXTENSION_SUFFIXES) + + def test_free_threaded_debug(self): + # A GIL-enabled release build Python generating for this target + # used to report its own flags and suffixes. + data = generate_data( + ABIFLAGS='td', MACHDEP='linux', DYNLOADFILE='dynload_shlib.o', + SOABI_PLATFORM='aarch64-linux-gnu', + SOABI='cpython-3XXtd-aarch64-linux-gnu', + ALT_SOABI='cpython-3XXt-aarch64-linux-gnu', + EXT_SUFFIX='.cpython-3XXtd-aarch64-linux-gnu.so', + Py_GIL_DISABLED=1) + self.assertEqual(data['abi']['flags'], ['t', 'd']) + self.assertEqual(data['abi']['extension_suffix'], + '.cpython-3XXtd-aarch64-linux-gnu.so') + self.assertEqual(data['abi']['stable_abi_suffix'], + '.abi3t-aarch64-linux-gnu.so') + self.assertEqual(data['suffixes']['extensions'], [ + '.cpython-3XXtd-aarch64-linux-gnu.so', + '.cpython-3XXt-aarch64-linux-gnu.so', + '.abi3t-aarch64-linux-gnu.so', + '.abi3t.so', + '.so', + ]) + + def test_no_dynamic_loading(self): + data = generate_data(MACHDEP='wasi', DYNLOADFILE='dynload_stub.o') + self.assertEqual(data['suffixes']['extensions'], []) + self.assertNotIn('extension_suffix', data['abi']) + self.assertNotIn('stable_abi_suffix', data['abi']) + self.assertFalse( + data['implementation']['supports_isolated_interpreters']) + + @unittest.skipIf( generate_build_details is None, "Failed to import generate-build-details" @@ -299,26 +366,25 @@ class BuildDetailsLibpythonTests(unittest.TestCase): LIBDIR = '/prefix/lib' LIBPL = '/prefix/lib/python3.X/config-3.X' + def generate(self, **config_vars): + return generate_data(LIBDIR=self.LIBDIR, LIBPL=self.LIBPL, + LIBRARY='libpython3.X.a', **config_vars) + def generate_libpython(self, **config_vars): """Generate the libpython section with overridden config vars.""" - config_vars = dict(LIBDIR=self.LIBDIR, LIBPL=self.LIBPL, - LIBRARY='libpython3.X.a', **config_vars) - get_config_var = sysconfig.get_config_var - def patched_get_config_var(name): - if name in config_vars: - return config_vars[name] - return get_config_var(name) - with mock.patch('sysconfig.get_config_var', patched_get_config_var): - data = generate_build_details.generate_data(schema_version='1.0') - return json.loads(json.dumps(data)).get('libpython', {}) + return self.generate(**config_vars).get('libpython', {}) def test_static_only(self): - libpython = self.generate_libpython( + data = self.generate( LDLIBRARY='libpython3.X.a', STATIC_LIBPYTHON=1, PYTHONFRAMEWORK='') + libpython = data['libpython'] self.assertNotIn('dynamic', libpython) self.assertEqual(libpython['static'], os.path.join(self.LIBDIR, 'libpython3.X.a')) + # Extension modules are supported without a shared libpython. + self.assertIn('extension_suffix', data['abi']) + self.assertIn('stable_abi_suffix', data['abi']) def test_shared_and_static(self): # The static library is only installed into LIBPL, not LIBDIR. diff --git a/Misc/NEWS.d/next/Build/2026-11-06-12-00-01.gh-issue-136267.Qv7mRs.rst b/Misc/NEWS.d/next/Build/2026-11-06-12-00-01.gh-issue-136267.Qv7mRs.rst new file mode 100644 index 000000000000000..43cc054a04ac7a2 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-11-06-12-00-01.gh-issue-136267.Qv7mRs.rst @@ -0,0 +1,8 @@ +``build-details.json`` now describes the interpreter being built rather than +the interpreter running :program:`generate-build-details.py`: ``abi.flags``, +``suffixes.extensions``, ``abi.stable_abi_suffix`` and +``implementation.supports_isolated_interpreters`` are derived from the +target's configuration, which fixes cross builds and builds that use +:option:`--with-build-python`. ``abi.extension_suffix`` and +``abi.stable_abi_suffix`` are now also present when libpython is only built +as a static library. diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index 4fb49bf139b225f..f6cb93beabec43d 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -29,6 +29,46 @@ def get_dict_key(container: dict[str, Any], key: str) -> dict[str, Any]: return container +def get_extension_suffixes() -> list[str]: + """Return the extension module suffixes of the target interpreter. + + importlib.machinery.EXTENSION_SUFFIXES describes the running interpreter, + which is not the target under --with-build-python. On POSIX, rebuild the + list from the target configuration instead, following + _PyImport_DynLoadFiletab in Python/dynload_*.c. The two are kept in sync + by test_build_details.BuildDetailsTargetTests.test_native_extension_suffixes. + """ + if os.name != 'posix': + return importlib.machinery.EXTENSION_SUFFIXES + + dynloadfile = sysconfig.get_config_var('DYNLOADFILE') + if dynloadfile == 'dynload_stub.o': + return [] + if dynloadfile == 'dynload_hpux.o': + return [sysconfig.get_config_var('EXT_SUFFIX'), '.sl'] + if dynloadfile != 'dynload_shlib.o': + raise ValueError(f'Unsupported DYNLOADFILE: {dynloadfile!r}') + + soabi = sysconfig.get_config_var('SOABI') + if sysconfig.get_config_var('MACHDEP') == 'cygwin': + return [f'.{soabi}.dll', '.dll'] + + suffixes = [f'.{soabi}.so'] + alt_soabi = sysconfig.get_config_var('ALT_SOABI') + if alt_soabi: + suffixes.append(f'.{alt_soabi}.so') + platform = sysconfig.get_config_var('SOABI_PLATFORM') + if not sysconfig.get_config_var('Py_GIL_DISABLED'): + if platform: + suffixes.append(f'.abi3-{platform}.so') + suffixes.append('.abi3.so') + if platform: + suffixes.append(f'.abi3t-{platform}.so') + suffixes.append('.abi3t.so') + suffixes.append('.so') + return suffixes + + def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: """Generate the build-details.json data (PEP 739). @@ -62,14 +102,20 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: # Fix cross-compilation if '_multiarch' in data['implementation']: data['implementation']['_multiarch'] = sysconfig.get_config_var('MULTIARCH') + if 'supports_isolated_interpreters' in data['implementation']: + # Mirrors Python/sysmodule.c. + data['implementation']['supports_isolated_interpreters'] = ( + sysconfig.get_config_var('MACHDEP') not in ('wasi', 'emscripten') + ) - data['abi']['flags'] = list(sys.abiflags) + data['abi']['flags'] = list(sysconfig.get_config_var('ABIFLAGS')) data['suffixes']['source'] = importlib.machinery.SOURCE_SUFFIXES data['suffixes']['bytecode'] = importlib.machinery.BYTECODE_SUFFIXES #data['suffixes']['optimized_bytecode'] = importlib.machinery.OPTIMIZED_BYTECODE_SUFFIXES #data['suffixes']['debug_bytecode'] = importlib.machinery.DEBUG_BYTECODE_SUFFIXES - data['suffixes']['extensions'] = importlib.machinery.EXTENSION_SUFFIXES + extension_suffixes = get_extension_suffixes() + data['suffixes']['extensions'] = extension_suffixes LIBDIR = sysconfig.get_config_var('LIBDIR') LIBPL = sysconfig.get_config_var('LIBPL') @@ -118,17 +164,16 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: # regardless. # # We only want to set abi.extension_suffix and stable_abi_suffix if - # extension modules are supported. - if has_dynamic_library: + # extension modules are supported, regardless of how libpython is built. + if extension_suffixes: data['abi']['extension_suffix'] = sysconfig.get_config_var('EXT_SUFFIX') - # EXTENSION_SUFFIXES has been constant for a long time, and currently we - # don't have a better information source to find the stable ABI suffix. - for suffix in importlib.machinery.EXTENSION_SUFFIXES: + for suffix in extension_suffixes: if suffix.startswith('.abi'): data['abi']['stable_abi_suffix'] = suffix break + if has_dynamic_library: data['libpython']['dynamic'] = os.path.join(dynamic_library_dir, LDLIBRARY) # FIXME: Not sure if windows has a different dll for the stable ABI, and # even if it does, currently we don't have a way to get its name. From 32197d8fd1b3199de54d54e2137482270292ef91 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Nov 2026 22:14:34 +0100 Subject: [PATCH 192/229] gh-162214: Define TARGET_OS_OSX macro in Modules/_testcapi/mem.c (#162220) Example of fixed compiler warning when building Python with -Wundef: Modules/_testcapi/mem.c:802:5: warning: 'TARGET_OS_OSX' is not defined, evaluates to '0' [-Wundef] 802 | #if TARGET_OS_OSX || defined(__FreeBSD__) --- Modules/_testcapi/mem.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index 257568172de8b19..822ae661944a24b 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -13,6 +13,8 @@ # include // proc_pidinfo(), PROC_PIDTASKINFO # include // struct proc_taskinfo # endif +#else +# define TARGET_OS_OSX 0 #endif #ifdef __FreeBSD__ From f3c11aa2548d3af91cbe4c46f5380a914255e374 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Nov 2026 22:49:48 +0100 Subject: [PATCH 193/229] gh-162214: Define STRINGLIB_IS_UNICODE macro in bytes_methods.c (#162222) Example of fixed compiler warnings when Python is built with -Wundef: In file included from Objects/bytes_methods.c:386: Objects/stringlib/count.h:10:6: warning: 'STRINGLIB_IS_UNICODE' is not defined, evaluates to '0' [-Wundef] 10 | #if !STRINGLIB_IS_UNICODE || STRINGLIB_MAX_CHAR > 0x7Fu --- Objects/bytes_methods.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index 414afeb7bb003c7..4021a0e82236942 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -377,6 +377,7 @@ _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to) } #define FASTSEARCH fastsearch +#define STRINGLIB_IS_UNICODE 0 #define STRINGLIB(F) stringlib_##F #define STRINGLIB_CHAR char #define STRINGLIB_SIZEOF_CHAR 1 From abbfbce57bc24afc72704616b14d29a380ccdda7 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 18 Nov 2026 15:13:17 -0800 Subject: [PATCH 194/229] gh-162182: Fix c-analyzer on static_assert containing a comma (#162186) With -std=c99, glibc expands `static_assert(cond, msg)` to a declaration that has cond in a struct bitfield size. For example static_assert(offsetof(T, m) >= 1, "msg"); is preprocessed to extern int (*__Static_assert_function (void)) [!!sizeof (struct { int __error_if_negative: (__builtin_offsetof (T, m) >= 1) ? 2 : -1; })]; The parser ended the size at the first comma, the one in "(T, m)", which left it out of sync and `make check-c-globals` failed with "too many lines". Let a bitfield size containing parentheses run to the ";". Co-authored-by: Claude Fable 5.2 --- Tools/c-analyzer/c_parser/parser/_regexes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tools/c-analyzer/c_parser/parser/_regexes.py b/Tools/c-analyzer/c_parser/parser/_regexes.py index c1a8ab3ad2f15da..65fe2f929a942e3 100644 --- a/Tools/c-analyzer/c_parser/parser/_regexes.py +++ b/Tools/c-analyzer/c_parser/parser/_regexes.py @@ -340,7 +340,10 @@ def _ind(text, level=1, edges='both'): # This is actually a "constant expression". \d+ | - [^'",}}]+ + [^'",(}}]+ + | + # Parens may hold commas (e.g. from offsetof()). + [^'";}}]+ (?= ; ) ) )? \s* From 94250a7c0f6cbf51ff6b13873d1e611e417061af Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 19 Nov 2026 00:52:05 +0100 Subject: [PATCH 195/229] gh-157695: Make HAVE_RL_CHANGE_ENVIRONMENT macro private (#162213) Rename HAVE_RL_CHANGE_ENVIRONMENT macro to _Py_HAVE_RL_CHANGE_ENVIRONMENT (make it private). --- Lib/test/test_readline.py | 2 +- Modules/readline.c | 2 +- Tools/build/check_capi_macros_ignored.txt | 1 - configure | 2 +- configure.ac | 2 +- pyconfig.h.in | 6 +++--- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index f999ea6cc1affdd..074faf94d2d792d 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -471,7 +471,7 @@ def test_write_read_limited_history(self): # So, we've only tested that the read did not fail. # See TestHistoryManipulation for the full test. - @unittest.skipUnless(sysconfig.get_config_var("HAVE_RL_CHANGE_ENVIRONMENT"), + @unittest.skipUnless(sysconfig.get_config_var("_Py_HAVE_RL_CHANGE_ENVIRONMENT"), "readline can modify the environment") def test_environment_is_not_modified(self): # os.environ contains environment at the time "os" module was loaded, so diff --git a/Modules/readline.c b/Modules/readline.c index 643b1fcb040290d..fab0d9aeaa80aeb 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1543,7 +1543,7 @@ setup_readline(readlinestate *mod_state) /* The name must be defined before initialization */ rl_readline_name = "python"; -#ifdef HAVE_RL_CHANGE_ENVIRONMENT +#ifdef _Py_HAVE_RL_CHANGE_ENVIRONMENT /* Prevent readline from setting the LINES and COLUMNS environment * variables: ncurses prefers them over an ioctl() query, so a stale value * left after a resize breaks SIGWINCH / KEY_RESIZE handling (gh-46927). */ diff --git a/Tools/build/check_capi_macros_ignored.txt b/Tools/build/check_capi_macros_ignored.txt index 73d4f7be0766b9f..dfde88c0c800d6c 100644 --- a/Tools/build/check_capi_macros_ignored.txt +++ b/Tools/build/check_capi_macros_ignored.txt @@ -435,7 +435,6 @@ HAVE_RECVFROM HAVE_RENAMEAT HAVE_RL_APPEND_HISTORY HAVE_RL_CATCH_SIGNAL -HAVE_RL_CHANGE_ENVIRONMENT HAVE_RL_COMPDISP_FUNC_T HAVE_RL_COMPLETION_APPEND_CHARACTER HAVE_RL_COMPLETION_DISPLAY_MATCHES_HOOK diff --git a/configure b/configure index a2e9c1711adb794..655558774c4b5fa 100755 --- a/configure +++ b/configure @@ -30104,7 +30104,7 @@ printf "%s\n" "$ac_cv_readline_rl_change_environment" >&6; } then : -printf "%s\n" "#define HAVE_RL_CHANGE_ENVIRONMENT 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_RL_CHANGE_ENVIRONMENT 1" >>confdefs.h fi diff --git a/configure.ac b/configure.ac index 2be6d852c6e977a..f6f4055079eaa1b 100644 --- a/configure.ac +++ b/configure.ac @@ -7261,7 +7261,7 @@ AS_VAR_IF([with_readline], [no], [ ) ]) AS_VAR_IF([ac_cv_readline_rl_change_environment], [yes], [ - AC_DEFINE([HAVE_RL_CHANGE_ENVIRONMENT], [1], [Define if you have readline 6.3]) + AC_DEFINE([_Py_HAVE_RL_CHANGE_ENVIRONMENT], [1], [Define if you have readline 6.3]) ]) # check for readline 4.2 diff --git a/pyconfig.h.in b/pyconfig.h.in index 18534d7e80e3893..90d8a07df66083e 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1143,9 +1143,6 @@ /* Define if you can turn off readline's signal handling. */ #undef HAVE_RL_CATCH_SIGNAL -/* Define if you have readline 6.3 */ -#undef HAVE_RL_CHANGE_ENVIRONMENT - /* Define to 1 if the system has the type 'rl_compdisp_func_t'. */ #undef HAVE_RL_COMPDISP_FUNC_T @@ -2164,6 +2161,9 @@ /* Define if you have the 'pthread_attr_get_np' function. */ #undef _Py_HAVE_PTHREAD_ATTR_GET_NP +/* Define if you have readline 6.3 */ +#undef _Py_HAVE_RL_CHANGE_ENVIRONMENT + /* Define if you have the 'sinpi' function. */ #undef _Py_HAVE_SINPI From 391c7d704bede576c5993ecd15de30ec59eebd7b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:00:52 +0530 Subject: [PATCH 196/229] Update Emscripten libffi to 3.8.1 (#162188) Bumps the libffi version to incorporate recent memory handling improvements in libffi. --- Platforms/emscripten/config.toml | 4 ++-- Platforms/emscripten/make_libffi.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Platforms/emscripten/config.toml b/Platforms/emscripten/config.toml index 39835844bbb365f..68ac87371fb9030 100644 --- a/Platforms/emscripten/config.toml +++ b/Platforms/emscripten/config.toml @@ -17,8 +17,8 @@ pythoninfo-args = [ [dependencies.libffi] url = "https://github.com/libffi/libffi/releases/download/v{version}/libffi-{version}.tar.gz" -version = "3.4.6" -shasum = "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e" +version = "3.8.1" +shasum = "81e879c094cc5ec3186c48510c6385ac6b3e866192e8bc71a35245703757ee40" [dependencies.mpdec] url = "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{version}.tar.gz" diff --git a/Platforms/emscripten/make_libffi.sh b/Platforms/emscripten/make_libffi.sh index 3c75c4d51271025..c05e7c92904a811 100755 --- a/Platforms/emscripten/make_libffi.sh +++ b/Platforms/emscripten/make_libffi.sh @@ -1,7 +1,7 @@ #!/bin/bash set +e -export CFLAGS="-O2 -fPIC -DWASM_BIGINT" +export CFLAGS="-O2 -fPIC" export CXXFLAGS="$CFLAGS" # Build paths From 9e181c4267aa761ad96d18765037d4360df371af Mon Sep 17 00:00:00 2001 From: Pengyu Date: Thu, 19 Nov 2026 08:33:54 +0800 Subject: [PATCH 197/229] gh-160873: Remove obsolete Emscripten workarounds (#162192) Remove some more obsolete Emscripten workarounds. --- Lib/test/test_pathlib/test_pathlib.py | 7 +------ Platforms/emscripten/config.site-wasm32-emscripten | 5 +---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 10e3ad6d92daec8..8013b0df26ec81c 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -3065,12 +3065,7 @@ def test_glob_dotdot(self): else: # ".." segments are normalized first on Windows, so this path is stat()able. self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) - if sys.platform == "emscripten": - # Emscripten will return ELOOP if there are 49 or more ..'s. - # Can remove when https://github.com/emscripten-core/emscripten/pull/24591 is merged. - NDOTDOTS = 48 - else: - NDOTDOTS = 50 + NDOTDOTS = 50 self.assertEqual(set(p.glob("/".join([".."] * NDOTDOTS))), { P(self.base, *[".."] * NDOTDOTS)}) def test_glob_inaccessible(self): diff --git a/Platforms/emscripten/config.site-wasm32-emscripten b/Platforms/emscripten/config.site-wasm32-emscripten index 3c9a1c2c2568b0e..62640cc772d7e9b 100644 --- a/Platforms/emscripten/config.site-wasm32-emscripten +++ b/Platforms/emscripten/config.site-wasm32-emscripten @@ -19,7 +19,6 @@ ac_cv_func_posix_spawn=no ac_cv_func_posix_spawnp=no ac_cv_func_eventfd=no ac_cv_func_memfd_create=no -ac_cv_func_prlimit=no # The rest is based on pyodide # https://github.com/pyodide/pyodide/blob/main/cpython/pyconfig.undefs.h @@ -45,8 +44,6 @@ ac_cv_func_seteuid=no ac_cv_func_setegid=no ac_cv_func_getresuid=no ac_cv_func_getresgid=no -ac_cv_func_setresuid=no -ac_cv_func_setresgid=no # Syscalls not implemented in emscripten # [Errno 52] Function not implemented @@ -89,7 +86,7 @@ ac_cv_func_setresuid=no ac_cv_func_setresgid=no # Emscripten does not support hard links, always fails with errno 34 -# "Too many links". See emscripten_syscall_stubs.c +# "Too many links". ac_cv_func_link=no ac_cv_func_linkat=no From aced381f5058206eb09ff7f48b44998cd4e8efcc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 19 Nov 2026 01:55:19 +0100 Subject: [PATCH 198/229] gh-161767: Fix refleak in PyDict_AsFrozenDictAndClear() (#162250) Py_DECREF() the frozendict if transfer_keys_and_values_lock_held() fails. --- Objects/dictobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ed8b53b8a08d495..4a1d1b069a21ed8 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -8717,6 +8717,7 @@ PyDict_AsFrozenDictAndClear(PyObject *dict) transfer = transfer_keys_and_values_lock_held(res, dict); Py_END_CRITICAL_SECTION(); if (transfer < 0) { + Py_DECREF(res); return NULL; } From 6c9f2b691aaf0cf88cf2c3ba862f5de237a17344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Thu, 19 Nov 2026 04:15:53 +0100 Subject: [PATCH 199/229] gh-149584: Stop rereading the base frame sentinel in _remote_debugging (#159124) Skip the remote read of the base frame sentinel while preserving the GC marker and complete-stack checks. --- .../2026-10-10-13-35-08.gh-issue-149584.Yq2bF7.rst | 2 ++ Modules/_remote_debugging/frame_cache.c | 5 ++--- Modules/_remote_debugging/frames.c | 8 ++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-10-13-35-08.gh-issue-149584.Yq2bF7.rst diff --git a/Misc/NEWS.d/next/Library/2026-10-10-13-35-08.gh-issue-149584.Yq2bF7.rst b/Misc/NEWS.d/next/Library/2026-10-10-13-35-08.gh-issue-149584.Yq2bF7.rst new file mode 100644 index 000000000000000..7f92a1b19877cd4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-10-13-35-08.gh-issue-149584.Yq2bF7.rst @@ -0,0 +1,2 @@ +Improve the performance of the Tachyon profiler by avoiding unnecessary remote +reads of the base frame sentinel. Patch by Maurycy Pawłowski-Wieroński. diff --git a/Modules/_remote_debugging/frame_cache.c b/Modules/_remote_debugging/frame_cache.c index 1b2d9a60f408b91..63d1331015a548d 100644 --- a/Modules/_remote_debugging/frame_cache.c +++ b/Modules/_remote_debugging/frame_cache.c @@ -318,9 +318,8 @@ frame_cache_store( // Validate we have a complete stack before caching. // Only cache if last_frame_visited matches base_frame_addr (the sentinel // at the bottom of the stack). Note: we use last_frame_visited rather than - // addrs[num_addrs-1] because the base frame is visited but not added to the - // addrs array (it returns frame==NULL from is_frame_valid due to - // owner==FRAME_OWNED_BY_INTERPRETER). + // addrs[num_addrs-1] because the base frame's address is not added to the + // addrs array. if (base_frame_addr != 0 && last_frame_visited != base_frame_addr) { // Incomplete stack - don't cache (graceful degradation) return 0; diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index da2d9f865e536f1..3f8d628a6314e6e 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -294,6 +294,14 @@ process_frame_chain( } assert(frame_count <= MAX_FRAME_CHAIN_DEPTH); + // The base frame is a sentinel with previous == NULL: reaching it is + // enough, no need to read it. Still read it when it is the GC frame + // so the marker gets emitted. + if (frame_addr == ctx->base_frame_addr + && !(unwinder->gc && frame_addr == ctx->gc_frame)) { + break; + } + if (ctx->chunks && ctx->chunks->count > 0) { parse_result = parse_frame_from_chunks( unwinder, &frame, frame_addr, &next_frame_addr, ctx->chunks); From 441c010fef90ab2860ae8142cbfd1fd52f53ba75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Thu, 19 Nov 2026 06:44:23 +0100 Subject: [PATCH 200/229] gh-162230: Relay `BufferedRWPair.readline()` to the reader (#162231) Call the internal reader's `readline()` in `_io` and `_pyio`. There is behaviour change. `BufferedRWPair.readline()` would behave like `BufferedReader.readline()`: in C, a non-blocking read with no data returns the partial line or `b''` instead of raising `OSError`, a closed pair always raises `ValueError`, and subclass overrides of `peek()` and `read()` would no longer apply. --- Lib/_pyio.py | 9 +++++ Lib/test/test_io/test_bufferedio.py | 14 +++++++ ...-11-18-18-28-37.gh-issue-162230.ezR86S.rst | 3 ++ Modules/_io/bufferedio.c | 20 ++++++++++ Modules/_io/clinic/bufferedio.c.h | 38 ++++++++++++++++++- 5 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-18-18-28-37.gh-issue-162230.ezR86S.rst diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 77239f026efbb17..21f40add53593b4 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1404,6 +1404,15 @@ def peek(self, size=0): def read1(self, size=-1): return self.reader.read1(size) + def readline(self, size=-1): + r"""Read and return a line from the stream. + + If size is specified, at most size bytes will be read. + + The line terminator is always b'\n'. + """ + return self.reader.readline(size) + def readinto1(self, b): return self.reader.readinto1(b) diff --git a/Lib/test/test_io/test_bufferedio.py b/Lib/test/test_io/test_bufferedio.py index e83dd0d4e28d006..ce594762c325ac0 100644 --- a/Lib/test/test_io/test_bufferedio.py +++ b/Lib/test/test_io/test_bufferedio.py @@ -1057,6 +1057,20 @@ def test_read(self): pair = self.tp(self.BytesIO(b"abc"), self.MockRawIO()) self.assertEqual(pair.read(None), b"abc") + def test_readline(self): + pair = self.tp(self.BytesIO(b"ab\ncd"), self.MockRawIO()) + + self.assertEqual(pair.readline(1), b"a") + self.assertEqual(pair.readline(5), b"b\n") + self.assertEqual(pair.readline(0), b"") + self.assertEqual(pair.readline(None), b"cd") + + def test_readline_nonblocking(self): + raw = self.MockRawIO([b"abc", None, None]) + pair = self.tp(raw, self.MockRawIO()) + + self.assertEqual(pair.readline(), b"abc") + def test_readlines(self): pair = lambda: self.tp(self.BytesIO(b"abc\ndef\nh"), self.MockRawIO()) self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"]) diff --git a/Misc/NEWS.d/next/Library/2026-11-18-18-28-37.gh-issue-162230.ezR86S.rst b/Misc/NEWS.d/next/Library/2026-11-18-18-28-37.gh-issue-162230.ezR86S.rst new file mode 100644 index 000000000000000..00b96c972ece6f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-18-18-28-37.gh-issue-162230.ezR86S.rst @@ -0,0 +1,3 @@ +Speed up :meth:`!io.BufferedRWPair.readline`. If a non-blocking read would +block, return the bytes read so far (possibly ``b''``). Previously, it raised +:exc:`OSError`. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 44c3f56caee18b6..757e6a10558e485 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -2394,6 +2394,25 @@ _io_BufferedRWPair_read1_impl(rwpair *self, PyObject *size) return _forward_call(self->reader, &_Py_ID(read1), size); } +/*[clinic input] +_io.BufferedRWPair.readline + size: object(c_default="NULL") = -1 + / + +Read and return a line from the stream. + +If size is specified, at most size bytes will be read. + +The line terminator is always b'\n'. +[clinic start generated code]*/ + +static PyObject * +_io_BufferedRWPair_readline_impl(rwpair *self, PyObject *size) +/*[clinic end generated code: output=ee5bb2364231e6e1 input=1c552272a69db1ad]*/ +{ + return _forward_call(self->reader, &_Py_ID(readline), size); +} + /*[clinic input] _io.BufferedRWPair.readinto buffer: object @@ -2735,6 +2754,7 @@ static PyMethodDef bufferedrwpair_methods[] = { _IO_BUFFEREDRWPAIR_READ_METHODDEF _IO_BUFFEREDRWPAIR_PEEK_METHODDEF _IO_BUFFEREDRWPAIR_READ1_METHODDEF + _IO_BUFFEREDRWPAIR_READLINE_METHODDEF _IO_BUFFEREDRWPAIR_READINTO_METHODDEF _IO_BUFFEREDRWPAIR_READINTO1_METHODDEF diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index a85dd6a20bc2c1d..0cedf4a9948e9da 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -1197,6 +1197,42 @@ _io_BufferedRWPair_read1(PyObject *self, PyObject *const *args, Py_ssize_t nargs return return_value; } +PyDoc_STRVAR(_io_BufferedRWPair_readline__doc__, +"readline($self, size=-1, /)\n" +"--\n" +"\n" +"Read and return a line from the stream.\n" +"\n" +"If size is specified, at most size bytes will be read.\n" +"\n" +"The line terminator is always b\'\\n\'."); + +#define _IO_BUFFEREDRWPAIR_READLINE_METHODDEF \ + {"readline", _PyCFunction_CAST(_io_BufferedRWPair_readline), METH_FASTCALL, _io_BufferedRWPair_readline__doc__}, + +static PyObject * +_io_BufferedRWPair_readline_impl(rwpair *self, PyObject *size); + +static PyObject * +_io_BufferedRWPair_readline(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *size = NULL; + + if (!_PyArg_CheckPositional("readline", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + size = args[0]; +skip_optional: + return_value = _io_BufferedRWPair_readline_impl((rwpair *)self, size); + +exit: + return return_value; +} + PyDoc_STRVAR(_io_BufferedRWPair_readinto__doc__, "readinto($self, buffer, /)\n" "--\n" @@ -1441,4 +1477,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) #define _IO_BUFFEREDRWPAIR_CLOSED_GETSETDEF {"closed", (getter)_io_BufferedRWPair_closed_get, (setter)NULL, NULL}, -/*[clinic end generated code: output=d9e1eb0efa4a2431 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1596bb353f00a1ea input=a9049054013a1b77]*/ From 9f205da816fbfff43d8eb9cc3e55fefdf4fad366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Thu, 19 Nov 2026 10:52:25 +0100 Subject: [PATCH 201/229] gh-129813: Use `PyBytesWriter` for strided mmap slices (#162257) --- Modules/mmapmodule.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index ce5c59cb0212707..06099a3b8b2e0be 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1567,22 +1567,16 @@ mmap_subscript_impl(mmap_object *self, PyObject *item) else if (step == 1) return _safe_PyBytes_FromStringAndSize(self->data + start, slicelen); else { - char *result_buf = (char *)PyMem_Malloc(slicelen); - PyObject *result; - - if (result_buf == NULL) - return PyErr_NoMemory(); - - if (safe_copy_to_slice(result_buf, self->data, start, step, - slicelen) < 0) - { - result = NULL; + PyBytesWriter *writer = PyBytesWriter_Create(slicelen); + if (writer == NULL) { + return NULL; } - else { - result = PyBytes_FromStringAndSize(result_buf, slicelen); + if (safe_copy_to_slice(PyBytesWriter_GetData(writer), self->data, + start, step, slicelen) < 0) { + PyBytesWriter_Discard(writer); + return NULL; } - PyMem_Free(result_buf); - return result; + return PyBytesWriter_Finish(writer); } } else { From 3edcc345c3adba410821ecd74ec8e38fc96fa774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Thu, 19 Nov 2026 10:53:57 +0100 Subject: [PATCH 202/229] gh-129813: Use `PyBytesWriter` in `ctypes` (#162260) --- Modules/_ctypes/_ctypes.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 217b5cfedf2adb0..81fb9c9f57b998b 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5154,7 +5154,6 @@ Array_subscript_impl(PyObject *myself, PyObject *item) if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { char *ptr = (char *)self->b_ptr; - char *dest; if (slicelen <= 0) return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); @@ -5164,19 +5163,17 @@ Array_subscript_impl(PyObject *myself, PyObject *item) slicelen); return res; } - dest = (char *)PyMem_Malloc(slicelen); - - if (dest == NULL) - return PyErr_NoMemory(); - + PyBytesWriter *writer = PyBytesWriter_Create(slicelen); + if (writer == NULL) { + return NULL; + } + char *dest = PyBytesWriter_GetData(writer); for (cur = start, i = 0; i < slicelen; cur += step, i++) { dest[i] = ptr[cur]; } - np = PyBytes_FromStringAndSize(dest, slicelen); - PyMem_Free(dest); - return np; + return PyBytesWriter_Finish(writer); } if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { wchar_t *ptr = (wchar_t *)self->b_ptr; @@ -5964,8 +5961,6 @@ Pointer_subscript(PyObject *myself, PyObject *item) } assert(iteminfo); if (iteminfo->getfunc == _ctypes_get_fielddesc("c")->getfunc) { - char *dest; - if (len <= 0) return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); if (step == 1) { @@ -5977,18 +5972,18 @@ Pointer_subscript(PyObject *myself, PyObject *item) Py_END_CRITICAL_SECTION(); return res; } - dest = (char *)PyMem_Malloc(len); - if (dest == NULL) - return PyErr_NoMemory(); + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { + return NULL; + } + char *dest = PyBytesWriter_GetData(writer); Py_BEGIN_CRITICAL_SECTION(self); char *ptr = *(void **)self->b_ptr; for (cur = start, i = 0; i < len; cur += step, i++) { dest[i] = ptr[cur]; } Py_END_CRITICAL_SECTION(); - np = PyBytes_FromStringAndSize(dest, len); - PyMem_Free(dest); - return np; + return PyBytesWriter_Finish(writer); } if (iteminfo->getfunc == _ctypes_get_fielddesc("u")->getfunc) { wchar_t *dest; From 03e7a8854d714fabb3ab5b8c8f984fc1214f65b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Thu, 19 Nov 2026 11:02:02 +0100 Subject: [PATCH 203/229] gh-129813: Use `PyBytesWriter` in `binascii` (#162259) --- Modules/binascii.c | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/Modules/binascii.c b/Modules/binascii.c index 57684f8336e64ed..406234cac1ed238 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -2390,10 +2390,8 @@ binascii_a2b_qp_impl(PyObject *module, Py_buffer *data, int header, Py_ssize_t in, out; char ch; const unsigned char *ascii_data; - unsigned char *odata; unsigned char *stripped = NULL; Py_ssize_t datalen = 0; - PyObject *rv; ascii_data = data->buf; datalen = data->len; @@ -2443,14 +2441,12 @@ binascii_a2b_qp_impl(PyObject *module, Py_buffer *data, int header, } } - /* We allocate the output same size as input, this is overkill. - */ - odata = (unsigned char *) PyMem_Calloc(1, datalen); - if (odata == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(datalen); + if (writer == NULL) { PyMem_Free(stripped); - PyErr_NoMemory(); return NULL; } + unsigned char *odata = PyBytesWriter_GetData(writer); in = out = 0; while (in < datalen) { @@ -2492,10 +2488,8 @@ binascii_a2b_qp_impl(PyObject *module, Py_buffer *data, int header, out++; } } - rv = PyBytes_FromStringAndSize((char *)odata, out); - PyMem_Free(odata); PyMem_Free(stripped); - return rv; + return PyBytesWriter_FinishWithSize(writer, out); } static int @@ -2535,9 +2529,7 @@ binascii_b2a_qp_impl(PyObject *module, Py_buffer *data, int quotetabs, { Py_ssize_t in, out; const unsigned char *databuf; - unsigned char *odata; Py_ssize_t datalen = 0, odatalen = 0; - PyObject *rv; unsigned int linelen = 0; unsigned char ch; int crlf = 0; @@ -2622,13 +2614,11 @@ binascii_b2a_qp_impl(PyObject *module, Py_buffer *data, int quotetabs, odatalen += delta; } - /* We allocate the output same size as input, this is overkill. - */ - odata = (unsigned char *) PyMem_Calloc(1, odatalen); - if (odata == NULL) { - PyErr_NoMemory(); + PyBytesWriter *writer = PyBytesWriter_Create(odatalen); + if (writer == NULL) { return NULL; } + unsigned char *odata = PyBytesWriter_GetData(writer); in = out = linelen = 0; while (in < datalen) { @@ -2698,9 +2688,7 @@ binascii_b2a_qp_impl(PyObject *module, Py_buffer *data, int quotetabs, } } } - rv = PyBytes_FromStringAndSize((char *)odata, out); - PyMem_Free(odata); - return rv; + return PyBytesWriter_FinishWithSize(writer, out); } /* List of functions defined in the module */ From 4cfa012f8a49c91ff64b509d1295813c25911c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Thu, 19 Nov 2026 12:33:38 +0100 Subject: [PATCH 204/229] gh-157227: Remove dead integer-location handling from sampling collectors (#162241) Remove the integer-location branches from the sampling collectors. Live samples and binary replay use four-field locations or None. --- Lib/profiling/sampling/collector.py | 10 ++------ Lib/profiling/sampling/gecko_collector.py | 17 +++++-------- .../sampling/live_collector/collector.py | 1 - Lib/profiling/sampling/pstats_collector.py | 1 - .../test_binary_format.py | 25 +++++-------------- .../test_sampling_profiler/test_collectors.py | 14 ----------- 6 files changed, 14 insertions(+), 54 deletions(-) diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index 1dc3656e0ebe976..3e2baa0f1a6f91d 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -20,16 +20,13 @@ def normalize_location(location): """Normalize location to a 4-tuple format. Args: - location: tuple (lineno, end_lineno, col_offset, end_col_offset), - an integer line number, or None + location: tuple (lineno, end_lineno, col_offset, end_col_offset) or None Returns: tuple: (lineno, end_lineno, col_offset, end_col_offset) """ if location is None: return DEFAULT_LOCATION - if isinstance(location, int): - return (location, location, -1, -1) return location @@ -37,16 +34,13 @@ def extract_lineno(location): """Extract lineno from location. Args: - location: tuple (lineno, end_lineno, col_offset, end_col_offset), - an integer line number, or None + location: tuple (lineno, end_lineno, col_offset, end_col_offset) or None Returns: int: The line number (0 for synthetic frames) """ if location is None: return 0 - if isinstance(location, int): - return location return location[0] def _is_internal_frame(frame): diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 2de8cce387e7f26..90092ac7d86fe72 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -368,11 +368,10 @@ def _record_stack_sample(self, thread_data, frames, tid, times, first_time): if self.opcodes_enabled and frames: leaf_frame = frames[0] filename, location, funcname, opcode = leaf_frame - if isinstance(location, tuple): - lineno, _, col_offset, _ = location + if location is None: + lineno, col_offset = None, -1 else: - lineno = location - col_offset = -1 + lineno, _, col_offset, _ = location current_state = (opcode, lineno, col_offset, funcname, filename) @@ -541,15 +540,11 @@ def _process_stack(self, thread_data, frames): for frame_tuple in reversed(frames): # frame_tuple is (filename, location, funcname, opcode) - # location is (lineno, end_lineno, col_offset, end_col_offset) or just lineno filename, location, funcname, opcode = frame_tuple - if isinstance(location, tuple): - lineno, end_lineno, col_offset, end_col_offset = location + if location is None: + lineno, col_offset = None, -1 else: - # Legacy format: location is just lineno - lineno = location - col_offset = -1 - end_col_offset = -1 + lineno, _, col_offset, _ = location # Get or create function func_idx = self._get_or_create_func( diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py index 553c856c6f8c562..a69351e4c795c4a 100644 --- a/Lib/profiling/sampling/live_collector/collector.py +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -313,7 +313,6 @@ def process_frames(self, frames, thread_id=None): self._seen_locations.clear() # Process each frame in the stack to track cumulative calls - # frame.location is (lineno, end_lineno, col_offset, end_col_offset), int, or None for frame in frames: lineno = extract_lineno(frame.location) location = (frame.filename, lineno, frame.funcname) diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py index 0f31d93582b83e7..66144f9fdbf192a 100644 --- a/Lib/profiling/sampling/pstats_collector.py +++ b/Lib/profiling/sampling/pstats_collector.py @@ -30,7 +30,6 @@ def _process_frames(self, frames, weight=1): self._seen_locations.clear() # Process each frame in the stack to track cumulative calls - # frame.location is int, tuple (lineno, end_lineno, col_offset, end_col_offset), or None for frame in frames: lineno = extract_lineno(frame.location) location = (frame.filename, lineno, frame.funcname) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index d73af62d3b02351..67b2708129b7970 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -63,29 +63,16 @@ def make_interpreter(interp_id, threads): return InterpreterInfo((interp_id, threads)) -def extract_lineno(location): - """Extract line number from location (tuple or int or None).""" - if location is None: - return 0 # Treat None as 0 - if isinstance(location, tuple): - return location[0] if location[0] is not None else 0 - return location - - def extract_location(location): """Extract full location info as dict from location tuple or None.""" if location is None: return {"lineno": 0, "end_lineno": 0, "column": 0, "end_column": 0} - if isinstance(location, tuple) and len(location) >= 4: - return { - "lineno": location[0] if location[0] is not None else 0, - "end_lineno": location[1] if location[1] is not None else 0, - "column": location[2] if location[2] is not None else 0, - "end_column": location[3] if location[3] is not None else 0, - } - # Fallback for old-style location - lineno = location[0] if isinstance(location, tuple) else location - return {"lineno": lineno or 0, "end_lineno": lineno or 0, "column": 0, "end_column": 0} + return { + "lineno": location[0], + "end_lineno": location[1], + "column": location[2], + "end_column": location[3], + } def frame_to_dict(frame): diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index d4ba2df5044559f..dcac6f3e7dfd9e7 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -2957,20 +2957,6 @@ def test_extract_lineno_from_none(self): """Test extracting lineno from None (synthetic frames).""" self.assertEqual(extract_lineno(None), 0) - def test_extract_lineno_from_int(self): - """Test extracting lineno from a bare integer line number. - - Mirrors normalize_location's int contract so callers like the - collapsed/flamegraph collectors do not crash on a bare-int location. - """ - self.assertEqual(extract_lineno(42), 42) - self.assertEqual(extract_lineno(0), 0) - - def test_normalize_location_with_int(self): - """Test normalize_location expands a legacy integer line number.""" - result = normalize_location(42) - self.assertEqual(result, (42, 42, -1, -1)) - def test_normalize_location_with_location_info(self): """Test normalize_location passes through LocationInfo.""" loc = LocationInfo(10, 15, 0, 5) From 484bc16723ef534e693bcd13e139ccd710a64da4 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Thu, 19 Nov 2026 21:44:49 +0900 Subject: [PATCH 205/229] gh-161751: Fix race in mimalloc abandoned segment visit (gh-162003) gh-161751: Fix race in mimalloc abandoned segment visit in free-threaded GC --- .../2026-11-14-23-17-18.gh-issue-161751.-silGa.rst | 2 ++ Objects/mimalloc/arena-abandon.c | 13 ++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-14-23-17-18.gh-issue-161751.-silGa.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-14-23-17-18.gh-issue-161751.-silGa.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-14-23-17-18.gh-issue-161751.-silGa.rst new file mode 100644 index 000000000000000..0c39e484386dc58 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-14-23-17-18.gh-issue-161751.-silGa.rst @@ -0,0 +1,2 @@ +Fix race in mimalloc abandoned segment visit in free-threaded GC. Patch by +Donghee Na. diff --git a/Objects/mimalloc/arena-abandon.c b/Objects/mimalloc/arena-abandon.c index 460c80fc22782ff..fdc2a4a902a1daa 100644 --- a/Objects/mimalloc/arena-abandon.c +++ b/Objects/mimalloc/arena-abandon.c @@ -240,17 +240,20 @@ static mi_segment_t* mi_arena_segment_clear_abandoned_next_field(mi_arena_field_ mi_arena_t* arena = mi_arena_from_index(arena_idx); if (arena != NULL) { bool has_lock = false; + // CPython: gh-161751: for a full visit, take the lock before reading the bitmap. + // A reclaim from another sub-process can clear our bit for a short time. + if (previous->visit_all && mi_option_is_enabled(mi_option_visit_abandoned)) { + mi_lock_acquire(&arena->abandoned_visit_lock); + has_lock = true; + } // visit the abandoned fields (starting at previous_idx) for (; field_idx < arena->field_count; field_idx++, bit_idx = 0) { size_t field = mi_atomic_load_relaxed(&arena->blocks_abandoned[field_idx]); if mi_unlikely(field != 0) { // skip zero fields quickly - // we only take the arena lock if there are actually abandoned segments present + // a reclaim tries to take the lock only when the field has abandoned segments if (!has_lock && mi_option_is_enabled(mi_option_visit_abandoned)) { - has_lock = (previous->visit_all ? (mi_lock_acquire(&arena->abandoned_visit_lock),true) : mi_lock_try_acquire(&arena->abandoned_visit_lock)); + has_lock = mi_lock_try_acquire(&arena->abandoned_visit_lock); if (!has_lock) { - if (previous->visit_all) { - _mi_error_message(EFAULT, "internal error: failed to visit all abandoned segments due to failure to acquire the visitor lock"); - } // skip to next arena break; } From b0ccebce6f91561025879820ce4d349b7bdd2a5b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 20 Nov 2026 00:30:58 +0900 Subject: [PATCH 206/229] Doc: Correct logging HOWTO and cookbook (GH-162198) Doc: Correct examples and API descriptions in the logging HOWTO and cookbook --- Doc/howto/logging-cookbook.rst | 71 ++++++++++++---------------------- Doc/howto/logging.rst | 25 ++++-------- 2 files changed, 32 insertions(+), 64 deletions(-) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index c105620fc3c04aa..d8a177020b921bf 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -330,11 +330,10 @@ Suppose you configure logging with the following JSON: } } -This configuration does *almost* what we want, except that ``sys.stdout`` would show messages -of severity ``ERROR`` and only events of this severity and higher will be tracked -as well as ``INFO`` and ``WARNING`` messages. To prevent this, we can set up a filter which -excludes those messages and add it to the relevant handler. This can be configured by -adding a ``filters`` section parallel to ``formatters`` and ``handlers``: +This configuration does *almost* what we want, except that ``sys.stdout`` would +also show messages of severity ``ERROR`` and above. To exclude those messages, +we can add a filter to the stdout handler. This can be configured by adding a +``filters`` section parallel to ``formatters`` and ``handlers``: .. code-block:: json @@ -562,9 +561,9 @@ One solution is to use a two-part approach. For the first part, attach only a :class:`QueueHandler` to those loggers which are accessed from performance-critical threads. They simply write to their queue, which can be sized to a large enough capacity or initialized with no upper bound to their -size. The write to the queue will typically be accepted quickly, though you -will probably need to catch the :exc:`queue.Full` exception as a precaution -in your code. If you are a library developer who has performance-critical +size. The write to the queue will typically be accepted quickly. If a bounded +queue is full, :class:`QueueHandler` calls :meth:`~logging.Handler.handleError` +with the record. If you are a library developer who has performance-critical threads in their code, be sure to document this (together with a suggestion to attach only ``QueueHandlers`` to your loggers) for the benefit of other developers who will use your code. @@ -1608,8 +1607,8 @@ application: The most current file is always :file:`logging_rotatingfile_example.out`, and each time it reaches the size limit it is renamed with the suffix -``.1``. Each of the existing backup files is renamed to increment the suffix -(``.1`` becomes ``.2``, etc.) and the ``.6`` file is erased. +``.1``. Before this happens, the oldest backup (``.5``), if present, is +deleted and the remaining backups are renamed (``.4`` becomes ``.5``, etc.). Obviously this example sets the log length much too small as an extreme example. You would want to set *maxBytes* to an appropriate value. @@ -1801,7 +1800,7 @@ Before Python 3.2, there were only two places where this creation was done: * :func:`makeLogRecord`, which is called with a dictionary containing attributes to be added to the LogRecord. This is typically invoked when a suitable dictionary has been received over the network (e.g. in pickle form - via a :class:`~handlers.SocketHandler`, or in JSON form via an + via a :class:`~handlers.SocketHandler`, or as URL-encoded data via an :class:`~handlers.HTTPHandler`). This has usually meant that if you need to do anything special with a @@ -1827,11 +1826,7 @@ do simply by adding new packages or modules and doing :: logger = logging.getLogger(__name__) -at module level). It's probably one too many things to think about. Developers -could also add the filter to a :class:`~logging.NullHandler` attached to their -top-level logger, but this would not be invoked if an application developer -attached a handler to a lower-level library logger --- so output from that -handler would not reflect the intentions of the library developer. +at module level). It's probably one too many things to think about. In Python 3.2 and later, :class:`~logging.LogRecord` creation is done through a factory, which you can specify. The factory is just a callable you can set with @@ -1913,15 +1908,15 @@ You can also subclass :class:`QueueListener` to get messages from other kinds of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: class ZeroMQSocketListener(QueueListener): - def __init__(self, uri, /, *handlers, **kwargs): - self.ctx = kwargs.get('ctx') or zmq.Context() + def __init__(self, uri, /, *handlers, ctx=None, **kwargs): + self.ctx = ctx or zmq.Context() socket = zmq.Socket(self.ctx, zmq.SUB) socket.setsockopt_string(zmq.SUBSCRIBE, '') # subscribe to everything socket.connect(uri) super().__init__(socket, *handlers, **kwargs) - def dequeue(self): - msg = self.queue.recv_json() + def dequeue(self, block): + msg = self.queue.recv_json(flags=0 if block else zmq.NOBLOCK) return logging.makeLogRecord(msg) .. _pynng-handlers: @@ -2245,7 +2240,7 @@ works:: import logging.config import logging.handlers from multiprocessing import Process, Queue, Event, current_process - import os + from multiprocessing import get_start_method import random import time @@ -2281,13 +2276,8 @@ works:: logging.config.dictConfig(config) listener = logging.handlers.QueueListener(q, MyHandler()) listener.start() - if os.name == 'posix': - # On POSIX, the setup logger will have been configured in the - # parent process, but should have been disabled following the - # dictConfig call. - # On Windows, since fork isn't used, the setup logger won't - # exist in the child, so it would be created and the message - # would appear - hence the "if posix" clause. + if get_start_method() == 'fork': + # dictConfig disables the setup logger inherited from the parent. logger = logging.getLogger('setup') logger.critical('Should not appear, because of disabled logger ...') stop_event.wait() @@ -2312,13 +2302,8 @@ works:: logging.CRITICAL] loggers = ['foo', 'foo.bar', 'foo.bar.baz', 'spam', 'spam.ham', 'spam.ham.eggs'] - if os.name == 'posix': - # On POSIX, the setup logger will have been configured in the - # parent process, but should have been disabled following the - # dictConfig call. - # On Windows, since fork isn't used, the setup logger won't - # exist in the child, so it would be created and the message - # would appear - hence the "if posix" clause. + if get_start_method() == 'fork': + # dictConfig disables the setup logger inherited from the parent. logger = logging.getLogger('setup') logger.critical('Should not appear, because of disabled logger ...') for i in range(100): @@ -2345,9 +2330,7 @@ works:: } # The worker process configuration is just a QueueHandler attached to the # root logger, which allows all messages to be sent to the queue. - # We disable existing loggers to disable the "setup" logger used in the - # parent process. This is needed on POSIX because the logger will - # be there in the child following a fork(). + # Disable the parent's "setup" logger if it was inherited via fork. config_worker = { 'version': 1, 'disable_existing_loggers': True, @@ -2365,9 +2348,7 @@ works:: # The listener process configuration shows that the full flexibility of # logging configuration is available to dispatch events to handlers however # you want. - # We disable existing loggers to disable the "setup" logger used in the - # parent process. This is needed on POSIX because the logger will - # be there in the child following a fork(). + # Disable the parent's "setup" logger if it was inherited via fork. config_listener = { 'version': 1, 'disable_existing_loggers': True, @@ -4077,11 +4058,9 @@ How to uniformly handle newlines in logging output Usually, messages that are logged (say to console or file) consist of a single line of text. However, sometimes there is a need to handle messages with -multiple lines - whether because a logging format string contains newlines, or -logged data contains newlines. If you want to handle such messages uniformly, so -that each line in the logged message appears uniformly formatted as if it was -logged separately, you can do this using a handler mixin, as in the following -snippet: +multiple lines because the message or its arguments contain newlines. +To format each line as if it were logged separately, you can use a handler +mixin, as in the following snippet: .. code-block:: python diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index c8ce0df9e937f8c..584f0901b8a69fc 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -146,13 +146,6 @@ interpreter, and don't just continue from the session described above:: logger.warning('And this, too') logger.error('And non-ASCII stuff, too, like Øresund and Malmö') -.. versionchanged:: 3.9 - The *encoding* argument was added. In earlier Python versions, or if not - specified, the encoding used is the default value used by :func:`open`. While - not shown in the above example, an *errors* argument can also now be passed, - which determines how encoding errors are handled. For available values and - the default, see the documentation for :func:`open`. - And now if we open the file and look at what we have, we should find the log messages: @@ -465,10 +458,9 @@ With the logger object configured, the following methods create log messages: a message and a level that corresponds to their respective method names. The message is actually a format string, which may contain the standard string substitution syntax of ``%s``, ``%d``, ``%f``, and so on. The - rest of their arguments is a list of objects that correspond with the - substitution fields in the message. With regard to ``**kwargs``, the - logging methods care only about a keyword of ``exc_info`` and use it to - determine whether to log exception information. + remaining positional arguments correspond to the substitution fields in the + message. These methods also accept the keyword arguments *exc_info*, *extra*, + *stack_info*, and *stacklevel*, described in :meth:`Logger.debug`. * :meth:`Logger.exception` creates a log message similar to :meth:`Logger.error`. The difference is that :meth:`Logger.exception` dumps a @@ -1006,9 +998,9 @@ Logged messages are formatted for presentation through instances of the use with the % operator and a dictionary. For formatting multiple messages in a batch, instances of -:class:`BufferingFormatter` can be used. In addition to the format -string (which is applied to each message in the batch), there is provision for -header and trailer format strings. +:class:`BufferingFormatter` can be used. A line formatter formats each record +in the batch. To add a header or footer, override +:meth:`~BufferingFormatter.formatHeader` or :meth:`~BufferingFormatter.formatFooter`. When filtering based on logger level and/or handler level is not enough, instances of :class:`Filter` can be added to both :class:`Logger` and @@ -1059,10 +1051,7 @@ In the preceding sections and examples, it has been assumed that the message passed when logging the event is a string. However, this is not the only possibility. You can pass an arbitrary object as a message, and its :meth:`~object.__str__` method will be called when the logging system needs to -convert it to a string representation. In fact, if you want to, you can avoid -computing a string representation altogether - for example, the -:class:`~handlers.SocketHandler` emits an event by pickling it and sending it -over the wire. +convert it to a string representation. Optimization From 0b5e26c6b1567f93abb6d3c755879d4e8070d503 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 19 Nov 2026 17:46:03 +0100 Subject: [PATCH 207/229] gh-157695: Make HAVE_CURSES_xxx macros private (#162212) Rename HAVE_CURSES_xxx macros, added to Python 3.16, to _Py_HAVE_CURSES_xxx. Rename macros of functions: * define_key(): HAVE_CURSES_DEFINE_KEY * has_mouse(): HAVE_CURSES_HAS_MOUSE * is_keypad(): HAVE_CURSES_IS_KEYPAD * is_leaveok(): HAVE_CURSES_IS_LEAVEOK * key_defined(): HAVE_CURSES_KEY_DEFINED * keyok(): HAVE_CURSES_KEYOK * new_prescr(): HAVE_CURSES_NEW_PRESCR * nofilter(): HAVE_CURSES_NOFILTER * scr_dump(): HAVE_CURSES_SCR_DUMP * scr_set(): HAVE_CURSES_SCR_SET * slk_attr_off(): HAVE_CURSES_SLK_ATTR_OFF * slk_attr_on(): HAVE_CURSES_SLK_ATTR_ON * slk_attr_set(): HAVE_CURSES_SLK_ATTR_SET * slk_color(): HAVE_CURSES_SLK_COLOR * term_attrs(): HAVE_CURSES_TERM_ATTRS * use_screen(): HAVE_CURSES_USE_SCREEN * use_window(): HAVE_CURSES_USE_WINDOW * wattr_get(): HAVE_CURSES_WATTR_GET * wattr_off(): HAVE_CURSES_WATTR_OFF * wattr_on(): HAVE_CURSES_WATTR_ON * wattr_set(): HAVE_CURSES_WATTR_SET * wcolor_set(): HAVE_CURSES_WCOLOR_SET Add PY_CHECK_CURSES_FUNC_PRIVATE() function to configure.ac. --- Modules/_cursesmodule.c | 100 ++-- Modules/clinic/_cursesmodule.c.h | 82 +-- Tools/build/check_capi_macros_ignored.txt | 22 - configure | 679 ++++++++++++---------- configure.ac | 54 +- pyconfig.h.in | 132 ++--- 6 files changed, 559 insertions(+), 510 deletions(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 52b87e7cc0e8b98..e82c78691c7f6a5 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -1936,10 +1936,10 @@ Window_NoArgTrueFalseFunction(is_scrollok) Window_NoArgTrueFalseFunction(is_subwin) Window_NoArgTrueFalseFunction(is_syncok) #endif -#if defined(HAVE_CURSES_IS_KEYPAD) || defined(PDCURSES) +#if defined(_Py__Py_HAVE_CURSES_IS_KEYPAD) || defined(PDCURSES) Window_NoArgTrueFalseFunction(is_keypad) #endif -#if defined(HAVE_CURSES_IS_LEAVEOK) || defined(PDCURSES) +#if defined(_Py_HAVE_CURSES_IS_LEAVEOK) || defined(PDCURSES) Window_NoArgTrueFalseFunction(is_leaveok) #endif #if defined(HAVE_CURSES_IS_PAD) || defined(PDCURSES) @@ -2199,7 +2199,7 @@ static int curses_wattr_save(PyCursesWindowObject *self, attr_t *attrs, int *pair, const char *funcname) { -#if defined(HAVE_CURSES_WATTR_GET) && defined(HAVE_CURSES_WATTR_SET) +#if defined(_Py_HAVE_CURSES_WATTR_GET) && defined(_Py_HAVE_CURSES_WATTR_SET) int rtn; #if _NCURSES_EXTENDED_COLOR_FUNCS short legacy_pair; @@ -2222,7 +2222,7 @@ curses_wattr_save(PyCursesWindowObject *self, attr_t *attrs, int *pair, /* Put the rendition back. The name of the curses function used is _CURSES_WATTR_RESTORE_FUNC, for the caller to name it in an error. */ -#if defined(HAVE_CURSES_WATTR_GET) && defined(HAVE_CURSES_WATTR_SET) +#if defined(_Py_HAVE_CURSES_WATTR_GET) && defined(_Py_HAVE_CURSES_WATTR_SET) #define _CURSES_WATTR_RESTORE_FUNC "wattr_set" #else #define _CURSES_WATTR_RESTORE_FUNC "wattrset" @@ -2231,7 +2231,7 @@ curses_wattr_save(PyCursesWindowObject *self, attr_t *attrs, int *pair, static int curses_wattr_restore(PyCursesWindowObject *self, attr_t attrs, int pair) { -#if defined(HAVE_CURSES_WATTR_GET) && defined(HAVE_CURSES_WATTR_SET) +#if defined(_Py_HAVE_CURSES_WATTR_GET) && defined(_Py_HAVE_CURSES_WATTR_SET) #if _NCURSES_EXTENDED_COLOR_FUNCS return wattr_set(self->win, attrs, 0, &pair); #else @@ -2558,7 +2558,7 @@ _curses_window_attrset_impl(PyCursesWindowObject *self, attr_t attr) return curses_window_check_err(self, rtn, "wattrset", "attrset"); } -#ifdef HAVE_CURSES_WATTR_GET +#ifdef _Py_HAVE_CURSES_WATTR_GET /*[clinic input] _curses.window.attr_get @@ -2584,9 +2584,9 @@ _curses_window_attr_get_impl(PyCursesWindowObject *self) } return Py_BuildValue("(ki)", (unsigned long)attrs, (int)pair); } -#endif /* HAVE_CURSES_WATTR_GET */ +#endif /* _Py_HAVE_CURSES_WATTR_GET */ -#ifdef HAVE_CURSES_WATTR_SET +#ifdef _Py_HAVE_CURSES_WATTR_SET /*[clinic input] _curses.window.attr_set @@ -2610,9 +2610,9 @@ _curses_window_attr_set_impl(PyCursesWindowObject *self, attr_t attr, #endif return curses_window_check_err(self, rtn, "wattr_set", "attr_set"); } -#endif /* HAVE_CURSES_WATTR_SET */ +#endif /* _Py_HAVE_CURSES_WATTR_SET */ -#ifdef HAVE_CURSES_WATTR_ON +#ifdef _Py_HAVE_CURSES_WATTR_ON /*[clinic input] _curses.window.attr_on @@ -2629,9 +2629,9 @@ _curses_window_attr_on_impl(PyCursesWindowObject *self, attr_t attr) int rtn = wattr_on(self->win, attr, NULL); return curses_window_check_err(self, rtn, "wattr_on", "attr_on"); } -#endif /* HAVE_CURSES_WATTR_ON */ +#endif /* _Py_HAVE_CURSES_WATTR_ON */ -#ifdef HAVE_CURSES_WATTR_OFF +#ifdef _Py_HAVE_CURSES_WATTR_OFF /*[clinic input] _curses.window.attr_off @@ -2648,9 +2648,9 @@ _curses_window_attr_off_impl(PyCursesWindowObject *self, attr_t attr) int rtn = wattr_off(self->win, attr, NULL); return curses_window_check_err(self, rtn, "wattr_off", "attr_off"); } -#endif /* HAVE_CURSES_WATTR_OFF */ +#endif /* _Py_HAVE_CURSES_WATTR_OFF */ -#ifdef HAVE_CURSES_WCOLOR_SET +#ifdef _Py_HAVE_CURSES_WCOLOR_SET /*[clinic input] _curses.window.color_set @@ -2672,7 +2672,7 @@ _curses_window_color_set_impl(PyCursesWindowObject *self, int pair) #endif return curses_window_check_err(self, rtn, "wcolor_set", "color_set"); } -#endif /* HAVE_CURSES_WCOLOR_SET */ +#endif /* _Py_HAVE_CURSES_WCOLOR_SET */ /*[clinic input] _curses.window.getattrs @@ -5035,7 +5035,7 @@ static PyType_Spec PyCursesComplexStr_Type_spec = { }; #undef clinic_state -#if defined(HAVE_CURSES_USE_SCREEN) || defined(HAVE_CURSES_USE_WINDOW) +#if defined(_Py_HAVE_CURSES_USE_SCREEN) || defined(_Py_HAVE_CURSES_USE_WINDOW) /* Shared trampoline for window.use()/screen.use(): call func(obj, *extra, **kwargs) and store the result (NULL on exception) in data->result. */ @@ -5085,7 +5085,7 @@ curses_use_parse(PyObject *args, PyObject **func, PyObject **extra) } #endif -#ifdef HAVE_CURSES_USE_WINDOW +#ifdef _Py_HAVE_CURSES_USE_WINDOW static int curses_use_window_cb(WINDOW *Py_UNUSED(win), void *data) { @@ -5111,7 +5111,7 @@ PyCursesWindow_use(PyObject *self, PyObject *args, PyObject *kwargs) Py_DECREF(extra); return data.result; } -#endif /* HAVE_CURSES_USE_WINDOW */ +#endif /* _Py_HAVE_CURSES_USE_WINDOW */ static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_ADDCH_METHODDEF @@ -5250,12 +5250,12 @@ static PyMethodDef PyCursesWindow_methods[] = { "is_syncok($self, /)\n--\n\n" "Return the current value set by syncok()."}, #endif -#if defined(HAVE_CURSES_IS_KEYPAD) || defined(PDCURSES) +#if defined(_Py__Py_HAVE_CURSES_IS_KEYPAD) || defined(PDCURSES) {"is_keypad", PyCursesWindow_is_keypad, METH_NOARGS, "is_keypad($self, /)\n--\n\n" "Return the current value set by keypad()."}, #endif -#if defined(HAVE_CURSES_IS_LEAVEOK) || defined(PDCURSES) +#if defined(_Py_HAVE_CURSES_IS_LEAVEOK) || defined(PDCURSES) {"is_leaveok", PyCursesWindow_is_leaveok, METH_NOARGS, "is_leaveok($self, /)\n--\n\n" "Return the current value set by leaveok()."}, @@ -5336,7 +5336,7 @@ static PyMethodDef PyCursesWindow_methods[] = { "untouchwin($self, /)\n--\n\n" "Mark all lines in the window as unchanged since last refresh()."}, _CURSES_WINDOW_VLINE_METHODDEF -#ifdef HAVE_CURSES_USE_WINDOW +#ifdef _Py_HAVE_CURSES_USE_WINDOW {"use", _PyCFunction_CAST(PyCursesWindow_use), METH_VARARGS | METH_KEYWORDS, PyCursesWindow_use__doc__}, #endif @@ -5478,7 +5478,7 @@ static PyGetSetDef PyCursesScreen_getsets[] = { {NULL, NULL, NULL, NULL, NULL} /* sentinel */ }; -#ifdef HAVE_CURSES_USE_SCREEN +#ifdef _Py_HAVE_CURSES_USE_SCREEN static int curses_use_screen_cb(SCREEN *Py_UNUSED(sp), void *data) { @@ -5533,7 +5533,7 @@ PyCursesScreen_use(PyObject *self, PyObject *args, PyObject *kwargs) } return data.result; } -#endif /* HAVE_CURSES_USE_SCREEN */ +#endif /* _Py_HAVE_CURSES_USE_SCREEN */ PyDoc_STRVAR(PyCursesScreen_close__doc__, "close($self, /)\n--\n\n" @@ -5552,7 +5552,7 @@ PyCursesScreen_close(PyObject *self, PyObject *Py_UNUSED(ignored)) static PyMethodDef PyCursesScreen_methods[] = { {"close", PyCursesScreen_close, METH_NOARGS, PyCursesScreen_close__doc__}, -#ifdef HAVE_CURSES_USE_SCREEN +#ifdef _Py_HAVE_CURSES_USE_SCREEN {"use", _PyCFunction_CAST(PyCursesScreen_use), METH_VARARGS | METH_KEYWORDS, PyCursesScreen_use__doc__}, #endif @@ -5704,7 +5704,7 @@ _curses_filter_impl(PyObject *module) } #endif -#ifdef HAVE_CURSES_NOFILTER +#ifdef _Py_HAVE_CURSES_NOFILTER /*[clinic input] _curses.nofilter @@ -6269,7 +6269,7 @@ _curses_getwin(PyObject *module, PyObject *file) return res; } -#ifdef HAVE_CURSES_SCR_DUMP +#ifdef _Py_HAVE_CURSES_SCR_DUMP /*[clinic input] _curses.scr_dump @@ -6324,7 +6324,7 @@ _curses_scr_init(PyObject *module, PyObject *filename) /*[clinic end generated code: output=2e861d381d710419 input=81c45e4702124ef6]*/ ScreenDumpFunctionBody(scr_init) -#ifdef HAVE_CURSES_SCR_SET +#ifdef _Py_HAVE_CURSES_SCR_SET /*[clinic input] _curses.scr_set @@ -6341,8 +6341,8 @@ static PyObject * _curses_scr_set(PyObject *module, PyObject *filename) /*[clinic end generated code: output=6056fdec12c5935f input=d248c20543cc289b]*/ ScreenDumpFunctionBody(scr_set) -#endif /* HAVE_CURSES_SCR_SET */ -#endif /* HAVE_CURSES_SCR_DUMP */ +#endif /* _Py_HAVE_CURSES_SCR_SET */ +#endif /* _Py_HAVE_CURSES_SCR_DUMP */ /*[clinic input] _curses.halfdelay @@ -6421,7 +6421,7 @@ _curses_has_key_impl(PyObject *module, int key) } #endif -#ifdef HAVE_CURSES_DEFINE_KEY +#ifdef _Py_HAVE_CURSES_DEFINE_KEY /*[clinic input] _curses.define_key @@ -6447,9 +6447,9 @@ _curses_define_key_impl(PyObject *module, const char *definition, return curses_check_err(module, define_key(definition, keycode), "define_key", NULL); } -#endif /* HAVE_CURSES_DEFINE_KEY */ +#endif /* _Py_HAVE_CURSES_DEFINE_KEY */ -#ifdef HAVE_CURSES_KEY_DEFINED +#ifdef _Py_HAVE_CURSES_KEY_DEFINED /*[clinic input] _curses.key_defined @@ -6471,9 +6471,9 @@ _curses_key_defined_impl(PyObject *module, const char *definition) return PyLong_FromLong(key_defined(definition)); } -#endif /* HAVE_CURSES_KEY_DEFINED */ +#endif /* _Py_HAVE_CURSES_KEY_DEFINED */ -#ifdef HAVE_CURSES_KEYOK +#ifdef _Py_HAVE_CURSES_KEYOK /*[clinic input] _curses.keyok @@ -6494,7 +6494,7 @@ _curses_keyok_impl(PyObject *module, int keycode, int enable) return curses_check_err(module, keyok(keycode, enable), "keyok", NULL); } -#endif /* HAVE_CURSES_KEYOK */ +#endif /* _Py_HAVE_CURSES_KEYOK */ /*[clinic input] _curses.init_color @@ -7283,7 +7283,7 @@ _curses_set_term(PyObject *module, PyObject *screen) return prev != NULL ? prev : Py_NewRef(Py_None); } -#ifdef HAVE_CURSES_NEW_PRESCR +#ifdef _Py_HAVE_CURSES_NEW_PRESCR /*[clinic input] _curses.new_prescr @@ -7317,7 +7317,7 @@ _curses_new_prescr_impl(PyObject *module) state->prescreen = Py_NewRef(screenobj); return screenobj; } -#endif /* HAVE_CURSES_NEW_PRESCR */ +#endif /* _Py_HAVE_CURSES_NEW_PRESCR */ #ifdef HAVE_CURSES_ESCDELAY /*[clinic input] @@ -7573,7 +7573,7 @@ _curses_meta_impl(PyObject *module, int yes) } #if defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES) -#if defined(HAVE_CURSES_HAS_MOUSE) || defined(PDCURSES) +#if defined(_Py_HAVE_CURSES_HAS_MOUSE) || defined(PDCURSES) /*[clinic input] _curses.has_mouse @@ -7588,7 +7588,7 @@ _curses_has_mouse_impl(PyObject *module) return PyBool_FromLong(has_mouse()); } -#endif /* HAVE_CURSES_HAS_MOUSE || PDCURSES */ +#endif /* _Py_HAVE_CURSES_HAS_MOUSE || PDCURSES */ /*[clinic input] _curses.mouseinterval @@ -8272,7 +8272,7 @@ _curses_termattrs_impl(PyObject *module) return PyLong_FromUnsignedLong((unsigned long)(chtype)termattrs()); } -#ifdef HAVE_CURSES_TERM_ATTRS +#ifdef _Py_HAVE_CURSES_TERM_ATTRS /*[clinic input] _curses.term_attrs @@ -8290,7 +8290,7 @@ _curses_term_attrs_impl(PyObject *module) return PyLong_FromUnsignedLongLong(term_attrs()); } -#endif /* HAVE_CURSES_TERM_ATTRS */ +#endif /* _Py_HAVE_CURSES_TERM_ATTRS */ /*[clinic input] @permit_long_summary @@ -8877,7 +8877,7 @@ _curses_slk_attr_impl(PyObject *module) } #endif -#ifdef HAVE_CURSES_SLK_ATTR_ON +#ifdef _Py_HAVE_CURSES_SLK_ATTR_ON /*[clinic input] _curses.slk_attr_on @@ -8895,9 +8895,9 @@ _curses_slk_attr_on_impl(PyObject *module, attr_t attr) return curses_check_err(module, slk_attr_on(attr, NULL), "slk_attr_on", NULL); } -#endif /* HAVE_CURSES_SLK_ATTR_ON */ +#endif /* _Py_HAVE_CURSES_SLK_ATTR_ON */ -#ifdef HAVE_CURSES_SLK_ATTR_OFF +#ifdef _Py_HAVE_CURSES_SLK_ATTR_OFF /*[clinic input] _curses.slk_attr_off @@ -8915,9 +8915,9 @@ _curses_slk_attr_off_impl(PyObject *module, attr_t attr) return curses_check_err(module, slk_attr_off(attr, NULL), "slk_attr_off", NULL); } -#endif /* HAVE_CURSES_SLK_ATTR_OFF */ +#endif /* _Py_HAVE_CURSES_SLK_ATTR_OFF */ -#ifdef HAVE_CURSES_SLK_ATTR_SET +#ifdef _Py_HAVE_CURSES_SLK_ATTR_SET /*[clinic input] _curses.slk_attr_set @@ -8941,9 +8941,9 @@ _curses_slk_attr_set_impl(PyObject *module, attr_t attr, int pair) #endif return curses_check_err(module, rtn, "slk_attr_set", NULL); } -#endif /* HAVE_CURSES_SLK_ATTR_SET */ +#endif /* _Py_HAVE_CURSES_SLK_ATTR_SET */ -#ifdef HAVE_CURSES_SLK_COLOR +#ifdef _Py_HAVE_CURSES_SLK_COLOR /*[clinic input] _curses.slk_color @@ -8966,7 +8966,7 @@ _curses_slk_color_impl(PyObject *module, int pair) #endif return curses_check_err(module, rtn, "slk_color", NULL); } -#endif /* HAVE_CURSES_SLK_COLOR */ +#endif /* _Py_HAVE_CURSES_SLK_COLOR */ #ifdef HAVE_CURSES_USE_ENV /*[clinic input] @@ -9178,7 +9178,7 @@ static PyMethodDef cursesmodule_methods[] = { _CURSES_NEWPAD_METHODDEF _CURSES_NEWTERM_METHODDEF _CURSES_NEWWIN_METHODDEF -#ifdef HAVE_CURSES_NEW_PRESCR +#ifdef _Py_HAVE_CURSES_NEW_PRESCR _CURSES_NEW_PRESCR_METHODDEF #endif _CURSES_NL_METHODDEF diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index 553843ab96a7af7..39512bc6cad3d68 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -520,7 +520,7 @@ _curses_window_attrset(PyObject *self, PyObject *arg) return return_value; } -#if defined(HAVE_CURSES_WATTR_GET) +#if defined(_Py_HAVE_CURSES_WATTR_GET) PyDoc_STRVAR(_curses_window_attr_get__doc__, "attr_get($self, /)\n" @@ -540,9 +540,9 @@ _curses_window_attr_get(PyObject *self, PyObject *Py_UNUSED(ignored)) return _curses_window_attr_get_impl((PyCursesWindowObject *)self); } -#endif /* defined(HAVE_CURSES_WATTR_GET) */ +#endif /* defined(_Py_HAVE_CURSES_WATTR_GET) */ -#if defined(HAVE_CURSES_WATTR_SET) +#if defined(_Py_HAVE_CURSES_WATTR_SET) PyDoc_STRVAR(_curses_window_attr_set__doc__, "attr_set($self, attr, pair=0, /)\n" @@ -583,9 +583,9 @@ _curses_window_attr_set(PyObject *self, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* defined(HAVE_CURSES_WATTR_SET) */ +#endif /* defined(_Py_HAVE_CURSES_WATTR_SET) */ -#if defined(HAVE_CURSES_WATTR_ON) +#if defined(_Py_HAVE_CURSES_WATTR_ON) PyDoc_STRVAR(_curses_window_attr_on__doc__, "attr_on($self, attr, /)\n" @@ -614,9 +614,9 @@ _curses_window_attr_on(PyObject *self, PyObject *arg) return return_value; } -#endif /* defined(HAVE_CURSES_WATTR_ON) */ +#endif /* defined(_Py_HAVE_CURSES_WATTR_ON) */ -#if defined(HAVE_CURSES_WATTR_OFF) +#if defined(_Py_HAVE_CURSES_WATTR_OFF) PyDoc_STRVAR(_curses_window_attr_off__doc__, "attr_off($self, attr, /)\n" @@ -645,9 +645,9 @@ _curses_window_attr_off(PyObject *self, PyObject *arg) return return_value; } -#endif /* defined(HAVE_CURSES_WATTR_OFF) */ +#endif /* defined(_Py_HAVE_CURSES_WATTR_OFF) */ -#if defined(HAVE_CURSES_WCOLOR_SET) +#if defined(_Py_HAVE_CURSES_WCOLOR_SET) PyDoc_STRVAR(_curses_window_color_set__doc__, "color_set($self, pair, /)\n" @@ -676,7 +676,7 @@ _curses_window_color_set(PyObject *self, PyObject *arg) return return_value; } -#endif /* defined(HAVE_CURSES_WCOLOR_SET) */ +#endif /* defined(_Py_HAVE_CURSES_WCOLOR_SET) */ PyDoc_STRVAR(_curses_window_getattrs__doc__, "getattrs($self, /)\n" @@ -2867,7 +2867,7 @@ _curses_filter(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_CURSES_FILTER) */ -#if defined(HAVE_CURSES_NOFILTER) +#if defined(_Py_HAVE_CURSES_NOFILTER) PyDoc_STRVAR(_curses_nofilter__doc__, "nofilter($module, /)\n" @@ -2891,7 +2891,7 @@ _curses_nofilter(PyObject *module, PyObject *Py_UNUSED(ignored)) return _curses_nofilter_impl(module); } -#endif /* defined(HAVE_CURSES_NOFILTER) */ +#endif /* defined(_Py_HAVE_CURSES_NOFILTER) */ PyDoc_STRVAR(_curses_baudrate__doc__, "baudrate($module, /)\n" @@ -3566,7 +3566,7 @@ PyDoc_STRVAR(_curses_getwin__doc__, #define _CURSES_GETWIN_METHODDEF \ {"getwin", (PyCFunction)_curses_getwin, METH_O, _curses_getwin__doc__}, -#if defined(HAVE_CURSES_SCR_DUMP) +#if defined(_Py_HAVE_CURSES_SCR_DUMP) PyDoc_STRVAR(_curses_scr_dump__doc__, "scr_dump($module, filename, /)\n" @@ -3583,9 +3583,9 @@ PyDoc_STRVAR(_curses_scr_dump__doc__, #define _CURSES_SCR_DUMP_METHODDEF \ {"scr_dump", (PyCFunction)_curses_scr_dump, METH_O, _curses_scr_dump__doc__}, -#endif /* defined(HAVE_CURSES_SCR_DUMP) */ +#endif /* defined(_Py_HAVE_CURSES_SCR_DUMP) */ -#if defined(HAVE_CURSES_SCR_DUMP) +#if defined(_Py_HAVE_CURSES_SCR_DUMP) PyDoc_STRVAR(_curses_scr_restore__doc__, "scr_restore($module, filename, /)\n" @@ -3602,9 +3602,9 @@ PyDoc_STRVAR(_curses_scr_restore__doc__, #define _CURSES_SCR_RESTORE_METHODDEF \ {"scr_restore", (PyCFunction)_curses_scr_restore, METH_O, _curses_scr_restore__doc__}, -#endif /* defined(HAVE_CURSES_SCR_DUMP) */ +#endif /* defined(_Py_HAVE_CURSES_SCR_DUMP) */ -#if defined(HAVE_CURSES_SCR_DUMP) +#if defined(_Py_HAVE_CURSES_SCR_DUMP) PyDoc_STRVAR(_curses_scr_init__doc__, "scr_init($module, filename, /)\n" @@ -3621,9 +3621,9 @@ PyDoc_STRVAR(_curses_scr_init__doc__, #define _CURSES_SCR_INIT_METHODDEF \ {"scr_init", (PyCFunction)_curses_scr_init, METH_O, _curses_scr_init__doc__}, -#endif /* defined(HAVE_CURSES_SCR_DUMP) */ +#endif /* defined(_Py_HAVE_CURSES_SCR_DUMP) */ -#if defined(HAVE_CURSES_SCR_DUMP) && defined(HAVE_CURSES_SCR_SET) +#if defined(_Py_HAVE_CURSES_SCR_DUMP) && defined(_Py_HAVE_CURSES_SCR_SET) PyDoc_STRVAR(_curses_scr_set__doc__, "scr_set($module, filename, /)\n" @@ -3639,7 +3639,7 @@ PyDoc_STRVAR(_curses_scr_set__doc__, #define _CURSES_SCR_SET_METHODDEF \ {"scr_set", (PyCFunction)_curses_scr_set, METH_O, _curses_scr_set__doc__}, -#endif /* defined(HAVE_CURSES_SCR_DUMP) && defined(HAVE_CURSES_SCR_SET) */ +#endif /* defined(_Py_HAVE_CURSES_SCR_DUMP) && defined(_Py_HAVE_CURSES_SCR_SET) */ PyDoc_STRVAR(_curses_halfdelay__doc__, "halfdelay($module, tenths, /)\n" @@ -3778,7 +3778,7 @@ _curses_has_key(PyObject *module, PyObject *arg) #endif /* defined(HAVE_CURSES_HAS_KEY) */ -#if defined(HAVE_CURSES_DEFINE_KEY) +#if defined(_Py_HAVE_CURSES_DEFINE_KEY) PyDoc_STRVAR(_curses_define_key__doc__, "define_key($module, definition, keycode, /)\n" @@ -3834,9 +3834,9 @@ _curses_define_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* defined(HAVE_CURSES_DEFINE_KEY) */ +#endif /* defined(_Py_HAVE_CURSES_DEFINE_KEY) */ -#if defined(HAVE_CURSES_KEY_DEFINED) +#if defined(_Py_HAVE_CURSES_KEY_DEFINED) PyDoc_STRVAR(_curses_key_defined__doc__, "key_defined($module, definition, /)\n" @@ -3876,9 +3876,9 @@ _curses_key_defined(PyObject *module, PyObject *arg) return return_value; } -#endif /* defined(HAVE_CURSES_KEY_DEFINED) */ +#endif /* defined(_Py_HAVE_CURSES_KEY_DEFINED) */ -#if defined(HAVE_CURSES_KEYOK) +#if defined(_Py_HAVE_CURSES_KEYOK) PyDoc_STRVAR(_curses_keyok__doc__, "keyok($module, keycode, enable, /)\n" @@ -3921,7 +3921,7 @@ _curses_keyok(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* defined(HAVE_CURSES_KEYOK) */ +#endif /* defined(_Py_HAVE_CURSES_KEYOK) */ PyDoc_STRVAR(_curses_init_color__doc__, "init_color($module, color_number, r, g, b, /)\n" @@ -4364,7 +4364,7 @@ PyDoc_STRVAR(_curses_set_term__doc__, #define _CURSES_SET_TERM_METHODDEF \ {"set_term", (PyCFunction)_curses_set_term, METH_O, _curses_set_term__doc__}, -#if defined(HAVE_CURSES_NEW_PRESCR) +#if defined(_Py_HAVE_CURSES_NEW_PRESCR) PyDoc_STRVAR(_curses_new_prescr__doc__, "new_prescr($module, /)\n" @@ -4387,7 +4387,7 @@ _curses_new_prescr(PyObject *module, PyObject *Py_UNUSED(ignored)) return _curses_new_prescr_impl(module); } -#endif /* defined(HAVE_CURSES_NEW_PRESCR) */ +#endif /* defined(_Py_HAVE_CURSES_NEW_PRESCR) */ #if defined(HAVE_CURSES_ESCDELAY) @@ -4731,7 +4731,7 @@ _curses_meta(PyObject *module, PyObject *arg) return return_value; } -#if (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) && (defined(HAVE_CURSES_HAS_MOUSE) || defined(PDCURSES)) +#if (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) && (defined(_Py_HAVE_CURSES_HAS_MOUSE) || defined(PDCURSES)) PyDoc_STRVAR(_curses_has_mouse__doc__, "has_mouse($module, /)\n" @@ -4751,7 +4751,7 @@ _curses_has_mouse(PyObject *module, PyObject *Py_UNUSED(ignored)) return _curses_has_mouse_impl(module); } -#endif /* (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) && (defined(HAVE_CURSES_HAS_MOUSE) || defined(PDCURSES)) */ +#endif /* (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) && (defined(_Py_HAVE_CURSES_HAS_MOUSE) || defined(PDCURSES)) */ #if (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) @@ -5645,7 +5645,7 @@ _curses_termattrs(PyObject *module, PyObject *Py_UNUSED(ignored)) return _curses_termattrs_impl(module); } -#if defined(HAVE_CURSES_TERM_ATTRS) +#if defined(_Py_HAVE_CURSES_TERM_ATTRS) PyDoc_STRVAR(_curses_term_attrs__doc__, "term_attrs($module, /)\n" @@ -5668,7 +5668,7 @@ _curses_term_attrs(PyObject *module, PyObject *Py_UNUSED(ignored)) return _curses_term_attrs_impl(module); } -#endif /* defined(HAVE_CURSES_TERM_ATTRS) */ +#endif /* defined(_Py_HAVE_CURSES_TERM_ATTRS) */ PyDoc_STRVAR(_curses_termname__doc__, "termname($module, /)\n" @@ -6244,7 +6244,7 @@ _curses_slk_attr(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* (defined(NCURSES_EXT_FUNCS) || defined(PDCURSES)) */ -#if defined(HAVE_CURSES_SLK_ATTR_ON) +#if defined(_Py_HAVE_CURSES_SLK_ATTR_ON) PyDoc_STRVAR(_curses_slk_attr_on__doc__, "slk_attr_on($module, attr, /)\n" @@ -6273,9 +6273,9 @@ _curses_slk_attr_on(PyObject *module, PyObject *arg) return return_value; } -#endif /* defined(HAVE_CURSES_SLK_ATTR_ON) */ +#endif /* defined(_Py_HAVE_CURSES_SLK_ATTR_ON) */ -#if defined(HAVE_CURSES_SLK_ATTR_OFF) +#if defined(_Py_HAVE_CURSES_SLK_ATTR_OFF) PyDoc_STRVAR(_curses_slk_attr_off__doc__, "slk_attr_off($module, attr, /)\n" @@ -6304,9 +6304,9 @@ _curses_slk_attr_off(PyObject *module, PyObject *arg) return return_value; } -#endif /* defined(HAVE_CURSES_SLK_ATTR_OFF) */ +#endif /* defined(_Py_HAVE_CURSES_SLK_ATTR_OFF) */ -#if defined(HAVE_CURSES_SLK_ATTR_SET) +#if defined(_Py_HAVE_CURSES_SLK_ATTR_SET) PyDoc_STRVAR(_curses_slk_attr_set__doc__, "slk_attr_set($module, attr, pair=0, /)\n" @@ -6346,9 +6346,9 @@ _curses_slk_attr_set(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* defined(HAVE_CURSES_SLK_ATTR_SET) */ +#endif /* defined(_Py_HAVE_CURSES_SLK_ATTR_SET) */ -#if defined(HAVE_CURSES_SLK_COLOR) +#if defined(_Py_HAVE_CURSES_SLK_COLOR) PyDoc_STRVAR(_curses_slk_color__doc__, "slk_color($module, pair, /)\n" @@ -6377,7 +6377,7 @@ _curses_slk_color(PyObject *module, PyObject *arg) return return_value; } -#endif /* defined(HAVE_CURSES_SLK_COLOR) */ +#endif /* defined(_Py_HAVE_CURSES_SLK_COLOR) */ #if defined(HAVE_CURSES_USE_ENV) @@ -6715,4 +6715,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #define _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_ASSUME_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=422461e62cb86010 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=417ad397dfaa96ce input=a9049054013a1b77]*/ diff --git a/Tools/build/check_capi_macros_ignored.txt b/Tools/build/check_capi_macros_ignored.txt index dfde88c0c800d6c..154f1dbfca039c9 100644 --- a/Tools/build/check_capi_macros_ignored.txt +++ b/Tools/build/check_capi_macros_ignored.txt @@ -116,45 +116,23 @@ HAVE_CONNECT HAVE_COPY_FILE_RANGE HAVE_CTERMID HAVE_CTERMID_R -HAVE_CURSES_DEFINE_KEY HAVE_CURSES_ESCDELAY HAVE_CURSES_FILTER HAVE_CURSES_GETMOUSE HAVE_CURSES_H HAVE_CURSES_HAS_KEY -HAVE_CURSES_HAS_MOUSE HAVE_CURSES_IMMEDOK -HAVE_CURSES_IS_KEYPAD -HAVE_CURSES_IS_LEAVEOK HAVE_CURSES_IS_PAD HAVE_CURSES_IS_TERM_RESIZED -HAVE_CURSES_KEYOK -HAVE_CURSES_KEY_DEFINED -HAVE_CURSES_NEW_PRESCR -HAVE_CURSES_NOFILTER HAVE_CURSES_RESIZETERM HAVE_CURSES_RESIZE_TERM -HAVE_CURSES_SCR_DUMP -HAVE_CURSES_SCR_SET HAVE_CURSES_SET_ESCDELAY HAVE_CURSES_SET_TABSIZE -HAVE_CURSES_SLK_ATTR_OFF -HAVE_CURSES_SLK_ATTR_ON -HAVE_CURSES_SLK_ATTR_SET -HAVE_CURSES_SLK_COLOR HAVE_CURSES_SYNCOK HAVE_CURSES_TABSIZE -HAVE_CURSES_TERM_ATTRS HAVE_CURSES_TYPEAHEAD HAVE_CURSES_USE_ENV -HAVE_CURSES_USE_SCREEN -HAVE_CURSES_USE_WINDOW -HAVE_CURSES_WATTR_GET -HAVE_CURSES_WATTR_OFF -HAVE_CURSES_WATTR_ON -HAVE_CURSES_WATTR_SET HAVE_CURSES_WCHGAT -HAVE_CURSES_WCOLOR_SET HAVE_DB_H HAVE_DECLSPEC_DLL HAVE_DECL_PR_SET_VMA_ANON_NAME diff --git a/configure b/configure index 655558774c4b5fa..0e93826d3494c6d 100755 --- a/configure +++ b/configure @@ -32277,6 +32277,9 @@ fi + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_pad" >&5 printf %s "checking for curses function is_pad... " >&6; } if test ${ac_cv_lib_curses_is_pad+y} @@ -32338,6 +32341,7 @@ fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_term_resized" >&5 printf %s "checking for curses function is_term_resized... " >&6; } if test ${ac_cv_lib_curses_is_term_resized+y} @@ -32399,6 +32403,7 @@ fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function resize_term" >&5 printf %s "checking for curses function resize_term... " >&6; } if test ${ac_cv_lib_curses_resize_term+y} @@ -32460,6 +32465,7 @@ fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function resizeterm" >&5 printf %s "checking for curses function resizeterm... " >&6; } if test ${ac_cv_lib_curses_resizeterm+y} @@ -32521,6 +32527,7 @@ fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function immedok" >&5 printf %s "checking for curses function immedok... " >&6; } if test ${ac_cv_lib_curses_immedok+y} @@ -32582,6 +32589,7 @@ fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function syncok" >&5 printf %s "checking for curses function syncok... " >&6; } if test ${ac_cv_lib_curses_syncok+y} @@ -32643,6 +32651,7 @@ fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wchgat" >&5 printf %s "checking for curses function wchgat... " >&6; } if test ${ac_cv_lib_curses_wchgat+y} @@ -32704,6 +32713,7 @@ fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function filter" >&5 printf %s "checking for curses function filter... " >&6; } if test ${ac_cv_lib_curses_filter+y} @@ -32765,9 +32775,10 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function nofilter" >&5 -printf %s "checking for curses function nofilter... " >&6; } -if test ${ac_cv_lib_curses_nofilter+y} + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function has_key" >&5 +printf %s "checking for curses function has_key... " >&6; } +if test ${ac_cv_lib_curses_has_key+y} then : printf %s "(cached) " >&6 else case e in #( @@ -32793,8 +32804,8 @@ int main (void) { - #ifndef nofilter - void * volatile x = (void *)nofilter; (void)x; + #ifndef has_key + void * volatile x = (void *)has_key; (void)x; #endif ; @@ -32803,9 +32814,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_nofilter=yes + ac_cv_lib_curses_has_key=yes else case e in #( - e) ac_cv_lib_curses_nofilter=no ;; + e) ac_cv_lib_curses_has_key=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -32813,12 +32824,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_nofilter" >&5 -printf "%s\n" "$ac_cv_lib_curses_nofilter" >&6; } - if test "x$ac_cv_lib_curses_nofilter" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_key" >&5 +printf "%s\n" "$ac_cv_lib_curses_has_key" >&6; } + if test "x$ac_cv_lib_curses_has_key" = xyes then : -printf "%s\n" "#define HAVE_CURSES_NOFILTER 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_HAS_KEY 1" >>confdefs.h fi @@ -32826,9 +32837,10 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function has_key" >&5 -printf %s "checking for curses function has_key... " >&6; } -if test ${ac_cv_lib_curses_has_key+y} + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function typeahead" >&5 +printf %s "checking for curses function typeahead... " >&6; } +if test ${ac_cv_lib_curses_typeahead+y} then : printf %s "(cached) " >&6 else case e in #( @@ -32854,8 +32866,8 @@ int main (void) { - #ifndef has_key - void * volatile x = (void *)has_key; (void)x; + #ifndef typeahead + void * volatile x = (void *)typeahead; (void)x; #endif ; @@ -32864,9 +32876,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_has_key=yes + ac_cv_lib_curses_typeahead=yes else case e in #( - e) ac_cv_lib_curses_has_key=no ;; + e) ac_cv_lib_curses_typeahead=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -32874,12 +32886,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_key" >&5 -printf "%s\n" "$ac_cv_lib_curses_has_key" >&6; } - if test "x$ac_cv_lib_curses_has_key" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_typeahead" >&5 +printf "%s\n" "$ac_cv_lib_curses_typeahead" >&6; } + if test "x$ac_cv_lib_curses_typeahead" = xyes then : -printf "%s\n" "#define HAVE_CURSES_HAS_KEY 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_TYPEAHEAD 1" >>confdefs.h fi @@ -32887,9 +32899,10 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function has_mouse" >&5 -printf %s "checking for curses function has_mouse... " >&6; } -if test ${ac_cv_lib_curses_has_mouse+y} + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_env" >&5 +printf %s "checking for curses function use_env... " >&6; } +if test ${ac_cv_lib_curses_use_env+y} then : printf %s "(cached) " >&6 else case e in #( @@ -32915,8 +32928,8 @@ int main (void) { - #ifndef has_mouse - void * volatile x = (void *)has_mouse; (void)x; + #ifndef use_env + void * volatile x = (void *)use_env; (void)x; #endif ; @@ -32925,9 +32938,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_has_mouse=yes + ac_cv_lib_curses_use_env=yes else case e in #( - e) ac_cv_lib_curses_has_mouse=no ;; + e) ac_cv_lib_curses_use_env=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -32935,12 +32948,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_mouse" >&5 -printf "%s\n" "$ac_cv_lib_curses_has_mouse" >&6; } - if test "x$ac_cv_lib_curses_has_mouse" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_env" >&5 +printf "%s\n" "$ac_cv_lib_curses_use_env" >&6; } + if test "x$ac_cv_lib_curses_use_env" = xyes then : -printf "%s\n" "#define HAVE_CURSES_HAS_MOUSE 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_USE_ENV 1" >>confdefs.h fi @@ -32948,9 +32961,10 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_keypad" >&5 -printf %s "checking for curses function is_keypad... " >&6; } -if test ${ac_cv_lib_curses_is_keypad+y} + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function set_escdelay" >&5 +printf %s "checking for curses function set_escdelay... " >&6; } +if test ${ac_cv_lib_curses_set_escdelay+y} then : printf %s "(cached) " >&6 else case e in #( @@ -32976,8 +32990,8 @@ int main (void) { - #ifndef is_keypad - void * volatile x = (void *)is_keypad; (void)x; + #ifndef set_escdelay + void * volatile x = (void *)set_escdelay; (void)x; #endif ; @@ -32986,9 +33000,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_is_keypad=yes + ac_cv_lib_curses_set_escdelay=yes else case e in #( - e) ac_cv_lib_curses_is_keypad=no ;; + e) ac_cv_lib_curses_set_escdelay=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -32996,12 +33010,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_keypad" >&5 -printf "%s\n" "$ac_cv_lib_curses_is_keypad" >&6; } - if test "x$ac_cv_lib_curses_is_keypad" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_set_escdelay" >&5 +printf "%s\n" "$ac_cv_lib_curses_set_escdelay" >&6; } + if test "x$ac_cv_lib_curses_set_escdelay" = xyes then : -printf "%s\n" "#define HAVE_CURSES_IS_KEYPAD 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_SET_ESCDELAY 1" >>confdefs.h fi @@ -33009,9 +33023,10 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_leaveok" >&5 -printf %s "checking for curses function is_leaveok... " >&6; } -if test ${ac_cv_lib_curses_is_leaveok+y} + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function set_tabsize" >&5 +printf %s "checking for curses function set_tabsize... " >&6; } +if test ${ac_cv_lib_curses_set_tabsize+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33037,8 +33052,8 @@ int main (void) { - #ifndef is_leaveok - void * volatile x = (void *)is_leaveok; (void)x; + #ifndef set_tabsize + void * volatile x = (void *)set_tabsize; (void)x; #endif ; @@ -33047,9 +33062,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_is_leaveok=yes + ac_cv_lib_curses_set_tabsize=yes else case e in #( - e) ac_cv_lib_curses_is_leaveok=no ;; + e) ac_cv_lib_curses_set_tabsize=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33057,12 +33072,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_leaveok" >&5 -printf "%s\n" "$ac_cv_lib_curses_is_leaveok" >&6; } - if test "x$ac_cv_lib_curses_is_leaveok" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_set_tabsize" >&5 +printf "%s\n" "$ac_cv_lib_curses_set_tabsize" >&6; } + if test "x$ac_cv_lib_curses_set_tabsize" = xyes then : -printf "%s\n" "#define HAVE_CURSES_IS_LEAVEOK 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_SET_TABSIZE 1" >>confdefs.h fi @@ -33070,9 +33085,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function typeahead" >&5 -printf %s "checking for curses function typeahead... " >&6; } -if test ${ac_cv_lib_curses_typeahead+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses variable ESCDELAY" >&5 +printf %s "checking for curses variable ESCDELAY... " >&6; } +if test ${ac_cv_lib_curses_ESCDELAY+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33098,32 +33113,29 @@ int main (void) { - #ifndef typeahead - void * volatile x = (void *)typeahead; (void)x; - #endif + int x = ESCDELAY; (void)x; ; return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_typeahead=yes + ac_cv_lib_curses_ESCDELAY=yes else case e in #( - e) ac_cv_lib_curses_typeahead=no ;; + e) ac_cv_lib_curses_ESCDELAY=no ;; esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_typeahead" >&5 -printf "%s\n" "$ac_cv_lib_curses_typeahead" >&6; } - if test "x$ac_cv_lib_curses_typeahead" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_ESCDELAY" >&5 +printf "%s\n" "$ac_cv_lib_curses_ESCDELAY" >&6; } + if test "x$ac_cv_lib_curses_ESCDELAY" = xyes then : -printf "%s\n" "#define HAVE_CURSES_TYPEAHEAD 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_ESCDELAY 1" >>confdefs.h fi @@ -33131,9 +33143,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_env" >&5 -printf %s "checking for curses function use_env... " >&6; } -if test ${ac_cv_lib_curses_use_env+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses variable TABSIZE" >&5 +printf %s "checking for curses variable TABSIZE... " >&6; } +if test ${ac_cv_lib_curses_TABSIZE+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33159,32 +33171,29 @@ int main (void) { - #ifndef use_env - void * volatile x = (void *)use_env; (void)x; - #endif + int x = TABSIZE; (void)x; ; return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_use_env=yes + ac_cv_lib_curses_TABSIZE=yes else case e in #( - e) ac_cv_lib_curses_use_env=no ;; + e) ac_cv_lib_curses_TABSIZE=no ;; esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_env" >&5 -printf "%s\n" "$ac_cv_lib_curses_use_env" >&6; } - if test "x$ac_cv_lib_curses_use_env" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_TABSIZE" >&5 +printf "%s\n" "$ac_cv_lib_curses_TABSIZE" >&6; } + if test "x$ac_cv_lib_curses_TABSIZE" = xyes then : -printf "%s\n" "#define HAVE_CURSES_USE_ENV 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_TABSIZE 1" >>confdefs.h fi @@ -33192,9 +33201,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function new_prescr" >&5 -printf %s "checking for curses function new_prescr... " >&6; } -if test ${ac_cv_lib_curses_new_prescr+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function define_key" >&5 +printf %s "checking for curses function define_key... " >&6; } +if test ${ac_cv_lib_curses_define_key+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33220,8 +33231,8 @@ int main (void) { - #ifndef new_prescr - void * volatile x = (void *)new_prescr; (void)x; + #ifndef define_key + void * volatile x = (void *)define_key; (void)x; #endif ; @@ -33230,9 +33241,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_new_prescr=yes + ac_cv_lib_curses_define_key=yes else case e in #( - e) ac_cv_lib_curses_new_prescr=no ;; + e) ac_cv_lib_curses_define_key=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33240,12 +33251,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_new_prescr" >&5 -printf "%s\n" "$ac_cv_lib_curses_new_prescr" >&6; } - if test "x$ac_cv_lib_curses_new_prescr" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_define_key" >&5 +printf "%s\n" "$ac_cv_lib_curses_define_key" >&6; } + if test "x$ac_cv_lib_curses_define_key" = xyes then : -printf "%s\n" "#define HAVE_CURSES_NEW_PRESCR 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_DEFINE_KEY 1" >>confdefs.h fi @@ -33253,9 +33264,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_screen" >&5 -printf %s "checking for curses function use_screen... " >&6; } -if test ${ac_cv_lib_curses_use_screen+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function has_mouse" >&5 +printf %s "checking for curses function has_mouse... " >&6; } +if test ${ac_cv_lib_curses_has_mouse+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33281,8 +33294,8 @@ int main (void) { - #ifndef use_screen - void * volatile x = (void *)use_screen; (void)x; + #ifndef has_mouse + void * volatile x = (void *)has_mouse; (void)x; #endif ; @@ -33291,9 +33304,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_use_screen=yes + ac_cv_lib_curses_has_mouse=yes else case e in #( - e) ac_cv_lib_curses_use_screen=no ;; + e) ac_cv_lib_curses_has_mouse=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33301,12 +33314,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_screen" >&5 -printf "%s\n" "$ac_cv_lib_curses_use_screen" >&6; } - if test "x$ac_cv_lib_curses_use_screen" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_mouse" >&5 +printf "%s\n" "$ac_cv_lib_curses_has_mouse" >&6; } + if test "x$ac_cv_lib_curses_has_mouse" = xyes then : -printf "%s\n" "#define HAVE_CURSES_USE_SCREEN 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_HAS_MOUSE 1" >>confdefs.h fi @@ -33314,9 +33327,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_window" >&5 -printf %s "checking for curses function use_window... " >&6; } -if test ${ac_cv_lib_curses_use_window+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_keypad" >&5 +printf %s "checking for curses function is_keypad... " >&6; } +if test ${ac_cv_lib_curses_is_keypad+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33342,8 +33357,8 @@ int main (void) { - #ifndef use_window - void * volatile x = (void *)use_window; (void)x; + #ifndef is_keypad + void * volatile x = (void *)is_keypad; (void)x; #endif ; @@ -33352,9 +33367,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_use_window=yes + ac_cv_lib_curses_is_keypad=yes else case e in #( - e) ac_cv_lib_curses_use_window=no ;; + e) ac_cv_lib_curses_is_keypad=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33362,12 +33377,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_window" >&5 -printf "%s\n" "$ac_cv_lib_curses_use_window" >&6; } - if test "x$ac_cv_lib_curses_use_window" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_keypad" >&5 +printf "%s\n" "$ac_cv_lib_curses_is_keypad" >&6; } + if test "x$ac_cv_lib_curses_is_keypad" = xyes then : -printf "%s\n" "#define HAVE_CURSES_USE_WINDOW 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_IS_KEYPAD 1" >>confdefs.h fi @@ -33375,9 +33390,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function key_defined" >&5 -printf %s "checking for curses function key_defined... " >&6; } -if test ${ac_cv_lib_curses_key_defined+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_leaveok" >&5 +printf %s "checking for curses function is_leaveok... " >&6; } +if test ${ac_cv_lib_curses_is_leaveok+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33403,8 +33420,8 @@ int main (void) { - #ifndef key_defined - void * volatile x = (void *)key_defined; (void)x; + #ifndef is_leaveok + void * volatile x = (void *)is_leaveok; (void)x; #endif ; @@ -33413,9 +33430,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_key_defined=yes + ac_cv_lib_curses_is_leaveok=yes else case e in #( - e) ac_cv_lib_curses_key_defined=no ;; + e) ac_cv_lib_curses_is_leaveok=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33423,12 +33440,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_key_defined" >&5 -printf "%s\n" "$ac_cv_lib_curses_key_defined" >&6; } - if test "x$ac_cv_lib_curses_key_defined" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_leaveok" >&5 +printf "%s\n" "$ac_cv_lib_curses_is_leaveok" >&6; } + if test "x$ac_cv_lib_curses_is_leaveok" = xyes then : -printf "%s\n" "#define HAVE_CURSES_KEY_DEFINED 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_IS_LEAVEOK 1" >>confdefs.h fi @@ -33436,9 +33453,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function term_attrs" >&5 -printf %s "checking for curses function term_attrs... " >&6; } -if test ${ac_cv_lib_curses_term_attrs+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function key_defined" >&5 +printf %s "checking for curses function key_defined... " >&6; } +if test ${ac_cv_lib_curses_key_defined+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33464,8 +33483,8 @@ int main (void) { - #ifndef term_attrs - void * volatile x = (void *)term_attrs; (void)x; + #ifndef key_defined + void * volatile x = (void *)key_defined; (void)x; #endif ; @@ -33474,9 +33493,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_term_attrs=yes + ac_cv_lib_curses_key_defined=yes else case e in #( - e) ac_cv_lib_curses_term_attrs=no ;; + e) ac_cv_lib_curses_key_defined=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33484,12 +33503,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_term_attrs" >&5 -printf "%s\n" "$ac_cv_lib_curses_term_attrs" >&6; } - if test "x$ac_cv_lib_curses_term_attrs" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_key_defined" >&5 +printf "%s\n" "$ac_cv_lib_curses_key_defined" >&6; } + if test "x$ac_cv_lib_curses_key_defined" = xyes then : -printf "%s\n" "#define HAVE_CURSES_TERM_ATTRS 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_KEY_DEFINED 1" >>confdefs.h fi @@ -33497,9 +33516,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function define_key" >&5 -printf %s "checking for curses function define_key... " >&6; } -if test ${ac_cv_lib_curses_define_key+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function keyok" >&5 +printf %s "checking for curses function keyok... " >&6; } +if test ${ac_cv_lib_curses_keyok+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33525,8 +33546,8 @@ int main (void) { - #ifndef define_key - void * volatile x = (void *)define_key; (void)x; + #ifndef keyok + void * volatile x = (void *)keyok; (void)x; #endif ; @@ -33535,9 +33556,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_define_key=yes + ac_cv_lib_curses_keyok=yes else case e in #( - e) ac_cv_lib_curses_define_key=no ;; + e) ac_cv_lib_curses_keyok=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33545,12 +33566,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_define_key" >&5 -printf "%s\n" "$ac_cv_lib_curses_define_key" >&6; } - if test "x$ac_cv_lib_curses_define_key" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_keyok" >&5 +printf "%s\n" "$ac_cv_lib_curses_keyok" >&6; } + if test "x$ac_cv_lib_curses_keyok" = xyes then : -printf "%s\n" "#define HAVE_CURSES_DEFINE_KEY 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_KEYOK 1" >>confdefs.h fi @@ -33558,9 +33579,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function keyok" >&5 -printf %s "checking for curses function keyok... " >&6; } -if test ${ac_cv_lib_curses_keyok+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function new_prescr" >&5 +printf %s "checking for curses function new_prescr... " >&6; } +if test ${ac_cv_lib_curses_new_prescr+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33586,8 +33609,8 @@ int main (void) { - #ifndef keyok - void * volatile x = (void *)keyok; (void)x; + #ifndef new_prescr + void * volatile x = (void *)new_prescr; (void)x; #endif ; @@ -33596,9 +33619,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_keyok=yes + ac_cv_lib_curses_new_prescr=yes else case e in #( - e) ac_cv_lib_curses_keyok=no ;; + e) ac_cv_lib_curses_new_prescr=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33606,12 +33629,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_keyok" >&5 -printf "%s\n" "$ac_cv_lib_curses_keyok" >&6; } - if test "x$ac_cv_lib_curses_keyok" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_new_prescr" >&5 +printf "%s\n" "$ac_cv_lib_curses_new_prescr" >&6; } + if test "x$ac_cv_lib_curses_new_prescr" = xyes then : -printf "%s\n" "#define HAVE_CURSES_KEYOK 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_NEW_PRESCR 1" >>confdefs.h fi @@ -33619,9 +33642,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function set_escdelay" >&5 -printf %s "checking for curses function set_escdelay... " >&6; } -if test ${ac_cv_lib_curses_set_escdelay+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function nofilter" >&5 +printf %s "checking for curses function nofilter... " >&6; } +if test ${ac_cv_lib_curses_nofilter+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33647,8 +33672,8 @@ int main (void) { - #ifndef set_escdelay - void * volatile x = (void *)set_escdelay; (void)x; + #ifndef nofilter + void * volatile x = (void *)nofilter; (void)x; #endif ; @@ -33657,9 +33682,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_set_escdelay=yes + ac_cv_lib_curses_nofilter=yes else case e in #( - e) ac_cv_lib_curses_set_escdelay=no ;; + e) ac_cv_lib_curses_nofilter=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33667,12 +33692,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_set_escdelay" >&5 -printf "%s\n" "$ac_cv_lib_curses_set_escdelay" >&6; } - if test "x$ac_cv_lib_curses_set_escdelay" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_nofilter" >&5 +printf "%s\n" "$ac_cv_lib_curses_nofilter" >&6; } + if test "x$ac_cv_lib_curses_nofilter" = xyes then : -printf "%s\n" "#define HAVE_CURSES_SET_ESCDELAY 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_NOFILTER 1" >>confdefs.h fi @@ -33680,9 +33705,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function set_tabsize" >&5 -printf %s "checking for curses function set_tabsize... " >&6; } -if test ${ac_cv_lib_curses_set_tabsize+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_attr_off" >&5 +printf %s "checking for curses function slk_attr_off... " >&6; } +if test ${ac_cv_lib_curses_slk_attr_off+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33708,8 +33735,8 @@ int main (void) { - #ifndef set_tabsize - void * volatile x = (void *)set_tabsize; (void)x; + #ifndef slk_attr_off + void * volatile x = (void *)slk_attr_off; (void)x; #endif ; @@ -33718,9 +33745,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_set_tabsize=yes + ac_cv_lib_curses_slk_attr_off=yes else case e in #( - e) ac_cv_lib_curses_set_tabsize=no ;; + e) ac_cv_lib_curses_slk_attr_off=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33728,12 +33755,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_set_tabsize" >&5 -printf "%s\n" "$ac_cv_lib_curses_set_tabsize" >&6; } - if test "x$ac_cv_lib_curses_set_tabsize" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_attr_off" >&5 +printf "%s\n" "$ac_cv_lib_curses_slk_attr_off" >&6; } + if test "x$ac_cv_lib_curses_slk_attr_off" = xyes then : -printf "%s\n" "#define HAVE_CURSES_SET_TABSIZE 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_SLK_ATTR_OFF 1" >>confdefs.h fi @@ -33741,9 +33768,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_get" >&5 -printf %s "checking for curses function wattr_get... " >&6; } -if test ${ac_cv_lib_curses_wattr_get+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_attr_on" >&5 +printf %s "checking for curses function slk_attr_on... " >&6; } +if test ${ac_cv_lib_curses_slk_attr_on+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33769,8 +33798,8 @@ int main (void) { - #ifndef wattr_get - void * volatile x = (void *)wattr_get; (void)x; + #ifndef slk_attr_on + void * volatile x = (void *)slk_attr_on; (void)x; #endif ; @@ -33779,9 +33808,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_wattr_get=yes + ac_cv_lib_curses_slk_attr_on=yes else case e in #( - e) ac_cv_lib_curses_wattr_get=no ;; + e) ac_cv_lib_curses_slk_attr_on=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33789,12 +33818,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_get" >&5 -printf "%s\n" "$ac_cv_lib_curses_wattr_get" >&6; } - if test "x$ac_cv_lib_curses_wattr_get" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_attr_on" >&5 +printf "%s\n" "$ac_cv_lib_curses_slk_attr_on" >&6; } + if test "x$ac_cv_lib_curses_slk_attr_on" = xyes then : -printf "%s\n" "#define HAVE_CURSES_WATTR_GET 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_SLK_ATTR_ON 1" >>confdefs.h fi @@ -33802,9 +33831,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_set" >&5 -printf %s "checking for curses function wattr_set... " >&6; } -if test ${ac_cv_lib_curses_wattr_set+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_attr_set" >&5 +printf %s "checking for curses function slk_attr_set... " >&6; } +if test ${ac_cv_lib_curses_slk_attr_set+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33830,8 +33861,8 @@ int main (void) { - #ifndef wattr_set - void * volatile x = (void *)wattr_set; (void)x; + #ifndef slk_attr_set + void * volatile x = (void *)slk_attr_set; (void)x; #endif ; @@ -33840,9 +33871,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_wattr_set=yes + ac_cv_lib_curses_slk_attr_set=yes else case e in #( - e) ac_cv_lib_curses_wattr_set=no ;; + e) ac_cv_lib_curses_slk_attr_set=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33850,12 +33881,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_set" >&5 -printf "%s\n" "$ac_cv_lib_curses_wattr_set" >&6; } - if test "x$ac_cv_lib_curses_wattr_set" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_attr_set" >&5 +printf "%s\n" "$ac_cv_lib_curses_slk_attr_set" >&6; } + if test "x$ac_cv_lib_curses_slk_attr_set" = xyes then : -printf "%s\n" "#define HAVE_CURSES_WATTR_SET 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_SLK_ATTR_SET 1" >>confdefs.h fi @@ -33863,9 +33894,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_on" >&5 -printf %s "checking for curses function wattr_on... " >&6; } -if test ${ac_cv_lib_curses_wattr_on+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_color" >&5 +printf %s "checking for curses function slk_color... " >&6; } +if test ${ac_cv_lib_curses_slk_color+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33891,8 +33924,8 @@ int main (void) { - #ifndef wattr_on - void * volatile x = (void *)wattr_on; (void)x; + #ifndef slk_color + void * volatile x = (void *)slk_color; (void)x; #endif ; @@ -33901,9 +33934,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_wattr_on=yes + ac_cv_lib_curses_slk_color=yes else case e in #( - e) ac_cv_lib_curses_wattr_on=no ;; + e) ac_cv_lib_curses_slk_color=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33911,12 +33944,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_on" >&5 -printf "%s\n" "$ac_cv_lib_curses_wattr_on" >&6; } - if test "x$ac_cv_lib_curses_wattr_on" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_color" >&5 +printf "%s\n" "$ac_cv_lib_curses_slk_color" >&6; } + if test "x$ac_cv_lib_curses_slk_color" = xyes then : -printf "%s\n" "#define HAVE_CURSES_WATTR_ON 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_SLK_COLOR 1" >>confdefs.h fi @@ -33924,9 +33957,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_off" >&5 -printf %s "checking for curses function wattr_off... " >&6; } -if test ${ac_cv_lib_curses_wattr_off+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function term_attrs" >&5 +printf %s "checking for curses function term_attrs... " >&6; } +if test ${ac_cv_lib_curses_term_attrs+y} then : printf %s "(cached) " >&6 else case e in #( @@ -33952,8 +33987,8 @@ int main (void) { - #ifndef wattr_off - void * volatile x = (void *)wattr_off; (void)x; + #ifndef term_attrs + void * volatile x = (void *)term_attrs; (void)x; #endif ; @@ -33962,9 +33997,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_wattr_off=yes + ac_cv_lib_curses_term_attrs=yes else case e in #( - e) ac_cv_lib_curses_wattr_off=no ;; + e) ac_cv_lib_curses_term_attrs=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -33972,12 +34007,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_off" >&5 -printf "%s\n" "$ac_cv_lib_curses_wattr_off" >&6; } - if test "x$ac_cv_lib_curses_wattr_off" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_term_attrs" >&5 +printf "%s\n" "$ac_cv_lib_curses_term_attrs" >&6; } + if test "x$ac_cv_lib_curses_term_attrs" = xyes then : -printf "%s\n" "#define HAVE_CURSES_WATTR_OFF 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_TERM_ATTRS 1" >>confdefs.h fi @@ -33985,9 +34020,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wcolor_set" >&5 -printf %s "checking for curses function wcolor_set... " >&6; } -if test ${ac_cv_lib_curses_wcolor_set+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_screen" >&5 +printf %s "checking for curses function use_screen... " >&6; } +if test ${ac_cv_lib_curses_use_screen+y} then : printf %s "(cached) " >&6 else case e in #( @@ -34013,8 +34050,8 @@ int main (void) { - #ifndef wcolor_set - void * volatile x = (void *)wcolor_set; (void)x; + #ifndef use_screen + void * volatile x = (void *)use_screen; (void)x; #endif ; @@ -34023,9 +34060,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_wcolor_set=yes + ac_cv_lib_curses_use_screen=yes else case e in #( - e) ac_cv_lib_curses_wcolor_set=no ;; + e) ac_cv_lib_curses_use_screen=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -34033,12 +34070,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wcolor_set" >&5 -printf "%s\n" "$ac_cv_lib_curses_wcolor_set" >&6; } - if test "x$ac_cv_lib_curses_wcolor_set" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_screen" >&5 +printf "%s\n" "$ac_cv_lib_curses_use_screen" >&6; } + if test "x$ac_cv_lib_curses_use_screen" = xyes then : -printf "%s\n" "#define HAVE_CURSES_WCOLOR_SET 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_USE_SCREEN 1" >>confdefs.h fi @@ -34046,9 +34083,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_attr_on" >&5 -printf %s "checking for curses function slk_attr_on... " >&6; } -if test ${ac_cv_lib_curses_slk_attr_on+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_window" >&5 +printf %s "checking for curses function use_window... " >&6; } +if test ${ac_cv_lib_curses_use_window+y} then : printf %s "(cached) " >&6 else case e in #( @@ -34074,8 +34113,8 @@ int main (void) { - #ifndef slk_attr_on - void * volatile x = (void *)slk_attr_on; (void)x; + #ifndef use_window + void * volatile x = (void *)use_window; (void)x; #endif ; @@ -34084,9 +34123,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_slk_attr_on=yes + ac_cv_lib_curses_use_window=yes else case e in #( - e) ac_cv_lib_curses_slk_attr_on=no ;; + e) ac_cv_lib_curses_use_window=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -34094,12 +34133,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_attr_on" >&5 -printf "%s\n" "$ac_cv_lib_curses_slk_attr_on" >&6; } - if test "x$ac_cv_lib_curses_slk_attr_on" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_window" >&5 +printf "%s\n" "$ac_cv_lib_curses_use_window" >&6; } + if test "x$ac_cv_lib_curses_use_window" = xyes then : -printf "%s\n" "#define HAVE_CURSES_SLK_ATTR_ON 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_USE_WINDOW 1" >>confdefs.h fi @@ -34107,9 +34146,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_attr_off" >&5 -printf %s "checking for curses function slk_attr_off... " >&6; } -if test ${ac_cv_lib_curses_slk_attr_off+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_get" >&5 +printf %s "checking for curses function wattr_get... " >&6; } +if test ${ac_cv_lib_curses_wattr_get+y} then : printf %s "(cached) " >&6 else case e in #( @@ -34135,8 +34176,8 @@ int main (void) { - #ifndef slk_attr_off - void * volatile x = (void *)slk_attr_off; (void)x; + #ifndef wattr_get + void * volatile x = (void *)wattr_get; (void)x; #endif ; @@ -34145,9 +34186,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_slk_attr_off=yes + ac_cv_lib_curses_wattr_get=yes else case e in #( - e) ac_cv_lib_curses_slk_attr_off=no ;; + e) ac_cv_lib_curses_wattr_get=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -34155,12 +34196,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_attr_off" >&5 -printf "%s\n" "$ac_cv_lib_curses_slk_attr_off" >&6; } - if test "x$ac_cv_lib_curses_slk_attr_off" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_get" >&5 +printf "%s\n" "$ac_cv_lib_curses_wattr_get" >&6; } + if test "x$ac_cv_lib_curses_wattr_get" = xyes then : -printf "%s\n" "#define HAVE_CURSES_SLK_ATTR_OFF 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_WATTR_GET 1" >>confdefs.h fi @@ -34168,9 +34209,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_attr_set" >&5 -printf %s "checking for curses function slk_attr_set... " >&6; } -if test ${ac_cv_lib_curses_slk_attr_set+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_off" >&5 +printf %s "checking for curses function wattr_off... " >&6; } +if test ${ac_cv_lib_curses_wattr_off+y} then : printf %s "(cached) " >&6 else case e in #( @@ -34196,8 +34239,8 @@ int main (void) { - #ifndef slk_attr_set - void * volatile x = (void *)slk_attr_set; (void)x; + #ifndef wattr_off + void * volatile x = (void *)wattr_off; (void)x; #endif ; @@ -34206,9 +34249,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_slk_attr_set=yes + ac_cv_lib_curses_wattr_off=yes else case e in #( - e) ac_cv_lib_curses_slk_attr_set=no ;; + e) ac_cv_lib_curses_wattr_off=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -34216,12 +34259,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_attr_set" >&5 -printf "%s\n" "$ac_cv_lib_curses_slk_attr_set" >&6; } - if test "x$ac_cv_lib_curses_slk_attr_set" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_off" >&5 +printf "%s\n" "$ac_cv_lib_curses_wattr_off" >&6; } + if test "x$ac_cv_lib_curses_wattr_off" = xyes then : -printf "%s\n" "#define HAVE_CURSES_SLK_ATTR_SET 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_WATTR_OFF 1" >>confdefs.h fi @@ -34229,9 +34272,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_color" >&5 -printf %s "checking for curses function slk_color... " >&6; } -if test ${ac_cv_lib_curses_slk_color+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_on" >&5 +printf %s "checking for curses function wattr_on... " >&6; } +if test ${ac_cv_lib_curses_wattr_on+y} then : printf %s "(cached) " >&6 else case e in #( @@ -34257,8 +34302,8 @@ int main (void) { - #ifndef slk_color - void * volatile x = (void *)slk_color; (void)x; + #ifndef wattr_on + void * volatile x = (void *)wattr_on; (void)x; #endif ; @@ -34267,9 +34312,9 @@ main (void) _ACEOF if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_slk_color=yes + ac_cv_lib_curses_wattr_on=yes else case e in #( - e) ac_cv_lib_curses_slk_color=no ;; + e) ac_cv_lib_curses_wattr_on=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ @@ -34277,12 +34322,12 @@ rm -f core conftest.err conftest.$ac_objext conftest.beam \ ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_color" >&5 -printf "%s\n" "$ac_cv_lib_curses_slk_color" >&6; } - if test "x$ac_cv_lib_curses_slk_color" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_on" >&5 +printf "%s\n" "$ac_cv_lib_curses_wattr_on" >&6; } + if test "x$ac_cv_lib_curses_wattr_on" = xyes then : -printf "%s\n" "#define HAVE_CURSES_SLK_COLOR 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_WATTR_ON 1" >>confdefs.h fi @@ -34290,9 +34335,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses variable ESCDELAY" >&5 -printf %s "checking for curses variable ESCDELAY... " >&6; } -if test ${ac_cv_lib_curses_ESCDELAY+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_set" >&5 +printf %s "checking for curses function wattr_set... " >&6; } +if test ${ac_cv_lib_curses_wattr_set+y} then : printf %s "(cached) " >&6 else case e in #( @@ -34318,29 +34365,32 @@ int main (void) { - int x = ESCDELAY; (void)x; + #ifndef wattr_set + void * volatile x = (void *)wattr_set; (void)x; + #endif ; return 0; } _ACEOF -if ac_fn_c_try_compile "$LINENO" +if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_ESCDELAY=yes + ac_cv_lib_curses_wattr_set=yes else case e in #( - e) ac_cv_lib_curses_ESCDELAY=no ;; + e) ac_cv_lib_curses_wattr_set=no ;; esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_ESCDELAY" >&5 -printf "%s\n" "$ac_cv_lib_curses_ESCDELAY" >&6; } - if test "x$ac_cv_lib_curses_ESCDELAY" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_set" >&5 +printf "%s\n" "$ac_cv_lib_curses_wattr_set" >&6; } + if test "x$ac_cv_lib_curses_wattr_set" = xyes then : -printf "%s\n" "#define HAVE_CURSES_ESCDELAY 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_WATTR_SET 1" >>confdefs.h fi @@ -34348,9 +34398,11 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses variable TABSIZE" >&5 -printf %s "checking for curses variable TABSIZE... " >&6; } -if test ${ac_cv_lib_curses_TABSIZE+y} + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wcolor_set" >&5 +printf %s "checking for curses function wcolor_set... " >&6; } +if test ${ac_cv_lib_curses_wcolor_set+y} then : printf %s "(cached) " >&6 else case e in #( @@ -34376,35 +34428,39 @@ int main (void) { - int x = TABSIZE; (void)x; + #ifndef wcolor_set + void * volatile x = (void *)wcolor_set; (void)x; + #endif ; return 0; } _ACEOF -if ac_fn_c_try_compile "$LINENO" +if ac_fn_c_try_link "$LINENO" then : - ac_cv_lib_curses_TABSIZE=yes + ac_cv_lib_curses_wcolor_set=yes else case e in #( - e) ac_cv_lib_curses_TABSIZE=no ;; + e) ac_cv_lib_curses_wcolor_set=no ;; esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_TABSIZE" >&5 -printf "%s\n" "$ac_cv_lib_curses_TABSIZE" >&6; } - if test "x$ac_cv_lib_curses_TABSIZE" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wcolor_set" >&5 +printf "%s\n" "$ac_cv_lib_curses_wcolor_set" >&6; } + if test "x$ac_cv_lib_curses_wcolor_set" = xyes then : -printf "%s\n" "#define HAVE_CURSES_TABSIZE 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_WCOLOR_SET 1" >>confdefs.h fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncurses-style curses function getmouse" >&5 printf %s "checking for ncurses-style curses function getmouse... " >&6; } if test ${ac_cv_lib_curses_getmouse+y} @@ -34458,6 +34514,7 @@ fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function scr_dump" >&5 printf %s "checking for curses function scr_dump... " >&6; } if test ${ac_cv_lib_curses_scr_dump+y} @@ -34511,7 +34568,7 @@ printf "%s\n" "$ac_cv_lib_curses_scr_dump" >&6; } if test "x$ac_cv_lib_curses_scr_dump" = xyes then : -printf "%s\n" "#define HAVE_CURSES_SCR_DUMP 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_SCR_DUMP 1" >>confdefs.h fi @@ -34519,6 +34576,8 @@ fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function scr_set" >&5 printf %s "checking for curses function scr_set... " >&6; } if test ${ac_cv_lib_curses_scr_set+y} @@ -34572,12 +34631,14 @@ printf "%s\n" "$ac_cv_lib_curses_scr_set" >&6; } if test "x$ac_cv_lib_curses_scr_set" = xyes then : -printf "%s\n" "#define HAVE_CURSES_SCR_SET 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_CURSES_SCR_SET 1" >>confdefs.h fi + + CPPFLAGS=$ac_save_cppflags fi diff --git a/configure.ac b/configure.ac index f6f4055079eaa1b..234e2fbc920822f 100644 --- a/configure.ac +++ b/configure.ac @@ -7666,6 +7666,7 @@ dnl lacks, for example term_attrs() in the narrow ncurses library. AC_DEFUN([PY_CHECK_CURSES_FUNC], [ AS_VAR_PUSHDEF([py_var], [ac_cv_lib_curses_$1]) AS_VAR_PUSHDEF([py_define], [HAVE_CURSES_]m4_toupper($1)) + AS_VAR_PUSHDEF([py_define], m4_ifblank([$3], [[HAVE_CURSES_]m4_toupper($1)], [$3])) AC_CACHE_CHECK( [for curses function $1], [py_var], @@ -7686,6 +7687,13 @@ AC_DEFUN([PY_CHECK_CURSES_FUNC], AS_VAR_POPDEF([py_define]) ]) +dnl PY_CHECK_CURSES_FUNC_PRIVATE(FUNCTION, [INCLUDES], [AC_DEFINE-VAR]) +dnl Similar to PY_CHECK_CURSES_FUNC but define macro _Py_HAVE_CURSES_xxx, +dnl instead of HAVE_CURSES_xxx +AC_DEFUN([PY_CHECK_CURSES_FUNC_PRIVATE], +[ PY_CHECK_CURSES_FUNC([$1], [$2], m4_ifblank([$3], [[_Py_HAVE_CURSES_]m4_toupper($1)], [$3])) +]) + dnl PY_CHECK_CURSES_VAR(VARIABLE) dnl Like PY_CHECK_CURSES_FUNC, but for an integer variable (or macro), such as dnl ESCDELAY, which a function probe cannot detect. @@ -7718,36 +7726,37 @@ PY_CHECK_CURSES_FUNC([immedok]) PY_CHECK_CURSES_FUNC([syncok]) PY_CHECK_CURSES_FUNC([wchgat]) PY_CHECK_CURSES_FUNC([filter]) -PY_CHECK_CURSES_FUNC([nofilter]) PY_CHECK_CURSES_FUNC([has_key]) -PY_CHECK_CURSES_FUNC([has_mouse]) -PY_CHECK_CURSES_FUNC([is_keypad]) -PY_CHECK_CURSES_FUNC([is_leaveok]) PY_CHECK_CURSES_FUNC([typeahead]) PY_CHECK_CURSES_FUNC([use_env]) -PY_CHECK_CURSES_FUNC([new_prescr]) -PY_CHECK_CURSES_FUNC([use_screen]) -PY_CHECK_CURSES_FUNC([use_window]) -PY_CHECK_CURSES_FUNC([key_defined]) -PY_CHECK_CURSES_FUNC([term_attrs]) -PY_CHECK_CURSES_FUNC([define_key]) -PY_CHECK_CURSES_FUNC([keyok]) PY_CHECK_CURSES_FUNC([set_escdelay]) PY_CHECK_CURSES_FUNC([set_tabsize]) dnl The X/Open attr_t and soft-label attribute functions are absent on old dnl SVr4 curses (e.g. illumos). -PY_CHECK_CURSES_FUNC([wattr_get]) -PY_CHECK_CURSES_FUNC([wattr_set]) -PY_CHECK_CURSES_FUNC([wattr_on]) -PY_CHECK_CURSES_FUNC([wattr_off]) -PY_CHECK_CURSES_FUNC([wcolor_set]) -PY_CHECK_CURSES_FUNC([slk_attr_on]) -PY_CHECK_CURSES_FUNC([slk_attr_off]) -PY_CHECK_CURSES_FUNC([slk_attr_set]) -PY_CHECK_CURSES_FUNC([slk_color]) PY_CHECK_CURSES_VAR([ESCDELAY]) PY_CHECK_CURSES_VAR([TABSIZE]) +PY_CHECK_CURSES_FUNC_PRIVATE([define_key]) +PY_CHECK_CURSES_FUNC_PRIVATE([has_mouse]) +PY_CHECK_CURSES_FUNC_PRIVATE([is_keypad]) +PY_CHECK_CURSES_FUNC_PRIVATE([is_leaveok]) +PY_CHECK_CURSES_FUNC_PRIVATE([key_defined]) +PY_CHECK_CURSES_FUNC_PRIVATE([keyok]) +PY_CHECK_CURSES_FUNC_PRIVATE([new_prescr]) +PY_CHECK_CURSES_FUNC_PRIVATE([nofilter]) +PY_CHECK_CURSES_FUNC_PRIVATE([slk_attr_off]) +PY_CHECK_CURSES_FUNC_PRIVATE([slk_attr_on]) +PY_CHECK_CURSES_FUNC_PRIVATE([slk_attr_set]) +PY_CHECK_CURSES_FUNC_PRIVATE([slk_color]) +PY_CHECK_CURSES_FUNC_PRIVATE([term_attrs]) +PY_CHECK_CURSES_FUNC_PRIVATE([use_screen]) +PY_CHECK_CURSES_FUNC_PRIVATE([use_window]) +PY_CHECK_CURSES_FUNC_PRIVATE([wattr_get]) +PY_CHECK_CURSES_FUNC_PRIVATE([wattr_off]) +PY_CHECK_CURSES_FUNC_PRIVATE([wattr_on]) +PY_CHECK_CURSES_FUNC_PRIVATE([wattr_set]) +PY_CHECK_CURSES_FUNC_PRIVATE([wcolor_set]) + dnl Probe for the X/Open getmouse(MEVENT *) signature specifically: PDCurses dnl declares an incompatible getmouse(void) unless built for the ncurses mouse API. AC_CACHE_CHECK([for ncurses-style curses function getmouse], @@ -7762,8 +7771,9 @@ AS_VAR_IF([ac_cv_lib_curses_getmouse], [yes], dnl scr_dump gates scr_dump/scr_restore/scr_init; scr_set is separate, since dnl old SVr4 curses (e.g. illumos) has the former but not scr_set. -PY_CHECK_CURSES_FUNC([scr_dump]) -PY_CHECK_CURSES_FUNC([scr_set]) +PY_CHECK_CURSES_FUNC_PRIVATE([scr_dump]) +PY_CHECK_CURSES_FUNC_PRIVATE([scr_set]) + CPPFLAGS=$ac_save_cppflags ])dnl have_curses != no ])dnl save env diff --git a/pyconfig.h.in b/pyconfig.h.in index 90d8a07df66083e..1d4f445a8b1fdaa 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -185,9 +185,6 @@ /* Define if you have the 'ctermid_r' function. */ #undef HAVE_CTERMID_R -/* Define if you have the 'define_key' function. */ -#undef HAVE_CURSES_DEFINE_KEY - /* Define if you have the 'ESCDELAY' variable. */ #undef HAVE_CURSES_ESCDELAY @@ -203,105 +200,42 @@ /* Define if you have the 'has_key' function. */ #undef HAVE_CURSES_HAS_KEY -/* Define if you have the 'has_mouse' function. */ -#undef HAVE_CURSES_HAS_MOUSE - /* Define if you have the 'immedok' function. */ #undef HAVE_CURSES_IMMEDOK -/* Define if you have the 'is_keypad' function. */ -#undef HAVE_CURSES_IS_KEYPAD - -/* Define if you have the 'is_leaveok' function. */ -#undef HAVE_CURSES_IS_LEAVEOK - /* Define if you have the 'is_pad' function. */ #undef HAVE_CURSES_IS_PAD /* Define if you have the 'is_term_resized' function. */ #undef HAVE_CURSES_IS_TERM_RESIZED -/* Define if you have the 'keyok' function. */ -#undef HAVE_CURSES_KEYOK - -/* Define if you have the 'key_defined' function. */ -#undef HAVE_CURSES_KEY_DEFINED - -/* Define if you have the 'new_prescr' function. */ -#undef HAVE_CURSES_NEW_PRESCR - -/* Define if you have the 'nofilter' function. */ -#undef HAVE_CURSES_NOFILTER - /* Define if you have the 'resizeterm' function. */ #undef HAVE_CURSES_RESIZETERM /* Define if you have the 'resize_term' function. */ #undef HAVE_CURSES_RESIZE_TERM -/* Define if you have the 'scr_dump' function. */ -#undef HAVE_CURSES_SCR_DUMP - -/* Define if you have the 'scr_set' function. */ -#undef HAVE_CURSES_SCR_SET - /* Define if you have the 'set_escdelay' function. */ #undef HAVE_CURSES_SET_ESCDELAY /* Define if you have the 'set_tabsize' function. */ #undef HAVE_CURSES_SET_TABSIZE -/* Define if you have the 'slk_attr_off' function. */ -#undef HAVE_CURSES_SLK_ATTR_OFF - -/* Define if you have the 'slk_attr_on' function. */ -#undef HAVE_CURSES_SLK_ATTR_ON - -/* Define if you have the 'slk_attr_set' function. */ -#undef HAVE_CURSES_SLK_ATTR_SET - -/* Define if you have the 'slk_color' function. */ -#undef HAVE_CURSES_SLK_COLOR - /* Define if you have the 'syncok' function. */ #undef HAVE_CURSES_SYNCOK /* Define if you have the 'TABSIZE' variable. */ #undef HAVE_CURSES_TABSIZE -/* Define if you have the 'term_attrs' function. */ -#undef HAVE_CURSES_TERM_ATTRS - /* Define if you have the 'typeahead' function. */ #undef HAVE_CURSES_TYPEAHEAD /* Define if you have the 'use_env' function. */ #undef HAVE_CURSES_USE_ENV -/* Define if you have the 'use_screen' function. */ -#undef HAVE_CURSES_USE_SCREEN - -/* Define if you have the 'use_window' function. */ -#undef HAVE_CURSES_USE_WINDOW - -/* Define if you have the 'wattr_get' function. */ -#undef HAVE_CURSES_WATTR_GET - -/* Define if you have the 'wattr_off' function. */ -#undef HAVE_CURSES_WATTR_OFF - -/* Define if you have the 'wattr_on' function. */ -#undef HAVE_CURSES_WATTR_ON - -/* Define if you have the 'wattr_set' function. */ -#undef HAVE_CURSES_WATTR_SET - /* Define if you have the 'wchgat' function. */ #undef HAVE_CURSES_WCHGAT -/* Define if you have the 'wcolor_set' function. */ -#undef HAVE_CURSES_WCOLOR_SET - /* Define to 1 if you have the header file. */ #undef HAVE_DB_H @@ -2117,6 +2051,72 @@ /* Define if you have the 'cospi' function. */ #undef _Py_HAVE_COSPI +/* Define if you have the 'define_key' function. */ +#undef _Py_HAVE_CURSES_DEFINE_KEY + +/* Define if you have the 'has_mouse' function. */ +#undef _Py_HAVE_CURSES_HAS_MOUSE + +/* Define if you have the 'is_keypad' function. */ +#undef _Py_HAVE_CURSES_IS_KEYPAD + +/* Define if you have the 'is_leaveok' function. */ +#undef _Py_HAVE_CURSES_IS_LEAVEOK + +/* Define if you have the 'keyok' function. */ +#undef _Py_HAVE_CURSES_KEYOK + +/* Define if you have the 'key_defined' function. */ +#undef _Py_HAVE_CURSES_KEY_DEFINED + +/* Define if you have the 'new_prescr' function. */ +#undef _Py_HAVE_CURSES_NEW_PRESCR + +/* Define if you have the 'nofilter' function. */ +#undef _Py_HAVE_CURSES_NOFILTER + +/* Define if you have the 'scr_dump' function. */ +#undef _Py_HAVE_CURSES_SCR_DUMP + +/* Define if you have the 'scr_set' function. */ +#undef _Py_HAVE_CURSES_SCR_SET + +/* Define if you have the 'slk_attr_off' function. */ +#undef _Py_HAVE_CURSES_SLK_ATTR_OFF + +/* Define if you have the 'slk_attr_on' function. */ +#undef _Py_HAVE_CURSES_SLK_ATTR_ON + +/* Define if you have the 'slk_attr_set' function. */ +#undef _Py_HAVE_CURSES_SLK_ATTR_SET + +/* Define if you have the 'slk_color' function. */ +#undef _Py_HAVE_CURSES_SLK_COLOR + +/* Define if you have the 'term_attrs' function. */ +#undef _Py_HAVE_CURSES_TERM_ATTRS + +/* Define if you have the 'use_screen' function. */ +#undef _Py_HAVE_CURSES_USE_SCREEN + +/* Define if you have the 'use_window' function. */ +#undef _Py_HAVE_CURSES_USE_WINDOW + +/* Define if you have the 'wattr_get' function. */ +#undef _Py_HAVE_CURSES_WATTR_GET + +/* Define if you have the 'wattr_off' function. */ +#undef _Py_HAVE_CURSES_WATTR_OFF + +/* Define if you have the 'wattr_on' function. */ +#undef _Py_HAVE_CURSES_WATTR_ON + +/* Define if you have the 'wattr_set' function. */ +#undef _Py_HAVE_CURSES_WATTR_SET + +/* Define if you have the 'wcolor_set' function. */ +#undef _Py_HAVE_CURSES_WCOLOR_SET + /* Define if compiler supports __builtin_shufflevector with 128-bit vectors AND the target architecture has native SIMD (not just API availability) */ #undef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR From 53f4cad43b87292aca163ea5a92d130bd024981d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 19 Nov 2026 18:15:05 +0100 Subject: [PATCH 208/229] Display test header when running python -m test --tsan (#162249) When running "python -m test" with --tsan or --tsan-parallel, display the test header (log TSAN_OPTIONS env var). --- Lib/test/libregrtest/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 032cb94223a7e65..547ef232c204940 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -210,9 +210,11 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList if self.tsan: setup_tsan_tests(self.cmdline_args) + self.want_header = True if self.tsan_parallel: setup_tsan_parallel_tests(self.cmdline_args) + self.want_header = True alltests = findtests(testdir=self.test_dir, exclude=exclude_tests) From 562f08d6910f01e98f90e589ca5ee065fcbae53e Mon Sep 17 00:00:00 2001 From: 0xd6cb6d73 <72226198+0xd6cb6d73@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:26:23 +0100 Subject: [PATCH 209/229] gh-143663: Perform path normalization in TarFile.extractall() (GH-143665) only perform path separator replacement at lstat callsite --- Lib/tarfile.py | 2 +- .../next/Library/2026-01-10-17-39-09.gh-issue-143663.9LEFAf.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-01-10-17-39-09.gh-issue-143663.9LEFAf.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index b052ca0dd407292..ca6aea8396aaa1e 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2554,7 +2554,7 @@ def extractall(self, path=".", members=None, *, numeric_owner=False, continue dirpath = os.path.join(path, tarinfo.name) try: - lstat = os.lstat(dirpath) + lstat = os.lstat(dirpath.replace(r'/', os.sep)) except FileNotFoundError: self._log_no_directory_fixup(tarinfo, 'missing') continue diff --git a/Misc/NEWS.d/next/Library/2026-01-10-17-39-09.gh-issue-143663.9LEFAf.rst b/Misc/NEWS.d/next/Library/2026-01-10-17-39-09.gh-issue-143663.9LEFAf.rst new file mode 100644 index 000000000000000..8428770d51d734b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-01-10-17-39-09.gh-issue-143663.9LEFAf.rst @@ -0,0 +1,2 @@ +:meth:`tarfile.TarFile.extractall` now uses OS-specific separators to call +lstat. From ec912e7c9d388b212cc6df01475a88124bbcd65e Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 19 Nov 2026 11:47:55 -0800 Subject: [PATCH 210/229] gh-162031: Return None from readinto() on non-blocking streams with no data (#162032) Match PEP-3116 specification for Non-blocking I/O. First proposed in 2015 as `nonblock-none.patch` on gh-57531. Co-authored-by: Claude Opus 5.5 --- Lib/_pyio.py | 12 +++++++++++- Lib/test/test_io/test_bufferedio.py | 10 ++++++++-- Lib/test/test_io/test_general.py | 9 ++++++--- .../2026-11-15-03-10-31.gh-issue-162031.2k-3CG.rst | 3 +++ Modules/_io/bufferedio.c | 5 +++++ 5 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-15-03-10-31.gh-issue-162031.2k-3CG.rst diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 21f40add53593b4..ec1b20d04267688 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -733,6 +733,8 @@ def _readinto(self, b, read1): data = self.read1(len(b)) else: data = self.read(len(b)) + if data is None: + return None n = len(data) b[:n] = data @@ -1222,14 +1224,22 @@ def _readinto(self, buf, read1): if len(buf) - written > self.buffer_size: n = self.raw.readinto(buf[written:]) if not n: + if n is None and not written: + return None # would block break # eof written += n # Otherwise refill internal buffer - unless we're # in read1 mode and already got some data elif not (read1 and written): - if not self._peek_unlocked(1): + # The internal buffer is empty here. + chunk = self.raw.read(self.buffer_size) + if not chunk: + if chunk is None and not written: + return None # would block break # eof + self._read_buf = chunk + self._read_pos = 0 # In readinto1 mode, return as soon as we have some data if read1 and written: diff --git a/Lib/test/test_io/test_bufferedio.py b/Lib/test/test_io/test_bufferedio.py index ce594762c325ac0..64789821afe59c2 100644 --- a/Lib/test/test_io/test_bufferedio.py +++ b/Lib/test/test_io/test_bufferedio.py @@ -306,12 +306,18 @@ def test_readinto(self): self.assertEqual(b, b"gf") self.assertEqual(bufio.readinto(b), 0) self.assertEqual(b, b"gf") - rawio = self.MockRawIO((b"abc", None)) - bufio = self.tp(rawio) + rawio = self.MockRawIO((b"abc", None, None, None, None, None)) + bufio = self.tp(rawio, buffer_size=4) self.assertEqual(bufio.readinto(b), 2) self.assertEqual(b, b"ab") self.assertEqual(bufio.readinto(b), 1) self.assertEqual(b, b"cb") + self.assertIsNone(bufio.readinto(b)) + self.assertIsNone(bufio.readinto1(b)) + # Larger than buffer_size: read directly into b + b = bytearray(5) + self.assertIsNone(bufio.readinto(b)) + self.assertIsNone(bufio.readinto1(b)) def test_readinto1(self): buffer_size = 10 diff --git a/Lib/test/test_io/test_general.py b/Lib/test/test_io/test_general.py index c4d5a80086718b9..c6c138c2388d3e6 100644 --- a/Lib/test/test_io/test_general.py +++ b/Lib/test/test_io/test_general.py @@ -713,15 +713,18 @@ def test_invalid_newline(self): def test_buffered_readinto_mixin(self): # Test the implementation provided by BufferedIOBase class Stream(self.BufferedIOBase): + def __init__(self, data): + self.data = data def read(self, size): - return b"12345" + return self.data read1 = read - stream = Stream() for method in ("readinto", "readinto1"): with self.subTest(method): buffer = byteslike(5) - self.assertEqual(getattr(stream, method)(buffer), 5) + self.assertEqual(getattr(Stream(b"12345"), method)(buffer), 5) self.assertEqual(bytes(buffer), b"12345") + # Non-blocking stream with no data + self.assertIsNone(getattr(Stream(None), method)(buffer)) def test_fspath_support(self): def check_path_succeeds(path): diff --git a/Misc/NEWS.d/next/Library/2026-11-15-03-10-31.gh-issue-162031.2k-3CG.rst b/Misc/NEWS.d/next/Library/2026-11-15-03-10-31.gh-issue-162031.2k-3CG.rst new file mode 100644 index 000000000000000..01f1311c735d9b1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-15-03-10-31.gh-issue-162031.2k-3CG.rst @@ -0,0 +1,3 @@ +Fix :meth:`io.BufferedIOBase.readinto` and :meth:`~io.BufferedIOBase.readinto1` +raising :exc:`TypeError` instead of returning ``None`` when called on a +non-blocking stream with no data available. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 757e6a10558e485..237b6ea53c367a5 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -60,6 +60,11 @@ _bufferediobase_readinto_generic(PyObject *self, Py_buffer *buffer, char readint if (data == NULL) return NULL; + /* Non-blocking stream with no data available */ + if (data == Py_None) { + return data; + } + if (!PyBytes_Check(data)) { Py_DECREF(data); PyErr_SetString(PyExc_TypeError, "read() should return bytes"); From f15999da9218942ac99211b1b68c3743d0813db6 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 19 Nov 2026 21:21:51 +0100 Subject: [PATCH 211/229] gh-161875: Push NULL and None in GET_AWAITABLE (#161876) * Push NULL and None in GET_AWAITABLE Every await site compiled to GET_AWAITABLE; PUSH_NULL; LOAD_CONST None; SEND. GET_AWAITABLE is always followed by that pair, so let it push (iter, NULL, None) itself. `await f()` now compiles to CALL; GET_AWAITABLE; SEND, saving two dispatches per await. The tier-2 optimizer gets an override so the NULL and None slots are known constants. Co-Authored-By: Claude Opus 5.5 * Add NEWS entry Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- Doc/library/dis.rst | 7 ++++++- Include/internal/pycore_magic_number.h | 3 ++- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 2 +- Include/internal/pycore_uop_metadata.h | 6 +++--- Lib/test/test_dis.py | 8 +------- .../2026-11-12-14-37-21.gh-issue-161875.W-nGSw.rst | 3 +++ Modules/_testinternalcapi/test_cases.c.h | 8 +++++++- Python/bytecodes.c | 4 +++- Python/codegen.c | 12 ------------ Python/executor_cases.c.h | 12 ++++++++---- Python/generated_cases.c.h | 8 +++++++- Python/optimizer_bytecodes.c | 6 ++++++ Python/optimizer_cases.c.h | 9 +++++++++ 14 files changed, 57 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-14-37-21.gh-issue-161875.W-nGSw.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 16c2dab5b8adba1..4d1740fe468a0a8 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -817,7 +817,8 @@ not have to be) the original ``STACK[-2]``. .. opcode:: GET_AWAITABLE (where) - Implements ``STACK[-1] = get_awaitable(STACK[-1])``, where ``get_awaitable(o)`` + Replaces ``STACK[-1]`` with ``get_awaitable(STACK[-1])``, ``NULL`` and + ``None``, ready for :opcode:`SEND`. ``get_awaitable(o)`` returns ``o`` if ``o`` is a coroutine object or a generator object with the :data:`~inspect.CO_ITERABLE_COROUTINE` flag, or resolves ``o.__await__``. @@ -833,6 +834,10 @@ not have to be) the original ``STACK[-2]``. .. versionchanged:: 3.11 Previously, this instruction did not have an oparg. + .. versionchanged:: next + Also pushes ``NULL`` and ``None``, which were previously pushed by + separate instructions. + .. opcode:: GET_AITER diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 15d3ad6f3a336b7..8413528c0a26839 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -304,6 +304,7 @@ Known values: Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) Python 3.16a1 3706 (Add INTRINSIC_MATCH_CLASS_ISINSTANCE) + Python 3.16a1 3707 (GET_AWAITABLE also pushes NULL and None for SEND) Python 3.17 will start with 3750 @@ -313,7 +314,7 @@ Known values: */ -#define PYC_MAGIC_NUMBER 3706 +#define PYC_MAGIC_NUMBER 3707 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 04b826ed09144e7..ca1004816d8b00d 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -716,7 +716,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case GET_ANEXT: return 2; case GET_AWAITABLE: - return 1; + return 3; case GET_ITER: return 2; case GET_ITER_SELF: diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ba6e0252024a6df..8006830801916a9 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -690,7 +690,7 @@ enum { _FOR_ITER_VIRTUAL_TIER_TWO_r23, _GET_AITER_r11, _GET_ANEXT_r12, - _GET_AWAITABLE_r11, + _GET_AWAITABLE_r13, _GET_ITER_r12, _GET_ITER_TRAD_r12, _GET_LEN_r12, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 0cd82c53aa932ae..e746fd7c3b79b0b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -1639,7 +1639,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 1, _GET_AWAITABLE_r11 }, + { 3, 1, _GET_AWAITABLE_r13 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -4284,7 +4284,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_RETURN_VALUE_r11] = _RETURN_VALUE, [_GET_AITER_r11] = _GET_AITER, [_GET_ANEXT_r12] = _GET_ANEXT, - [_GET_AWAITABLE_r11] = _GET_AWAITABLE, + [_GET_AWAITABLE_r13] = _GET_AWAITABLE, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, @@ -5238,7 +5238,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GET_ANEXT] = "_GET_ANEXT", [_GET_ANEXT_r12] = "_GET_ANEXT_r12", [_GET_AWAITABLE] = "_GET_AWAITABLE", - [_GET_AWAITABLE_r11] = "_GET_AWAITABLE_r11", + [_GET_AWAITABLE_r13] = "_GET_AWAITABLE_r13", [_GET_ITER] = "_GET_ITER", [_GET_ITER_r12] = "_GET_ITER_r12", [_GET_ITER_TRAD] = "_GET_ITER_TRAD", diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 709fa374704f1f5..2c3972c7394d01b 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -615,8 +615,6 @@ async def _asyncwith(c): LOAD_SPECIAL 2 (__aenter__) CALL 0 GET_AWAITABLE 1 - PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) L2: SEND 4 (to L5) L3: YIELD_VALUE 1 L4: RESUME 3 @@ -632,8 +630,6 @@ async def _asyncwith(c): LOAD_COMMON_CONSTANT 7 (None) CALL 3 GET_AWAITABLE 2 - PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) L8: SEND 4 (to L11) L9: YIELD_VALUE 1 L10: RESUME 3 @@ -647,14 +643,12 @@ async def _asyncwith(c): RETURN_VALUE %4d L12: CLEANUP_THROW - L13: JUMP_BACKWARD_NO_INTERRUPT 28 (to L5) + L13: JUMP_BACKWARD_NO_INTERRUPT 26 (to L5) L14: CLEANUP_THROW L15: JUMP_BACKWARD_NO_INTERRUPT 10 (to L11) L16: PUSH_EXC_INFO WITH_EXCEPT_START GET_AWAITABLE 2 - PUSH_NULL - LOAD_COMMON_CONSTANT 7 (None) L17: SEND 5 (to L21) L18: YIELD_VALUE 1 L19: RESUME 3 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-14-37-21.gh-issue-161875.W-nGSw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-14-37-21.gh-issue-161875.W-nGSw.rst new file mode 100644 index 000000000000000..693f39a2bef9177 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-14-37-21.gh-issue-161875.W-nGSw.rst @@ -0,0 +1,3 @@ +The :opcode:`GET_AWAITABLE` instruction now also pushes the ``NULL`` and +``None`` that start the :opcode:`SEND` loop, so ``await`` no longer needs +separate :opcode:`PUSH_NULL` and ``LOAD_CONST None`` instructions. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index cfa8faa15f4faea..457592374d73d1c 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -6824,6 +6824,8 @@ INSTRUCTION_STATS(GET_AWAITABLE); _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; + _PyStackRef none; iterable = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6839,8 +6841,12 @@ JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; stack_pointer[0] = iter; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer[2] = none; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bd5debcd01e9bfd..c914da85cc65777 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1649,11 +1649,13 @@ dummy_func( awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); } - inst(GET_AWAITABLE, (iterable -- iter)) { + inst(GET_AWAITABLE, (iterable -- iter, null, none)) { PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); PyStackRef_CLOSE(iterable); ERROR_IF(iter_o == NULL); iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; } family(SEND, INLINE_CACHE_ENTRIES_SEND) = { diff --git a/Python/codegen.c b/Python/codegen.c index a975705576a167c..ac5e7904c053a00 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -618,8 +618,6 @@ codegen_unwind_fblock(compiler *c, location *ploc, RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc)); if (info->fb_type == COMPILE_FBLOCK_ASYNC_WITH) { ADDOP_I(c, *ploc, GET_AWAITABLE, 2); - ADDOP(c, *ploc, PUSH_NULL); - ADDOP_LOAD_CONST(c, *ploc, Py_None); ADD_YIELD_FROM(c, *ploc, 1); } ADDOP(c, *ploc, POP_TOP); @@ -5167,8 +5165,6 @@ codegen_comprehension(compiler *c, expr_ty e, int type, if (is_async_comprehension && type != COMP_GENEXP) { ADDOP_I(c, loc, GET_AWAITABLE, 0); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); } @@ -5312,8 +5308,6 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP_I(c, loc, LOAD_SPECIAL, SPECIAL___AENTER__); ADDOP_I(c, loc, CALL, 0); ADDOP_I(c, loc, GET_AWAITABLE, 1); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); ADDOP_JUMP(c, loc, SETUP_WITH, final); @@ -5349,8 +5343,6 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) */ RETURN_IF_ERROR(codegen_call_exit_with_nones(c, loc)); ADDOP_I(c, loc, GET_AWAITABLE, 2); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); ADDOP(c, loc, POP_TOP); @@ -5364,8 +5356,6 @@ codegen_async_with_inner(compiler *c, stmt_ty s, int pos) ADDOP(c, loc, PUSH_EXC_INFO); ADDOP(c, loc, WITH_EXCEPT_START); ADDOP_I(c, loc, GET_AWAITABLE, 2); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); RETURN_IF_ERROR(codegen_with_except_finish(c, cleanup)); @@ -5552,8 +5542,6 @@ codegen_visit_expr_impl(compiler *c, expr_ty e, bool result_is_unused) case Await_kind: VISIT(c, expr, e->v.Await.value); ADDOP_I(c, loc, GET_AWAITABLE, 0); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); ADD_YIELD_FROM(c, loc, 1); break; case Compare_kind: diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 58473deddcebe3c..02689598a649e8b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8906,11 +8906,13 @@ break; } - case _GET_AWAITABLE_r11: { + case _GET_AWAITABLE_r13: { CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; + _PyStackRef none; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); iterable = _stack_item_0; @@ -8932,10 +8934,12 @@ JUMP_TO_ERROR(); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; + _tos_cache2 = none; + _tos_cache1 = null; _tos_cache0 = iter; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e5b3548b66412cd..05a68ead027b743 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6824,6 +6824,8 @@ INSTRUCTION_STATS(GET_AWAITABLE); _PyStackRef iterable; _PyStackRef iter; + _PyStackRef null; + _PyStackRef none; iterable = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6839,8 +6841,12 @@ JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); + null = PyStackRef_NULL; + none = PyStackRef_None; stack_pointer[0] = iter; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer[2] = none; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6284885c428832b..b1be62bbe0613e5 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -2772,6 +2772,12 @@ dummy_func(void) { } } + op(_GET_AWAITABLE, (iterable -- iter, null, none)) { + iter = sym_new_not_null(ctx); + null = PyJitRef_Borrow(sym_new_null(ctx)); + none = PyJitRef_Borrow(sym_new_const(ctx, Py_None)); + } + // END BYTECODES // } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9cd20c495db4418..6c5fb641c564e9f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1888,8 +1888,17 @@ case _GET_AWAITABLE: { JitOptRef iter; + JitOptRef null; + JitOptRef none; iter = sym_new_not_null(ctx); + null = PyJitRef_Borrow(sym_new_null(ctx)); + none = PyJitRef_Borrow(sym_new_const(ctx, Py_None)); + CHECK_STACK_BOUNDS(2); stack_pointer[-1] = iter; + stack_pointer[0] = null; + stack_pointer[1] = none; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } From d4561588baf89e90cf44663626e91433108da377 Mon Sep 17 00:00:00 2001 From: Simon Nelse Pallan <59430808+simonnelse@users.noreply.github.com> Date: Fri, 20 Nov 2026 02:53:36 +0530 Subject: [PATCH 212/229] Docs: fix typo in `TracebackException.__notes__` description (#162290) Co-authored-by: Stan Ulbrych --- Doc/library/traceback.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 9e1be62c568ea14..8baf4b7d34905af 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -333,8 +333,8 @@ the module-level functions described above. The :attr:`~BaseException.__notes__` value from the original exception, or ``None`` - if the exception does not have any notes. If it is not ``None`` - is it formatted in the traceback after the exception string. + if the exception does not have any notes. If it is not ``None``, + it is formatted in the traceback after the exception string. .. versionadded:: 3.11 From 865d8d8f890352f5bb0ff58b7707179c8b046f8e Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 19 Nov 2026 22:26:50 +0100 Subject: [PATCH 213/229] gh-161875: Push NULL and None in GET_ANEXT (#161877) Every async for iteration compiled to GET_ANEXT; PUSH_NULL; LOAD_CONST None; SEND. GET_ANEXT is always followed by that pair, so let it push (aiter, awaitable, NULL, None) itself, saving two dispatches per iteration. Co-authored-by: Claude Opus 5.5 --- Doc/library/dis.rst | 7 ++++++- Include/internal/pycore_magic_number.h | 3 ++- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 2 +- Include/internal/pycore_uop_metadata.h | 6 +++--- ...026-11-12-14-46-10.gh-issue-161875.oOjR9w.rst | 4 ++++ Modules/_testinternalcapi/test_cases.c.h | 8 +++++++- Python/bytecodes.c | 4 +++- Python/codegen.c | 4 ---- Python/executor_cases.c.h | 16 +++++++++------- Python/generated_cases.c.h | 8 +++++++- Python/optimizer_bytecodes.c | 4 +++- Python/optimizer_cases.c.h | 10 ++++++++-- 13 files changed, 54 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-14-46-10.gh-issue-161875.oOjR9w.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 4d1740fe468a0a8..1b5f0c92faf02c7 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -851,11 +851,16 @@ not have to be) the original ``STACK[-2]``. .. opcode:: GET_ANEXT - Implement ``STACK.append(get_awaitable(STACK[-1].__anext__()))`` to the stack. + Implement ``STACK.append(get_awaitable(STACK[-1].__anext__()))`` to the stack, + then push ``NULL`` and ``None``, ready for :opcode:`SEND`. See ``GET_AWAITABLE`` for details about ``get_awaitable``. .. versionadded:: 3.5 + .. versionchanged:: next + Also pushes ``NULL`` and ``None``, which were previously pushed by + separate instructions. + .. opcode:: END_ASYNC_FOR diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 8413528c0a26839..041256df56b4938 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -305,6 +305,7 @@ Known values: Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) Python 3.16a1 3706 (Add INTRINSIC_MATCH_CLASS_ISINSTANCE) Python 3.16a1 3707 (GET_AWAITABLE also pushes NULL and None for SEND) + Python 3.16a1 3708 (GET_ANEXT also pushes NULL and None for SEND) Python 3.17 will start with 3750 @@ -314,7 +315,7 @@ Known values: */ -#define PYC_MAGIC_NUMBER 3707 +#define PYC_MAGIC_NUMBER 3708 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index ca1004816d8b00d..f1e6f1beae644a5 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -714,7 +714,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case GET_AITER: return 1; case GET_ANEXT: - return 2; + return 4; case GET_AWAITABLE: return 3; case GET_ITER: diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 8006830801916a9..31a8e60259f4baf 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -689,7 +689,7 @@ enum { _FOR_ITER_VIRTUAL_r23, _FOR_ITER_VIRTUAL_TIER_TWO_r23, _GET_AITER_r11, - _GET_ANEXT_r12, + _GET_ANEXT_r13, _GET_AWAITABLE_r13, _GET_ITER_r12, _GET_ITER_TRAD_r12, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index e746fd7c3b79b0b..a264ad3f7848b76 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -1630,7 +1630,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 2, 1, _GET_ANEXT_r12 }, + { 3, 1, _GET_ANEXT_r13 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -4283,7 +4283,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_MAKE_HEAP_SAFE_r33] = _MAKE_HEAP_SAFE, [_RETURN_VALUE_r11] = _RETURN_VALUE, [_GET_AITER_r11] = _GET_AITER, - [_GET_ANEXT_r12] = _GET_ANEXT, + [_GET_ANEXT_r13] = _GET_ANEXT, [_GET_AWAITABLE_r13] = _GET_AWAITABLE, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, @@ -5236,7 +5236,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GET_AITER] = "_GET_AITER", [_GET_AITER_r11] = "_GET_AITER_r11", [_GET_ANEXT] = "_GET_ANEXT", - [_GET_ANEXT_r12] = "_GET_ANEXT_r12", + [_GET_ANEXT_r13] = "_GET_ANEXT_r13", [_GET_AWAITABLE] = "_GET_AWAITABLE", [_GET_AWAITABLE_r13] = "_GET_AWAITABLE_r13", [_GET_ITER] = "_GET_ITER", diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-14-46-10.gh-issue-161875.oOjR9w.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-14-46-10.gh-issue-161875.oOjR9w.rst new file mode 100644 index 000000000000000..9afa682ea8b6ffb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-12-14-46-10.gh-issue-161875.oOjR9w.rst @@ -0,0 +1,4 @@ +The :opcode:`GET_ANEXT` instruction now also pushes the ``NULL`` and +``None`` that start the :opcode:`SEND` loop, so each :keyword:`async for` +iteration no longer needs separate :opcode:`PUSH_NULL` and ``LOAD_CONST None`` +instructions. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 457592374d73d1c..8bb3f857437792a 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -6799,6 +6799,8 @@ INSTRUCTION_STATS(GET_ANEXT); _PyStackRef aiter; _PyStackRef awaitable; + _PyStackRef null; + _PyStackRef none; aiter = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6808,8 +6810,12 @@ JUMP_TO_LABEL(error); } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + null = PyStackRef_NULL; + none = PyStackRef_None; stack_pointer[0] = awaitable; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer[2] = none; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c914da85cc65777..c6f1f61f3e08fe1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1641,12 +1641,14 @@ dummy_func( iter = PyStackRef_FromPyObjectSteal(iter_o); } - inst(GET_ANEXT, (aiter -- aiter, awaitable)) { + inst(GET_ANEXT, (aiter -- aiter, awaitable, null, none)) { PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); if (awaitable_o == NULL) { ERROR_NO_POP(); } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + null = PyStackRef_NULL; + none = PyStackRef_None; } inst(GET_AWAITABLE, (iterable -- iter, null, none)) { diff --git a/Python/codegen.c b/Python/codegen.c index ac5e7904c053a00..ae60caf195f23ca 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -2223,8 +2223,6 @@ codegen_async_for(compiler *c, stmt_ty s) /* SETUP_FINALLY to guard the __anext__ call */ ADDOP_JUMP(c, loc, SETUP_FINALLY, except); ADDOP(c, loc, GET_ANEXT); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); USE_LABEL(c, send); ADD_YIELD_FROM(c, loc, 1); ADDOP(c, loc, POP_BLOCK); /* for SETUP_FINALLY */ @@ -4841,8 +4839,6 @@ codegen_async_comprehension_generator(compiler *c, location loc, ADDOP_JUMP(c, loc, SETUP_FINALLY, except); ADDOP(c, loc, GET_ANEXT); - ADDOP(c, loc, PUSH_NULL); - ADDOP_LOAD_CONST(c, loc, Py_None); USE_LABEL(c, send); ADD_YIELD_FROM(c, loc, 1); ADDOP(c, loc, POP_BLOCK); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 02689598a649e8b..5df8120e7424b93 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8877,11 +8877,13 @@ break; } - case _GET_ANEXT_r12: { + case _GET_ANEXT_r13: { CHECK_CURRENT_CACHED_VALUES(1); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef aiter; _PyStackRef awaitable; + _PyStackRef null; + _PyStackRef none; _PyStackRef _stack_item_0 = _tos_cache0; aiter = _stack_item_0; stack_pointer[0] = aiter; @@ -8896,12 +8898,12 @@ JUMP_TO_ERROR(); } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - _tos_cache1 = awaitable; - _tos_cache0 = aiter; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + null = PyStackRef_NULL; + none = PyStackRef_None; + _tos_cache2 = none; + _tos_cache1 = null; + _tos_cache0 = awaitable; + SET_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 05a68ead027b743..3af7863e80d2c6a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6799,6 +6799,8 @@ INSTRUCTION_STATS(GET_ANEXT); _PyStackRef aiter; _PyStackRef awaitable; + _PyStackRef null; + _PyStackRef none; aiter = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); _PyFrame_StackPointerValidate(frame); @@ -6808,8 +6810,12 @@ JUMP_TO_LABEL(error); } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + null = PyStackRef_NULL; + none = PyStackRef_None; stack_pointer[0] = awaitable; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer[2] = none; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index b1be62bbe0613e5..6648b72e66e281d 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -2763,13 +2763,15 @@ dummy_func(void) { } } - op(_GET_ANEXT, (aiter -- aiter, awaitable)) { + op(_GET_ANEXT, (aiter -- aiter, awaitable, null, none)) { if (sym_matches_type(aiter, &PyAsyncGen_Type)) { awaitable = sym_new_type(ctx, &_PyAsyncGenASend_Type); } else { awaitable = sym_new_not_null(ctx); } + null = PyJitRef_Borrow(sym_new_null(ctx)); + none = PyJitRef_Borrow(sym_new_const(ctx, Py_None)); } op(_GET_AWAITABLE, (iterable -- iter, null, none)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 6c5fb641c564e9f..c86121fcfc0e79a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1872,6 +1872,8 @@ case _GET_ANEXT: { JitOptRef aiter; JitOptRef awaitable; + JitOptRef null; + JitOptRef none; aiter = stack_pointer[-1]; if (sym_matches_type(aiter, &PyAsyncGen_Type)) { awaitable = sym_new_type(ctx, &_PyAsyncGenASend_Type); @@ -1879,9 +1881,13 @@ else { awaitable = sym_new_not_null(ctx); } - CHECK_STACK_BOUNDS(1); + null = PyJitRef_Borrow(sym_new_null(ctx)); + none = PyJitRef_Borrow(sym_new_const(ctx, Py_None)); + CHECK_STACK_BOUNDS(3); stack_pointer[0] = awaitable; - stack_pointer += 1; + stack_pointer[1] = null; + stack_pointer[2] = none; + stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } From 6957e70879d1c8e4ee8fdcec11f8cb9d7f83c0fc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Nov 2026 00:18:24 +0100 Subject: [PATCH 214/229] gh-129813: Use PyBytesWriter in _testbuffer (#162297) Avoid a temporary PyMem_Malloc() buffer: create directly a bytes object using PyBytesWriter API. --- Modules/_testbuffer.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 8b6b617aafa427d..8b37dc7bb34bed9 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -2171,7 +2171,6 @@ ndarray_tobytes(PyObject *self, PyObject *dummy) Py_buffer *src = &ndbuf->base; Py_buffer dest; PyObject *ret = NULL; - char *mem; if (ND_C_CONTIGUOUS(ndbuf->flags)) return PyBytes_FromStringAndSize(src->buf, src->len); @@ -2180,14 +2179,13 @@ ndarray_tobytes(PyObject *self, PyObject *dummy) assert(src->strides != NULL); assert(src->ndim > 0); - mem = PyMem_Malloc(src->len); - if (mem == NULL) { - PyErr_NoMemory(); + PyBytesWriter *writer = PyBytesWriter_Create(src->len); + if (writer == NULL) { return NULL; } dest = *src; - dest.buf = mem; + dest.buf = PyBytesWriter_GetData(writer); dest.suboffsets = NULL; dest.strides = strides_from_shape(ndbuf, 0); if (dest.strides == NULL) @@ -2195,11 +2193,12 @@ ndarray_tobytes(PyObject *self, PyObject *dummy) if (copy_buffer(&dest, src) < 0) goto out; - ret = PyBytes_FromStringAndSize(mem, src->len); + ret = PyBytesWriter_Finish(writer); + writer = NULL; out: PyMem_XFree(dest.strides); - PyMem_Free(mem); + PyBytesWriter_Discard(writer); return ret; } @@ -2445,7 +2444,7 @@ py_buffer_to_contiguous(PyObject *self, PyObject *args) int flags; char ord; Py_buffer view; - char *buf = NULL; + PyBytesWriter *writer = NULL; if (!PyArg_ParseTuple(args, "OOi", &obj, &order, &flags)) { return NULL; @@ -2460,21 +2459,21 @@ py_buffer_to_contiguous(PyObject *self, PyObject *args) goto out; } - buf = PyMem_Malloc(view.len); - if (buf == NULL) { - PyErr_NoMemory(); + writer = PyBytesWriter_Create(view.len); + if (writer == NULL) { goto out; } - if (PyBuffer_ToContiguous(buf, &view, view.len, ord) < 0) { + if (PyBuffer_ToContiguous(PyBytesWriter_GetData(writer), &view, view.len, ord) < 0) { goto out; } - ret = PyBytes_FromStringAndSize(buf, view.len); + ret = PyBytesWriter_Finish(writer); + writer = NULL; out: PyBuffer_Release(&view); - PyMem_XFree(buf); + PyBytesWriter_Discard(writer); return ret; } From f84083b08ab40b08c8d608a320008418b331932e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Nov 2026 00:42:03 +0100 Subject: [PATCH 215/229] Fix regrtest rerun: reset worker_bug (#162293) When a test fails with "worker bug" and tests are rerun in verbose mode, reset worker_bug so that tests are marked as success if the failed test doesn't fail again when rerun in verbose mode. TestResult.write_json_into(file) now calls also file.flush(). --- Lib/test/libregrtest/result.py | 1 + Lib/test/libregrtest/results.py | 1 + 2 files changed, 2 insertions(+) diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index f8aa256be1ecfe8..ba4ba4d13ea1d7d 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -226,6 +226,7 @@ def get_rerun_match_tests(self) -> FilterTuple | None: def write_json_into(self, file) -> None: json.dump(self, file, cls=_EncodeTestResult) + file.flush() @staticmethod def from_json(worker_json: StrJSON) -> 'TestResult': diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index eac8f4c40ebca17..c57b82e05a5b21d 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -168,6 +168,7 @@ def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: self.bad.clear() self.env_changed.clear() self.rerun_results.clear() + self.worker_bug = False return (tuple(tests), match_tests_dict) From b09a7e92fb0ab958a52ad94e376402cdd3da09b2 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Thu, 19 Nov 2026 16:02:40 -0800 Subject: [PATCH 216/229] gh-136267: Read the target version from patchlevel.h in build-details.json (#161532) With --with-build-python, the build Python can differ from the target in patch version or release serial, but version_info and hexversion came from the running interpreter. Read them from Include/patchlevel.h instead, falling back to sys.version_info when the header is not there. Co-authored-by: Claude Fable 5.1 --- Lib/test/test_build_details.py | 41 +++++++++++++++- ...-11-06-12-00-02.gh-issue-136267.Vt4pLm.rst | 4 ++ Tools/build/generate-build-details.py | 48 ++++++++++++++++++- 3 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-11-06-12-00-02.gh-issue-136267.Vt4pLm.rst diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index 552c7252c32d049..40bf1bef11a8885 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -5,11 +5,12 @@ import sys import sysconfig import string +import textwrap import unittest from pathlib import Path from unittest import mock -from test.support import is_android, is_apple_mobile, is_wasm32 +from test.support import is_android, is_apple_mobile, is_wasm32, os_helper BASE_PATH = Path( __file__, # Lib/test/test_build_details.py @@ -356,6 +357,44 @@ def test_no_dynamic_loading(self): data['implementation']['supports_isolated_interpreters']) +@unittest.skipIf( + generate_build_details is None, + "Failed to import generate-build-details" +) +class BuildDetailsVersionTests(unittest.TestCase): + def test_native(self): + version_info = generate_build_details.get_version_info() + self.assertEqual(version_info, + generate_build_details.version_info_to_dict( + sys.version_info)) + self.assertEqual(generate_build_details.hexversion(version_info), + sys.hexversion) + + def test_patchlevel(self): + patchlevel_h = os.path.join(os_helper.TESTFN, 'patchlevel.h') + self.addCleanup(os_helper.rmtree, os_helper.TESTFN) + os.mkdir(os_helper.TESTFN) + with open(patchlevel_h, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + #define PY_RELEASE_LEVEL_GAMMA 0xC /* For release candidates */ + #define PY_MAJOR_VERSION 3 + #define PY_MINOR_VERSION 99 + #define PY_MICRO_VERSION 7 + #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA + #define PY_RELEASE_SERIAL 2 + #define PY_VERSION "3.99.7rc2" + ''')) + with mock.patch.object(generate_build_details, 'PATCHLEVEL_H', + patchlevel_h): + version_info = generate_build_details.get_version_info() + self.assertEqual(version_info, { + 'major': 3, 'minor': 99, 'micro': 7, + 'releaselevel': 'candidate', 'serial': 2, + }) + self.assertEqual(generate_build_details.hexversion(version_info), + 0x036307c2) + + @unittest.skipIf( generate_build_details is None, "Failed to import generate-build-details" diff --git a/Misc/NEWS.d/next/Build/2026-11-06-12-00-02.gh-issue-136267.Vt4pLm.rst b/Misc/NEWS.d/next/Build/2026-11-06-12-00-02.gh-issue-136267.Vt4pLm.rst new file mode 100644 index 000000000000000..d99ae1c42133402 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-11-06-12-00-02.gh-issue-136267.Vt4pLm.rst @@ -0,0 +1,4 @@ +``build-details.json`` now takes ``language.version_info``, +``implementation.version`` and ``implementation.hexversion`` from +:file:`Include/patchlevel.h` instead of the interpreter running the generator, +which can be a different patch release under :option:`--with-build-python`. diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index f6cb93beabec43d..a6f537fc0d020da 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -10,6 +10,7 @@ import importlib.machinery import json import os +import re import sys import sysconfig @@ -18,11 +19,52 @@ from typing import Any +PATCHLEVEL_H = os.path.join( + os.path.dirname(__file__), '..', '..', 'Include', 'patchlevel.h', +) +RELEASE_LEVELS = {0xA: 'alpha', 0xB: 'beta', 0xC: 'candidate', 0xF: 'final'} + + def version_info_to_dict(obj: sys._version_info) -> dict[str, Any]: field_names = ('major', 'minor', 'micro', 'releaselevel', 'serial') return {field: getattr(obj, field) for field in field_names} +def get_version_info() -> dict[str, Any]: + """Return the target's version as a version_info dict. + + sys.version_info describes the running interpreter, which under + --with-build-python can differ from the target in micro version and + release level. Read Include/patchlevel.h instead when it is available. + """ + try: + with open(PATCHLEVEL_H, encoding='utf-8') as f: + patchlevel = f.read() + except FileNotFoundError: + return version_info_to_dict(sys.version_info) + defines = dict(re.findall(r'^#define\s+(PY_\w+)\s+(\S+)', patchlevel, re.M)) + # PY_RELEASE_LEVEL is defined as another macro, e.g. PY_RELEASE_LEVEL_ALPHA. + level = int(defines[defines['PY_RELEASE_LEVEL']], 16) + return { + 'major': int(defines['PY_MAJOR_VERSION']), + 'minor': int(defines['PY_MINOR_VERSION']), + 'micro': int(defines['PY_MICRO_VERSION']), + 'releaselevel': RELEASE_LEVELS[level], + 'serial': int(defines['PY_RELEASE_SERIAL']), + } + + +def hexversion(version_info: dict[str, Any]) -> int: + level = {name: value for value, name in RELEASE_LEVELS.items()} + return int( + version_info['major'] << 24 + | version_info['minor'] << 16 + | version_info['micro'] << 8 + | level[version_info['releaselevel']] << 4 + | version_info['serial'] + ) + + def get_dict_key(container: dict[str, Any], key: str) -> dict[str, Any]: for part in key.split('.'): container = container[part] @@ -94,11 +136,13 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: ) data['platform'] = sysconfig.get_platform() + version_info = get_version_info() data['language']['version'] = sysconfig.get_python_version() - data['language']['version_info'] = version_info_to_dict(sys.version_info) + data['language']['version_info'] = version_info data['implementation'] = vars(sys.implementation).copy() - data['implementation']['version'] = version_info_to_dict(sys.implementation.version) + data['implementation']['version'] = version_info + data['implementation']['hexversion'] = hexversion(version_info) # Fix cross-compilation if '_multiarch' in data['implementation']: data['implementation']['_multiarch'] = sysconfig.get_config_var('MULTIARCH') From c0f3fcf33681b3f5c5d50e7a301abf1eca257a4c Mon Sep 17 00:00:00 2001 From: Thales Moura Date: Fri, 20 Nov 2026 06:48:05 -0300 Subject: [PATCH 217/229] gh-160978: Return the same ctypes array type when created concurrently (#161494) --- Lib/test/test_ctypes/test_arrays.py | 49 +++++++++- ...-11-06-12-00-00.gh-issue-160978.Kx7mQa.rst | 3 + Modules/_ctypes/_ctypes.c | 97 +++++++++++++++---- 3 files changed, 131 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-06-12-00-00.gh-issue-160978.Kx7mQa.rst diff --git a/Lib/test/test_ctypes/test_arrays.py b/Lib/test/test_ctypes/test_arrays.py index 7f1f6cf58402c9f..5b7708afdaa1249 100644 --- a/Lib/test/test_ctypes/test_arrays.py +++ b/Lib/test/test_ctypes/test_arrays.py @@ -1,11 +1,14 @@ import ctypes import sys +import threading import unittest +from concurrent.futures import ThreadPoolExecutor from ctypes import (Structure, Array, ARRAY, sizeof, addressof, create_string_buffer, create_unicode_buffer, c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, c_long, c_ulonglong, c_float, c_double, c_longdouble) -from test.support import bigmemtest, _2G, threading_helper, Py_GIL_DISABLED +from test.support import (bigmemtest, _2G, threading_helper, + Py_GIL_DISABLED, SHORT_TIMEOUT) from ._support import (_CData, PyCArrayType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -288,5 +291,49 @@ def run(): raise cm.exc_value +class ArrayTypeCacheTest(unittest.TestCase): + # gh-160978: the array type cache could hand out distinct type objects for + # the same (item type, length) pair when the array type was created from + # more than one thread at a time. + + @threading_helper.requires_working_threading() + def test_concurrent_creation_with_set_name(self): + # Creating the array type runs Python code -- ``__set_name__`` here -- + # which lets a second thread reach the cache before the first one has + # stored its type. Without the fix this fails on the default build as + # well, so it guards the fix even where threads cannot interleave + # freely. + barrier = threading.Barrier(2) + + class Meta(type(c_int)): + def __set_name__(cls, owner, name): + barrier.wait(timeout=SHORT_TIMEOUT) + + class Item(c_int, metaclass=Meta): + pass + + with ThreadPoolExecutor(2) as pool: + first, second = pool.map(lambda _: Item * 100, range(2)) + + self.assertIs(first, second) + # Raises TypeError if the two types are distinct. + first.from_param(second()) + + @threading_helper.requires_working_threading() + def test_concurrent_creation_returns_identical_types(self): + nthreads = 8 + barrier = threading.Barrier(nthreads) + + def make_type(length): + barrier.wait(timeout=SHORT_TIMEOUT) + return c_int * length + + with ThreadPoolExecutor(nthreads) as pool: + for length in range(1, 51): + types = list(pool.map(make_type, [length] * nthreads)) + barrier.reset() + self.assertEqual(len(set(types)), 1) + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-11-06-12-00-00.gh-issue-160978.Kx7mQa.rst b/Misc/NEWS.d/next/Library/2026-11-06-12-00-00.gh-issue-160978.Kx7mQa.rst new file mode 100644 index 000000000000000..be5cc945f4eb61d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-06-12-00-00.gh-issue-160978.Kx7mQa.rst @@ -0,0 +1,3 @@ +Fix :mod:`ctypes` handing out distinct array types for the same item type +and length when the array type is created concurrently from several +threads. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 81fb9c9f57b998b..14614dca1c02cee 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -186,7 +186,23 @@ _DictRemover_call(PyObject *myself, PyObject *args, PyObject *kw) { DictRemoverObject *self = _DictRemoverObject_CAST(myself); if (self->key && self->dict) { - if (-1 == PyDict_DelItem(self->dict, self->key)) { + /* Only remove the entry if it is still the proxy that died: the + key may have been reused for a new object in the meantime. */ + PyObject *proxy = PyTuple_GET_SIZE(args) == 1 + ? PyTuple_GET_ITEM(args, 0) : NULL; + PyObject *current; + int rc; + Py_BEGIN_CRITICAL_SECTION(self->dict); + rc = PyDict_GetItemRef(self->dict, self->key, ¤t); + if (rc == 1) { + rc = 0; + if (current == proxy) { + rc = PyDict_DelItem(self->dict, self->key); + } + Py_DECREF(current); + } + Py_END_CRITICAL_SECTION(); + if (rc < 0) { PyErr_FormatUnraisable("Exception ignored while " "calling _ctypes.DictRemover"); } @@ -215,32 +231,72 @@ static PyType_Spec dictremover_spec = { .slots = dictremover_slots, }; -int -PyDict_SetItemProxy(ctypes_state *st, PyObject *dict, PyObject *key, PyObject *item) +/* Cache `item` in `dict` under `key` through a weakref proxy, unless a live + entry already exists. On success *presult is a new reference to whatever + is cached for `key` afterwards: `item` itself, or the object another + thread cached first. + Returns 0 on success, -1 with an exception set on failure. */ +static int +_PyDict_SetDefaultItemProxy(ctypes_state *st, PyObject *dict, PyObject *key, + PyObject *item, PyObject **presult) { - PyObject *obj; - DictRemoverObject *remover; - PyObject *proxy; - int result; - - obj = _PyObject_CallNoArgs((PyObject *)st->DictRemover_Type); - if (obj == NULL) + PyObject *obj = _PyObject_CallNoArgs((PyObject *)st->DictRemover_Type); + if (obj == NULL) { return -1; + } - remover = (DictRemoverObject *)obj; + DictRemoverObject *remover = (DictRemoverObject *)obj; assert(remover->key == NULL); assert(remover->dict == NULL); remover->key = Py_NewRef(key); remover->dict = Py_NewRef(dict); - proxy = PyWeakref_NewProxy(item, obj); + PyObject *proxy = PyWeakref_NewProxy(item, obj); Py_DECREF(obj); - if (proxy == NULL) + if (proxy == NULL) { + return -1; + } + + PyObject *stored; + int present = PyDict_SetDefaultRef(dict, key, proxy, &stored); + if (present < 0) { + Py_DECREF(proxy); return -1; + } + if (present == 0) { + /* Our proxy was inserted; `item` is the canonical object. */ + Py_DECREF(stored); + Py_DECREF(proxy); + *presult = Py_NewRef(item); + return 0; + } - result = PyDict_SetItem(dict, key, proxy); + /* Another thread cached an object for this key first. Prefer it, so that + every caller gets the same object. */ + PyObject *existing = NULL; + int rc = 0; + if (PyWeakref_CheckProxy(stored)) { + rc = PyWeakref_GetRef(stored, &existing); + } + Py_DECREF(stored); + if (rc < 0) { + Py_DECREF(proxy); + return -1; + } + if (existing != NULL) { + Py_DECREF(proxy); + *presult = existing; + return 0; + } + + /* The cached entry had already been cleared; install ours instead. */ + rc = PyDict_SetItem(dict, key, proxy); Py_DECREF(proxy); - return result; + if (rc < 0) { + return -1; + } + *presult = Py_NewRef(item); + return 0; } static int @@ -5434,13 +5490,20 @@ PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length) Py_DECREF(key); return NULL; } - if (PyDict_SetItemProxy(st, st->array_cache, key, result) < 0) { + PyObject *cached; + int rc; + Py_BEGIN_CRITICAL_SECTION(st->array_cache); + rc = _PyDict_SetDefaultItemProxy(st, st->array_cache, key, result, + &cached); + Py_END_CRITICAL_SECTION(); + if (rc < 0) { Py_DECREF(key); Py_DECREF(result); return NULL; } Py_DECREF(key); - return result; + Py_DECREF(result); + return cached; } From 6f3406076c589a1da85e675b1b9d2e8f142dec43 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 20 Nov 2026 11:51:38 +0100 Subject: [PATCH 218/229] gh-161184: Use a managed dict and weakref list for the io types (#161236) --- Lib/test/test_io/test_general.py | 35 ++++++++++++++ Lib/test/test_io/test_memoryio.py | 7 ++- ...-11-03-08-21-06.gh-issue-161184.PIhcD9.rst | 6 +++ Modules/_io/bufferedio.c | 33 +++---------- Modules/_io/bytesio.c | 46 ++++++++----------- Modules/_io/fileio.c | 14 ++---- Modules/_io/iobase.c | 27 ++++------- Modules/_io/stringio.c | 43 ++++++++--------- Modules/_io/textio.c | 12 ++--- Modules/_io/winconsoleio.c | 16 ++----- 10 files changed, 111 insertions(+), 128 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-03-08-21-06.gh-issue-161184.PIhcD9.rst diff --git a/Lib/test/test_io/test_general.py b/Lib/test/test_io/test_general.py index c6c138c2388d3e6..883144817dddc33 100644 --- a/Lib/test/test_io/test_general.py +++ b/Lib/test/test_io/test_general.py @@ -898,6 +898,41 @@ def test_immutable_types(self): with self.assertRaisesRegex(TypeError, "immutable"): tp.foo = "bar" + def test_managed_dict(self): + # The _io types use Py_TPFLAGS_MANAGED_DICT: there is no explicit + # dict slot and instance attributes are stored as inline values. + _testinternalcapi = import_helper.import_module("_testinternalcapi") + _io = self._io + no_dict = {_io.IncrementalNewlineDecoder, _io._BytesIOBuffer} + for tp in self.types: + with self.subTest(tp=tp): + if tp in no_dict: + self.assertEqual(tp.__dictoffset__, 0) + else: + self.assertEqual(tp.__dictoffset__, -1) + + # Use fresh subclasses: each class has its own shared keys, which + # may already be exhausted on the base types by earlier tests. + class BytesIO(_io.BytesIO): pass + class StringIO(_io.StringIO): pass + class BufferedReader(_io.BufferedReader): pass + class TextIOWrapper(_io.TextIOWrapper): pass + class RawIO(_io._RawIOBase): + def __init__(self): + self.attr = 1 + instances = [ + BytesIO(b"data"), + StringIO("data"), + BufferedReader(_io.BytesIO(b"data")), + TextIOWrapper(_io.BytesIO()), + RawIO(), + ] + for obj in instances: + with self.subTest(obj=type(obj).__name__): + obj.attr = 1 + self.assertTrue(_testinternalcapi.has_inline_values(obj)) + self.assertEqual(obj.__dict__["attr"], 1) + def test_class_hierarchy(self): def check_subs(types, base): for tp in types: diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 89b0e4c9463ac0a..20c2777b6c3ce07 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -10,6 +10,7 @@ import io import _pyio as pyio import pickle +import struct import sys import weakref @@ -1037,11 +1038,13 @@ def test_setstate(self): @support.cpython_only def test_sizeof(self): if support.Py_GIL_DISABLED: - basesize = support.calcobjsize('P2n2Pni') + basesize = support.calcobjsize('P2nni') else: - basesize = support.calcobjsize('P2n2Pn') + basesize = support.calcobjsize('P2nn') check = self.check_sizeof self.assertEqual(object.__sizeof__(io.BytesIO()), basesize) + # sys.getsizeof() includes the managed dict and weakref pre-header + basesize += 2 * struct.calcsize('P') check(io.BytesIO(), basesize ) n = 1000 # use a variable to prevent constant folding check(io.BytesIO(b'a' * n), basesize + sys.getsizeof(b'a' * n)) diff --git a/Misc/NEWS.d/next/Library/2026-11-03-08-21-06.gh-issue-161184.PIhcD9.rst b/Misc/NEWS.d/next/Library/2026-11-03-08-21-06.gh-issue-161184.PIhcD9.rst new file mode 100644 index 000000000000000..3b15925773b99ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-03-08-21-06.gh-issue-161184.PIhcD9.rst @@ -0,0 +1,6 @@ +Speed up attribute access on instances of :mod:`io` classes and their +subclasses by storing instance attributes and weak references in the object +pre-header. The ``__dictoffset__`` and ``__weakrefoffset__`` attributes +of these classes now report the managed offsets, and C extension types that +subclass an :mod:`io` class can no longer declare their own ``__dictoffset__`` +or ``__weaklistoffset__`` member. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 237b6ea53c367a5..1e900084b549f4f 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -9,11 +9,11 @@ #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_fileutils.h" // _PyFile_Flush #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() -#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" @@ -265,8 +265,6 @@ typedef struct { Py_ssize_t buffer_size; Py_ssize_t buffer_mask; - PyObject *dict; - PyObject *weakreflist; } buffered; #define buffered_CAST(op) ((buffered *)(op)) @@ -422,7 +420,7 @@ buffered_clear(PyObject *op) buffered *self = buffered_CAST(op); self->ok = 0; Py_CLEAR(self->raw); - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); return 0; } @@ -436,7 +434,7 @@ buffered_dealloc(PyObject *op) return; _PyObject_GC_UNTRACK(self); self->ok = 0; - FT_CLEAR_WEAKREFS(op, self->weakreflist); + PyObject_ClearWeakRefs(op); if (self->buffer) { PyMem_Free(self->buffer); self->buffer = NULL; @@ -472,8 +470,7 @@ buffered_traverse(PyObject *op, visitproc visit, void *arg) buffered *self = buffered_CAST(op); Py_VISIT(Py_TYPE(self)); Py_VISIT(self->raw); - Py_VISIT(self->dict); - return 0; + return PyObject_VisitManagedDict(op, visit, arg); } /* Because this can call arbitrary code, it shouldn't be called when @@ -2255,8 +2252,6 @@ typedef struct { PyObject_HEAD buffered *reader; buffered *writer; - PyObject *dict; - PyObject *weakreflist; } rwpair; #define rwpair_CAST(op) ((rwpair *)(op)) @@ -2314,10 +2309,9 @@ bufferedrwpair_traverse(PyObject *op, visitproc visit, void *arg) { rwpair *self = rwpair_CAST(op); Py_VISIT(Py_TYPE(self)); - Py_VISIT(self->dict); Py_VISIT(self->reader); Py_VISIT(self->writer); - return 0; + return PyObject_VisitManagedDict(op, visit, arg); } static int @@ -2326,7 +2320,7 @@ bufferedrwpair_clear(PyObject *op) rwpair *self = rwpair_CAST(op); Py_CLEAR(self->reader); Py_CLEAR(self->writer); - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); return 0; } @@ -2336,7 +2330,7 @@ bufferedrwpair_dealloc(PyObject *op) rwpair *self = rwpair_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - FT_CLEAR_WEAKREFS(op, self->weakreflist); + PyObject_ClearWeakRefs(op); (void)bufferedrwpair_clear(op); tp->tp_free(self); Py_DECREF(tp); @@ -2662,8 +2656,6 @@ static PyMethodDef bufferedreader_methods[] = { static PyMemberDef bufferedreader_members[] = { {"raw", _Py_T_OBJECT, offsetof(buffered, raw), Py_READONLY}, {"_finalizing", Py_T_BOOL, offsetof(buffered, finalizing), 0}, - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(buffered, weakreflist), Py_READONLY}, - {"__dictoffset__", Py_T_PYSSIZET, offsetof(buffered, dict), Py_READONLY}, {NULL} }; @@ -2721,8 +2713,6 @@ static PyMethodDef bufferedwriter_methods[] = { static PyMemberDef bufferedwriter_members[] = { {"raw", _Py_T_OBJECT, offsetof(buffered, raw), Py_READONLY}, {"_finalizing", Py_T_BOOL, offsetof(buffered, finalizing), 0}, - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(buffered, weakreflist), Py_READONLY}, - {"__dictoffset__", Py_T_PYSSIZET, offsetof(buffered, dict), Py_READONLY}, {NULL} }; @@ -2775,12 +2765,6 @@ static PyMethodDef bufferedrwpair_methods[] = { {NULL, NULL} }; -static PyMemberDef bufferedrwpair_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rwpair, weakreflist), Py_READONLY}, - {"__dictoffset__", Py_T_PYSSIZET, offsetof(rwpair, dict), Py_READONLY}, - {NULL} -}; - static PyGetSetDef bufferedrwpair_getset[] = { _IO_BUFFEREDRWPAIR_CLOSED_GETSETDEF {NULL} @@ -2792,7 +2776,6 @@ static PyType_Slot bufferedrwpair_slots[] = { {Py_tp_traverse, bufferedrwpair_traverse}, {Py_tp_clear, bufferedrwpair_clear}, {Py_tp_methods, bufferedrwpair_methods}, - {Py_tp_members, bufferedrwpair_members}, {Py_tp_getset, bufferedrwpair_getset}, {Py_tp_init, _io_BufferedRWPair___init__}, {0, NULL}, @@ -2839,8 +2822,6 @@ static PyMethodDef bufferedrandom_methods[] = { static PyMemberDef bufferedrandom_members[] = { {"raw", _Py_T_OBJECT, offsetof(buffered, raw), Py_READONLY}, {"_finalizing", Py_T_BOOL, offsetof(buffered, finalizing), 0}, - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(buffered, weakreflist), Py_READONLY}, - {"__dictoffset__", Py_T_PYSSIZET, offsetof(buffered, dict), Py_READONLY}, {NULL} }; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 9bec70acebb3598..fa828f218928990 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -5,7 +5,6 @@ #include "pycore_object.h" #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_sysmodule.h" // _PySys_GetSizeOf() -#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include // offsetof() #include "_iomodule.h" @@ -21,8 +20,6 @@ typedef struct { PyObject *buf; Py_ssize_t pos; Py_ssize_t string_size; - PyObject *dict; - PyObject *weakreflist; Py_ssize_t exports; #ifdef Py_GIL_DISABLED int buf_shared; @@ -974,11 +971,16 @@ _io_BytesIO___getstate___impl(bytesio *self) if (initvalue == NULL) return NULL; - if (self->dict == NULL) { + if (_PyObject_IsInstanceDictEmpty((PyObject *)self)) { dict = Py_NewRef(Py_None); } else { - dict = PyDict_Copy(self->dict); + dict = PyObject_GenericGetDict((PyObject *)self, NULL); + if (dict == NULL) { + Py_DECREF(initvalue); + return NULL; + } + Py_SETREF(dict, PyDict_Copy(dict)); if (dict == NULL) { Py_DECREF(initvalue); return NULL; @@ -1061,16 +1063,16 @@ _io_BytesIO___setstate___impl(bytesio *self, PyObject *state) Py_TYPE(dict)->tp_name); return NULL; } - if (self->dict) { - /* Alternatively, we could replace the internal dictionary - completely. However, it seems more practical to just update it. */ - if (PyDict_Update(self->dict, dict) < 0) - return NULL; + /* Alternatively, we could replace the internal dictionary + completely. However, it seems more practical to just update it. */ + PyObject *self_dict = PyObject_GenericGetDict((PyObject *)self, NULL); + if (self_dict == NULL) { + return NULL; } - else { - /* The LOAD_ATTR specializations read the dict slot lock-free - with an acquire load, so pair it with a release store. */ - FT_ATOMIC_STORE_PTR_RELEASE(self->dict, Py_NewRef(dict)); + int rc = PyDict_Update(self_dict, dict); + Py_DECREF(self_dict); + if (rc < 0) { + return NULL; } } @@ -1089,8 +1091,8 @@ bytesio_dealloc(PyObject *op) PyErr_Print(); } Py_CLEAR(self->buf); - Py_CLEAR(self->dict); - FT_CLEAR_WEAKREFS(op, self->weakreflist); + PyObject_ClearManagedDict(op); + PyObject_ClearWeakRefs(op); tp->tp_free(self); Py_DECREF(tp); } @@ -1187,16 +1189,15 @@ bytesio_traverse(PyObject *op, visitproc visit, void *arg) { bytesio *self = bytesio_CAST(op); Py_VISIT(Py_TYPE(self)); - Py_VISIT(self->dict); Py_VISIT(self->buf); - return 0; + return PyObject_VisitManagedDict(op, visit, arg); } static int bytesio_clear(PyObject *op) { bytesio *self = bytesio_CAST(op); - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) { Py_CLEAR(self->buf); } @@ -1239,12 +1240,6 @@ static struct PyMethodDef bytesio_methods[] = { {NULL, NULL} /* sentinel */ }; -static PyMemberDef bytesio_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(bytesio, weakreflist), Py_READONLY}, - {"__dictoffset__", Py_T_PYSSIZET, offsetof(bytesio, dict), Py_READONLY}, - {NULL} -}; - static PyType_Slot bytesio_slots[] = { {Py_tp_dealloc, bytesio_dealloc}, {Py_tp_doc, (void *)_io_BytesIO___init____doc__}, @@ -1253,7 +1248,6 @@ static PyType_Slot bytesio_slots[] = { {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, bytesio_iternext}, {Py_tp_methods, bytesio_methods}, - {Py_tp_members, bytesio_members}, {Py_tp_getset, bytesio_getsetlist}, {Py_tp_init, _io_BytesIO___init__}, {Py_tp_new, bytesio_new}, diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 732eb28e51ba8c1..a954bc5cd407d27 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -5,7 +5,6 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include // bool #ifdef HAVE_UNISTD_H @@ -81,8 +80,6 @@ typedef struct { modified outside of the fileio object / Python (ex. gh-90102, GH-121941, gh-109523). */ struct _Py_stat_struct *stat_atopen; - PyObject *weakreflist; - PyObject *dict; } fileio; #define PyFileIO_Check(state, op) (PyObject_TypeCheck((op), state->PyFileIO_Type)) @@ -221,7 +218,6 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->truncate = 0; self->stat_atopen = NULL; self->closefd = 1; - self->weakreflist = NULL; return (PyObject *) self; } @@ -562,15 +558,13 @@ fileio_traverse(PyObject *op, visitproc visit, void *arg) { fileio *self = PyFileIO_CAST(op); Py_VISIT(Py_TYPE(self)); - Py_VISIT(self->dict); - return 0; + return PyObject_VisitManagedDict(op, visit, arg); } static int fileio_clear(PyObject *op) { - fileio *self = PyFileIO_CAST(op); - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); return 0; } @@ -588,7 +582,7 @@ fileio_dealloc(PyObject *op) PyMem_Free(self->stat_atopen); self->stat_atopen = NULL; } - FT_CLEAR_WEAKREFS(op, self->weakreflist); + PyObject_ClearWeakRefs(op); (void)fileio_clear(op); PyTypeObject *tp = Py_TYPE(op); @@ -1370,8 +1364,6 @@ static PyGetSetDef fileio_getsetlist[] = { static PyMemberDef fileio_members[] = { {"_finalizing", Py_T_BOOL, offsetof(fileio, finalizing), 0}, - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(fileio, weakreflist), Py_READONLY}, - {"__dictoffset__", Py_T_PYSSIZET, offsetof(fileio, dict), Py_READONLY}, {NULL} }; diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 2b482687e85c5bc..13de04d3b9ce772 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -14,7 +14,6 @@ #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include // offsetof() #include "_iomodule.h" @@ -33,8 +32,6 @@ class _io._RawIOBase "PyObject *" "clinic_state()->PyRawIOBase_Type" typedef struct { PyObject_HEAD - PyObject *dict; - PyObject *weakreflist; } iobase; #define iobase_CAST(op) ((iobase *)(op)) @@ -383,15 +380,13 @@ iobase_traverse(PyObject *op, visitproc visit, void *arg) { iobase *self = iobase_CAST(op); Py_VISIT(Py_TYPE(self)); - Py_VISIT(self->dict); - return 0; + return PyObject_VisitManagedDict(op, visit, arg); } static int iobase_clear(PyObject *op) { - iobase *self = iobase_CAST(op); - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); return 0; } @@ -400,8 +395,8 @@ iobase_clear(PyObject *op) static void iobase_dealloc(PyObject *op) { - /* NOTE: since IOBaseObject has its own dict, Python-defined attributes - are still available here for close() to use. + /* NOTE: since the managed dict is only cleared below, Python-defined + attributes are still available here for close() to use. However, if the derived class declares a __slots__, those slots are already gone. */ @@ -416,8 +411,8 @@ iobase_dealloc(PyObject *op) } PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - FT_CLEAR_WEAKREFS(op, self->weakreflist); - Py_CLEAR(self->dict); + PyObject_ClearWeakRefs(op); + PyObject_ClearManagedDict(op); tp->tp_free(self); Py_DECREF(tp); } @@ -907,12 +902,6 @@ static PyGetSetDef iobase_getset[] = { {NULL} }; -static struct PyMemberDef iobase_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(iobase, weakreflist), Py_READONLY}, - {"__dictoffset__", Py_T_PYSSIZET, offsetof(iobase, dict), Py_READONLY}, - {NULL}, -}; - static PyType_Slot iobase_slots[] = { {Py_tp_dealloc, iobase_dealloc}, @@ -922,7 +911,6 @@ static PyType_Slot iobase_slots[] = { {Py_tp_iter, iobase_iter}, {Py_tp_iternext, iobase_iternext}, {Py_tp_methods, iobase_methods}, - {Py_tp_members, iobase_members}, {Py_tp_getset, iobase_getset}, {Py_tp_finalize, iobase_finalize}, {0, NULL}, @@ -932,7 +920,8 @@ PyType_Spec _Py_iobase_spec = { .name = "_io._IOBase", .basicsize = sizeof(iobase), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_DICT | + Py_TPFLAGS_MANAGED_WEAKREF), .slots = iobase_slots, }; diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 25ebe8909278487..a5687207d65e089 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -1,8 +1,8 @@ #include "Python.h" #include // offsetof() +#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_object.h" #include "pycore_unicodeobject.h" // _PyUnicode_ResizeCompact() -#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" /* Implementation note: the buffer is always at least one character longer @@ -42,8 +42,6 @@ typedef struct { PyObject *readnl; PyObject *writenl; - PyObject *dict; - PyObject *weakreflist; _PyIO_State *module_state; } stringio; @@ -675,8 +673,7 @@ stringio_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(self->readnl); Py_VISIT(self->writenl); Py_VISIT(self->decoder); - Py_VISIT(self->dict); - return 0; + return PyObject_VisitManagedDict(op, visit, arg); } static int @@ -686,7 +683,7 @@ stringio_clear(PyObject *op) Py_CLEAR(self->readnl); Py_CLEAR(self->writenl); Py_CLEAR(self->decoder); - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); return 0; } @@ -703,7 +700,7 @@ stringio_dealloc(PyObject *op) } PyUnicodeWriter_Discard(self->writer); (void)stringio_clear(op); - FT_CLEAR_WEAKREFS(op, self->weakreflist); + PyObject_ClearWeakRefs(op); tp->tp_free(self); Py_DECREF(tp); } @@ -930,11 +927,16 @@ _io_StringIO___getstate___impl(stringio *self) if (initvalue == NULL) return NULL; - if (self->dict == NULL) { + if (_PyObject_IsInstanceDictEmpty((PyObject *)self)) { dict = Py_NewRef(Py_None); } else { - dict = PyDict_Copy(self->dict); + dict = PyObject_GenericGetDict((PyObject *)self, NULL); + if (dict == NULL) { + Py_DECREF(initvalue); + return NULL; + } + Py_SETREF(dict, PyDict_Copy(dict)); if (dict == NULL) { Py_DECREF(initvalue); return NULL; @@ -1045,14 +1047,16 @@ _io_StringIO___setstate___impl(stringio *self, PyObject *state) Py_TYPE(dict)->tp_name); return NULL; } - if (self->dict) { - /* Alternatively, we could replace the internal dictionary - completely. However, it seems more practical to just update it. */ - if (PyDict_Update(self->dict, dict) < 0) - return NULL; + /* Alternatively, we could replace the internal dictionary + completely. However, it seems more practical to just update it. */ + PyObject *self_dict = PyObject_GenericGetDict((PyObject *)self, NULL); + if (self_dict == NULL) { + return NULL; } - else { - self->dict = Py_NewRef(dict); + int rc = PyDict_Update(self_dict, dict); + Py_DECREF(self_dict); + if (rc < 0) { + return NULL; } } @@ -1138,12 +1142,6 @@ static PyGetSetDef stringio_getset[] = { {NULL} }; -static struct PyMemberDef stringio_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(stringio, weakreflist), Py_READONLY}, - {"__dictoffset__", Py_T_PYSSIZET, offsetof(stringio, dict), Py_READONLY}, - {NULL}, -}; - static PyType_Slot stringio_slots[] = { {Py_tp_dealloc, stringio_dealloc}, {Py_tp_doc, (void *)_io_StringIO___init____doc__}, @@ -1151,7 +1149,6 @@ static PyType_Slot stringio_slots[] = { {Py_tp_clear, stringio_clear}, {Py_tp_iternext, stringio_iternext}, {Py_tp_methods, stringio_methods}, - {Py_tp_members, stringio_members}, {Py_tp_getset, stringio_getset}, {Py_tp_init, _io_StringIO___init__}, {Py_tp_new, stringio_new}, diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 719f9ef36f49570..967392313bea200 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -16,7 +16,6 @@ #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_unicodeobject.h" // _PyUnicode_AsASCIIString() -#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" @@ -731,8 +730,6 @@ struct textio /* Cache raw object if it's a FileIO object */ PyObject *raw; - PyObject *weakreflist; - PyObject *dict; _PyIO_State *state; }; @@ -1542,7 +1539,7 @@ textiowrapper_clear(PyObject *op) Py_CLEAR(self->errors); Py_CLEAR(self->raw); - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); return 0; } @@ -1556,7 +1553,7 @@ textiowrapper_dealloc(PyObject *op) return; self->ok = 0; _PyObject_GC_UNTRACK(self); - FT_CLEAR_WEAKREFS(op, self->weakreflist); + PyObject_ClearWeakRefs(op); (void)textiowrapper_clear(op); tp->tp_free(self); Py_DECREF(tp); @@ -1578,8 +1575,7 @@ textiowrapper_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(self->errors); Py_VISIT(self->raw); - Py_VISIT(self->dict); - return 0; + return PyObject_VisitManagedDict(op, visit, arg); } static PyObject * @@ -3565,8 +3561,6 @@ static PyMemberDef textiowrapper_members[] = { {"line_buffering", Py_T_BOOL, offsetof(textio, line_buffering), Py_READONLY}, {"write_through", Py_T_BOOL, offsetof(textio, write_through), Py_READONLY}, {"_finalizing", Py_T_BOOL, offsetof(textio, finalizing), 0}, - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(textio, weakreflist), Py_READONLY}, - {"__dictoffset__", Py_T_PYSSIZET, offsetof(textio, dict), Py_READONLY}, {NULL} }; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 183506adc678ea2..a6e884fb9c5fc82 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -10,7 +10,6 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() -#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #ifdef HAVE_WINDOWS_CONSOLE_IO @@ -216,8 +215,6 @@ typedef struct { unsigned int closefd : 1; char finalizing; unsigned int blksize; - PyObject *weakreflist; - PyObject *dict; char buf[SMALLBUF]; wchar_t wbuf; } winconsoleio; @@ -300,7 +297,6 @@ winconsoleio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->writable = 0; self->closefd = 0; self->blksize = 0; - self->weakreflist = NULL; } return (PyObject *) self; @@ -498,15 +494,13 @@ winconsoleio_traverse(PyObject *op, visitproc visit, void *arg) { winconsoleio *self = winconsoleio_CAST(op); Py_VISIT(Py_TYPE(self)); - Py_VISIT(self->dict); - return 0; + return PyObject_VisitManagedDict(op, visit, arg); } static int winconsoleio_clear(PyObject *op) { - winconsoleio *self = winconsoleio_CAST(op); - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); return 0; } @@ -519,8 +513,8 @@ winconsoleio_dealloc(PyObject *op) if (_PyIOBase_finalize(op) < 0) return; _PyObject_GC_UNTRACK(self); - FT_CLEAR_WEAKREFS(op, self->weakreflist); - Py_CLEAR(self->dict); + PyObject_ClearWeakRefs(op); + PyObject_ClearManagedDict(op); tp->tp_free(self); Py_DECREF(tp); } @@ -1261,8 +1255,6 @@ static PyGetSetDef winconsoleio_getsetlist[] = { static PyMemberDef winconsoleio_members[] = { {"_blksize", Py_T_UINT, offsetof(winconsoleio, blksize), 0}, {"_finalizing", Py_T_BOOL, offsetof(winconsoleio, finalizing), 0}, - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(winconsoleio, weakreflist), Py_READONLY}, - {"__dictoffset__", Py_T_PYSSIZET, offsetof(winconsoleio, dict), Py_READONLY}, {NULL} }; From 8dbf4cfc14482f44954bf296f861f3aaec4eb968 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Fri, 20 Nov 2026 11:57:01 +0100 Subject: [PATCH 219/229] gh-159580: Improve performance of isinstance() for misses and improve documentation (#159581) --- Doc/builtins/functions.rst | 5 ++++ ...-10-23-08-01-14.gh-issue-159580.lEl21_.rst | 1 + Objects/abstract.c | 25 +++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-08-01-14.gh-issue-159580.lEl21_.rst diff --git a/Doc/builtins/functions.rst b/Doc/builtins/functions.rst index 717efe50c130744..8186eae56626219 100644 --- a/Doc/builtins/functions.rst +++ b/Doc/builtins/functions.rst @@ -1181,6 +1181,11 @@ are always available. They are listed here in alphabetical order. a :exc:`TypeError` exception is raised. :exc:`TypeError` may not be raised for an invalid type if an earlier check succeeds. + If *object* has a ``__class__`` attribute that differs from + :func:`type(object) ` (for example a proxy, or a :mod:`unittest.mock` + object created with a *spec*), ``True`` is also returned when that class + is a subclass of *classinfo*. + .. versionchanged:: 3.10 *classinfo* can be a :ref:`types-union`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-08-01-14.gh-issue-159580.lEl21_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-08-01-14.gh-issue-159580.lEl21_.rst new file mode 100644 index 000000000000000..5acc2d3283d5ad2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-08-01-14.gh-issue-159580.lEl21_.rst @@ -0,0 +1 @@ +Improve performance of :func:`isinstance` when the check fails. diff --git a/Objects/abstract.c b/Objects/abstract.c index 6bb1ac572b0c6e8..ff56cdb9efb39b8 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2600,6 +2600,28 @@ check_class(PyObject *cls, const char *error) return -1; } +/* Return 1 if inst.__class__ would resolve to object's own getset descriptor, + i.e. evaluate to Py_TYPE(inst). Types with a custom tp_getattro or an + overridden __class__ return 0 and take the generic lookup. */ +static inline int +class_attr_is_type(PyObject *inst) +{ + PyTypeObject *tp = Py_TYPE(inst); + if (tp->tp_getattro != PyObject_GenericGetAttr) { + return 0; + } + _PyStackRef cref; + _PyType_LookupStackRefAndVersion(tp, &_Py_ID(__class__), &cref); + if (PyStackRef_IsNull(cref)) { + return 0; + } + PyObject *descr = PyStackRef_AsPyObjectBorrow(cref); + int res = (Py_IS_TYPE(descr, &PyGetSetDescr_Type) + && PyDescr_TYPE(descr) == &PyBaseObject_Type); + PyStackRef_CLOSE(cref); + return res; +} + static int object_isinstance(PyObject *inst, PyObject *cls) { @@ -2608,6 +2630,9 @@ object_isinstance(PyObject *inst, PyObject *cls) if (PyType_Check(cls)) { retval = PyObject_TypeCheck(inst, (PyTypeObject *)cls); if (retval == 0) { + if (class_attr_is_type(inst)) { + return 0; + } retval = PyObject_GetOptionalAttr(inst, &_Py_ID(__class__), &icls); if (icls != NULL) { if (icls != (PyObject *)(Py_TYPE(inst)) && PyType_Check(icls)) { From 9e5ab7e13f4fe9924055b624fec04de15c4561aa Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 20 Nov 2026 17:48:42 +0530 Subject: [PATCH 220/229] gh-152276: Fix assertion failure when an object becomes immortal while queued for BRC merging (#162316) --- Lib/test/test_free_threading/test_races.py | 16 ++++++++++++++++ ...026-11-20-12-00-00.gh-issue-152276.bRcImm.rst | 2 ++ Objects/object.c | 7 ++++++- 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-20-12-00-00.gh-issue-152276.bRcImm.rst diff --git a/Lib/test/test_free_threading/test_races.py b/Lib/test/test_free_threading/test_races.py index f08de7496eb526d..dc1617512f8795d 100644 --- a/Lib/test/test_free_threading/test_races.py +++ b/Lib/test/test_free_threading/test_races.py @@ -288,6 +288,22 @@ def set_recursion_limit(): do_race(something_recursive, set_recursion_limit) + def test_racing_intern_queued_for_merge(self): + # gh-152276: an object may become immortal while it is queued for + # its owning thread to merge its reference count. + # The same strings are used on every run, so that reruns do not + # create new immortal strings. + for i in range(10): + strings = [f"test_racing_intern_queued_for_merge_{i}_{j}" + for j in range(1000)] + shared = list(strings) + # Dropping the references from another thread queues the strings + # to be merged by this thread, which is busy interning them. + thread = threading.Thread(target=shared.clear) + thread.start() + list(map(sys.intern, strings)) + thread.join() + @threading_helper.requires_working_threading() class TestWarningsRaces(TestBase): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-20-12-00-00.gh-issue-152276.bRcImm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-20-12-00-00.gh-issue-152276.bRcImm.rst new file mode 100644 index 000000000000000..9cc5d98dae0d0f1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-20-12-00-00.gh-issue-152276.bRcImm.rst @@ -0,0 +1,2 @@ +For the :term:`free-threaded build`, avoid an assertion failure when an object +becomes immortal while queued for biased reference counting (BRC) merging. diff --git a/Objects/object.c b/Objects/object.c index 0695cc090c979c4..1589e3523e62053 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -468,7 +468,12 @@ _Py_MergeZeroLocalRefcount(PyObject *op) Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) { - assert(!_Py_IsImmortal(op)); + if (_Py_IsImmortal(op)) { + // The object may have become immortal after it was queued to be + // merged (for example, by sys.intern()). There is nothing to merge + // and the reference total no longer includes the object. + return _Py_IMMORTAL_INITIAL_REFCNT; + } #ifdef Py_REF_DEBUG _Py_AddRefTotal(_PyThreadState_GET(), extra); From 89d78fc6f9514914cec7a934af819f397b05d75e Mon Sep 17 00:00:00 2001 From: mpage Date: Fri, 20 Nov 2026 04:22:37 -0800 Subject: [PATCH 221/229] gh-154130: Fix double decref of the dict in shared dict iterators (#160312) Co-authored-by: Kumar Aditya --- Lib/test/test_free_threading/test_dict.py | 45 ++++++++++++++++++ Objects/dictobject.c | 57 ++++++++++++++++++----- 2 files changed, 90 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index faf0fa96f07a152..47fe8b2b0bc6342 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -565,6 +565,51 @@ def race(iter_times: int, CHURN_SIZE = 512 race(ITER_TIMES, CHURN_SIZE, APPENDER_THREADS, REPLACER_THREADS) + def test_racing_shared_iterator_exhaustion(self): + # gh-154130: exhausting a dict iterator shared between threads must + # not release the iterator's reference to the dict more than once. + makers = [ + lambda d: iter(d), + lambda d: iter(d.values()), + lambda d: iter(d.items()), + lambda d: reversed(d), + lambda d: reversed(d.items()), + ] + for make in makers: + cell = [make({i: i for i in range(8)})] + + def worker(): + for _ in range(5000): + it = cell[0] + try: + next(it) + except StopIteration: + cell[0] = make({i: i for i in range(8)}) + except RuntimeError: + # Concurrent next() calls may observe each other's + # updates as a modification; only crashes are bugs. + pass + + with self.subTest(make=make): + threading_helper.run_concurrently(worker, nthreads=4) + + def test_exhausted_iterator_stays_exhausted(self): + d = {1: 1} + for it in (iter(d), iter(d.items()), reversed(d)): + self.assertEqual(len(list(it)), 1) + d[2] = 2 + self.assertRaises(StopIteration, next, it) + self.assertEqual(it.__length_hint__(), 0) + del d[2] + + def test_reversed_empty_dict_iterator_not_exhausted(self): + # A reverse iterator over an empty dict starts at position -1 but is + # not exhausted, so a later size change must still be detected. + d = {} + it = reversed(d) + d[1] = 1 + self.assertRaises(RuntimeError, next, it) + def test_racing_update_from_mutating_pair(self): # gh-159244: dict() must not read the items of a key/value pair # that another thread empties concurrently. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 4a1d1b069a21ed8..c9cc5d8c566fc6f 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5705,7 +5705,10 @@ PyDict_DelItemString(PyObject *v, const char *key) typedef struct { PyObject_HEAD - PyDictObject *di_dict; /* Set to NULL when iterator is exhausted */ + /* Set to NULL when the iterator is exhausted, except in the free-threaded + build, where the reference is kept until the iterator is deallocated + (see dictiter_set_exhausted()). */ + PyDictObject *di_dict; Py_ssize_t di_used; Py_ssize_t di_pos; PyObject* di_result; /* reusable result tuple for iteritems */ @@ -5800,6 +5803,28 @@ static PyMethodDef dictiter_methods[] = { {NULL, NULL} /* sentinel */ }; +/* Mark the iterator as exhausted. In the free-threaded build an iterator + may be advanced by several threads at once, each holding a borrowed + pointer to di_dict, so the iterator keeps its reference to the dict until + it is deallocated (as list and tuple iterators do) and records exhaustion + by setting di_pos to DICTITER_EXHAUSTED. This is distinct from -1, the + starting position of a reverse iterator over an empty dict. */ +#ifdef Py_GIL_DISABLED +#define DICTITER_EXHAUSTED PY_SSIZE_T_MIN +#endif + +static void +dictiter_set_exhausted(dictiterobject *di, PyDictObject *d) +{ +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ssize_relaxed(&di->di_pos, DICTITER_EXHAUSTED); + _Py_atomic_store_ssize_relaxed(&di->len, 0); +#else + di->di_dict = NULL; + Py_DECREF(d); +#endif +} + #ifdef Py_GIL_DISABLED static int @@ -6065,6 +6090,11 @@ dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, assert (PyAnyDict_Check(d)); ASSERT_DICT_LOCKED(d); + i = FT_ATOMIC_LOAD_SSIZE_RELAXED(di->di_pos); + if (i < 0) { + return -1; /* exhausted */ + } + if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration"); @@ -6072,9 +6102,6 @@ dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, return -1; } - i = FT_ATOMIC_LOAD_SSIZE_RELAXED(di->di_pos); - - assert(i >= 0); if (_PyDict_HasSplitTable(d)) { if (i >= d->ma_used) goto fail; @@ -6125,8 +6152,7 @@ dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, return 0; fail: - di->di_dict = NULL; - Py_DECREF(d); + dictiter_set_exhausted(di, d); return -1; } @@ -6170,6 +6196,11 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, assert (PyAnyDict_Check(d)); + i = _Py_atomic_load_ssize_relaxed(&di->di_pos); + if (i == DICTITER_EXHAUSTED) { + return -1; + } + if (di->di_used != _Py_atomic_load_ssize_relaxed(&d->ma_used)) { PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration"); @@ -6179,9 +6210,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, ensure_shared_on_read(d); - i = _Py_atomic_load_ssize_relaxed(&di->di_pos); k = _Py_atomic_load_ptr_acquire(&d->ma_keys); - assert(i >= 0); if (_PyDict_HasSplitTable(d)) { PyDictValues *values = _Py_atomic_load_ptr_consume(&d->ma_values); if (values == NULL) { @@ -6260,8 +6289,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, "dictionary keys changed during iteration"); fail: - di->di_dict = NULL; - Py_DECREF(d); + dictiter_set_exhausted(di, d); return -1; try_locked: @@ -6363,6 +6391,12 @@ dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) assert (PyAnyDict_Check(d)); ASSERT_DICT_LOCKED(d); +#ifdef Py_GIL_DISABLED + if (di->di_pos == DICTITER_EXHAUSTED) { + return NULL; + } +#endif + if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration"); @@ -6449,8 +6483,7 @@ dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) } fail: - di->di_dict = NULL; - Py_DECREF(d); + dictiter_set_exhausted(di, d); return NULL; } From a3ea5d29f5b0285f7ac1b0053f15b30f2c8daec9 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 20 Nov 2026 18:04:49 +0530 Subject: [PATCH 222/229] gh-162217: Fix InvalidStateError when subprocess stdin is closed after a cancelled wait_closed() (#162317) --- Lib/asyncio/subprocess.py | 15 +++++------ Lib/test/test_asyncio/test_subprocess.py | 25 +++++++++++++++++++ ...-11-20-09-30-00.gh-issue-162217.wC7sHd.rst | 4 +++ 3 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-20-09-30-00.gh-issue-162217.wC7sHd.rst diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index 26d7b755105c9d5..f83d00af5b5d450 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -77,13 +77,14 @@ def pipe_connection_lost(self, fd, exc): if pipe is not None: pipe.close() self.connection_lost(exc) - if exc is None: - self._stdin_closed.set_result(None) - else: - self._stdin_closed.set_exception(exc) - # Since calling `wait_closed()` is not mandatory, - # we shouldn't log the traceback if this is not awaited. - self._stdin_closed._log_traceback = False + if not self._stdin_closed.done(): + if exc is None: + self._stdin_closed.set_result(None) + else: + self._stdin_closed.set_exception(exc) + # Since calling `wait_closed()` is not mandatory, + # we shouldn't log the traceback if this is not awaited. + self._stdin_closed._log_traceback = False return if fd == 1: reader = self.stdout diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 6e85bdd57c9085b..38c236c9441ee07 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -435,6 +435,31 @@ async def write_stdin(proc, data): self.loop.run_until_complete, coro) self.loop.run_until_complete(proc.wait()) + def test_stdin_wait_closed_cancelled(self): + # gh-162217: cancelling a task waiting in stdin.wait_closed() must + # not make pipe_connection_lost() fail with InvalidStateError. + async def main(): + proc = await asyncio.create_subprocess_exec( + *PROGRAM_CAT, + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + ) + try: + task = asyncio.create_task(proc.stdin.wait_closed()) + await asyncio.sleep(0) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + finally: + proc.stdin.close() + await proc.wait() + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + + self.loop.run_until_complete(main()) + self.assertEqual(messages, []) + def test_communicate_ignore_broken_pipe(self): # buffer large enough to feed the whole pipe buffer large_data = b'x' * support.PIPE_MAX_SIZE diff --git a/Misc/NEWS.d/next/Library/2026-11-20-09-30-00.gh-issue-162217.wC7sHd.rst b/Misc/NEWS.d/next/Library/2026-11-20-09-30-00.gh-issue-162217.wC7sHd.rst new file mode 100644 index 000000000000000..be12568cb8f7fec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-20-09-30-00.gh-issue-162217.wC7sHd.rst @@ -0,0 +1,4 @@ +Fix :exc:`asyncio.InvalidStateError` being logged when the standard input of +an :mod:`asyncio` subprocess is closed after a task waiting in +:meth:`proc.stdin.wait_closed() ` was +cancelled. From d1e3cae9d6d55abd763dfca18877c2bd0db13ad0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Nov 2026 13:44:43 +0100 Subject: [PATCH 223/229] gh-158401: Implement Py_INCREF() as function call in stable ABI (#162120) Py_INCREF(), Py_XINCREF(), Py_DECREF() and Py_XDECREF() are now implemented as function calls on all limited C API versions. Previously, they were only implemented as function calls on the limited C API version 3.12 or newer. Previously, if a C extension was built with the limited C API version 3.11 or newer, Py_DECREF() was implemented as "op->ob_refcnt--". But Python 3.12 made Py_DECREF() more complicated to implement immortal objects in an efficient way. "op->ob_refcnt--" can convert an immortal object to a mortal object which causes various kinds of nasty bugs. Co-authored-by: Hood Chatham Co-authored-by: Petr Viktorin --- Include/refcount.h | 26 +++++++++---------- ...-11-16-17-25-12.gh-issue-158401.ADBBRG.rst | 4 +++ 2 files changed, 16 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-11-16-17-25-12.gh-issue-158401.ADBBRG.rst diff --git a/Include/refcount.h b/Include/refcount.h index 9b681f6a254a42a..dc47a49cfbbb180 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -254,13 +254,12 @@ PyAPI_FUNC(void) _Py_DecRef(PyObject *); static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { -#if (defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))) \ - || defined(_Py_OPAQUE_PYOBJECT) - // Stable ABI implements Py_INCREF() as a function call on limited C API - // version 3.12 and newer, abi3t, and on Python built in debug mode. - // _Py_IncRef() was added to Python 3.10.0a7, use Py_IncRef() on older versions. - // Py_IncRef() accepts NULL whereas _Py_IncRef() doesn't. -# if Py_LIMITED_API+0 >= 0x030a00A7 +#if defined(Py_LIMITED_API) || defined(_Py_OPAQUE_PYOBJECT) + // gh-158401: The stable ABI now implements Py_INCREF() as a function + // call on all limited C API versions. _Py_IncRef() was added to Python + // 3.10.0a7, use Py_IncRef() on older versions. Py_IncRef() accepts NULL, + // whereas _Py_IncRef() doesn't. +# if Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 10) _Py_IncRef(op); # else Py_IncRef(op); @@ -324,14 +323,13 @@ PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *); #endif // Py_GIL_DISABLED #endif // Py_LIMITED_API -#if (defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))) \ - || defined(_Py_OPAQUE_PYOBJECT) -// Stable ABI implements Py_DECREF() as a function call on limited C API -// version 3.12 and newer, abi3t, and on Python built in debug mode. -// _Py_DecRef() was added to Python 3.10.0a7, use Py_DecRef() on older versions. -// Py_DecRef() accepts NULL whereas _Py_DecRef() doesn't. +#if defined(Py_LIMITED_API) || defined(_Py_OPAQUE_PYOBJECT) +// gh-158401: The stable ABI now implements Py_DECREF() as a function call +// on all limited C API versions. _Py_DecRef() was added to Python 3.10.0a7, +// use Py_DecRef() on older versions. Py_DecRef() accepts NULL, whereas +// _Py_DecRef() doesn't. static inline void Py_DECREF(PyObject *op) { -# if Py_LIMITED_API+0 >= 0x030a00A7 +# if Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 10) _Py_DecRef(op); # else Py_DecRef(op); diff --git a/Misc/NEWS.d/next/C_API/2026-11-16-17-25-12.gh-issue-158401.ADBBRG.rst b/Misc/NEWS.d/next/C_API/2026-11-16-17-25-12.gh-issue-158401.ADBBRG.rst new file mode 100644 index 000000000000000..f1b59a75095b2a0 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-11-16-17-25-12.gh-issue-158401.ADBBRG.rst @@ -0,0 +1,4 @@ +:c:func:`Py_INCREF`, :c:func:`Py_XINCREF`, :c:func:`Py_DECREF` and +:c:func:`Py_XDECREF` are now implemented as function calls on all limited C +API versions. Previously, they were only implemented as function calls on +the limited C API version 3.12 or newer. Patch by Victor Stinner: From 921de8fa82707b14ddcbf1f2e383c71adb727118 Mon Sep 17 00:00:00 2001 From: Tan Long Date: Fri, 20 Nov 2026 21:24:14 +0800 Subject: [PATCH 224/229] gh-162251: Fix external inspection test_popping_python_frame_is_not_native() (#162268) Wait for child execve() before creating `RemoteUnwinder`. Use _target_processing() to sync with the child process. Co-authored-by: Victor Stinner --- Lib/test/test_external_inspection.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 5aebe1fb6b18e95..d9cd7ee72fd2b22 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -2068,7 +2068,7 @@ def test_empty_native_thread_stack(self): "Test only runs on Linux with process_vm_readv support", ) def test_popping_python_frame_is_not_native(self): - script = """\ + script_body = """\ def leaf(depth): if depth: leaf(depth - 1) @@ -2076,15 +2076,11 @@ def leaf(depth): while True: leaf(300) """ - with _managed_subprocess([sys.executable, "-c", script]) as process: - for _ in busy_retry(SHORT_TIMEOUT): - try: - unwinder = RemoteUnwinder( - process.pid, native=True, gc=False, cache_frames=False, - ) - except RuntimeError: - continue - break + + with self._target_process(script_body) as (process, _, _): + unwinder = RemoteUnwinder( + process.pid, native=True, gc=False, cache_frames=False, + ) # The unwinder can attach before the main loop starts. _get_stack_trace_with_retry( unwinder, From 4d342eb907bff64ac99825faec95f73584ebf334 Mon Sep 17 00:00:00 2001 From: Paul Tobias Date: Fri, 20 Nov 2026 20:26:46 +0700 Subject: [PATCH 225/229] gh-155935: Stop `FileFinder._find_children` retrying a failing directory scan (#155939) --- Lib/importlib/_bootstrap_external.py | 11 +++- Lib/test/test_importlib/test_discover.py | 64 ++++++++++++++++++- ...-08-17-06-07-49.gh-issue-155935.lbMPKW.rst | 1 + 3 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-17-06-07-49.gh-issue-155935.lbMPKW.rst diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 2c8d482e4f64366..53b85f0dad8fcb5 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -1461,6 +1461,13 @@ def _find_children(self): while True: try: entry = next(scan_iterator) + except StopIteration: + break + except OSError: + # A failing scan cannot make progress; end the listing + # like _fill_cache() treats an unreadable directory. + break + try: if entry.name == _PYCACHE: continue # packages @@ -1474,9 +1481,7 @@ def _find_children(self): if entry.name.endswith(suffix) } except OSError: - pass # ignore exceptions from next(scan_iterator) and os.DirEntry - except StopIteration: - break + pass # skip entries whose os.DirEntry methods fail def discover(self, parent=None): if parent and parent.submodule_search_locations is None: diff --git a/Lib/test/test_importlib/test_discover.py b/Lib/test/test_importlib/test_discover.py index c4ab7b6982e3ba9..490b54569e09c42 100644 --- a/Lib/test/test_importlib/test_discover.py +++ b/Lib/test/test_importlib/test_discover.py @@ -1,4 +1,4 @@ -from unittest.mock import Mock +from unittest.mock import Mock, patch from test.test_importlib import util @@ -114,6 +114,68 @@ def test_invalid_parent(self): with self.assertRaises(ValueError): list(finder.discover(example)) + def _patch_scandir(self, scandir): + module_os = self.machinery.FileFinder._fill_cache.__globals__['_os'] + return patch.object(module_os, 'scandir', scandir) + + def test_discover_persistently_failing_scan(self): + # gh-155935: an iterator that raises OSError on every next() call + # must end the listing instead of looping forever. + class FailingScandirIterator: + calls = 0 + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def __next__(self): + self.calls += 1 + if self.calls > 100: + # Safety net so regressed code fails fast on the call + # count below instead of hanging the test forever. + raise StopIteration + raise OSError('persistently failing directory scan') + + scan_iterator = FailingScandirIterator() + with self._patch_scandir(lambda path: scan_iterator): + finder = self.get_finder('dummy') + discovered = list(finder.discover()) + self.assertEqual(discovered, []) + # A failed scan must not be retried. + self.assertEqual(scan_iterator.calls, 1) + + def test_find_children_failing_direntry(self): + # An entry whose DirEntry methods raise OSError is skipped; the + # remaining entries are still listed. + failing = Mock() + failing.name = 'failing' + failing.is_dir.side_effect = OSError('stat failed') + good = Mock() + good.name = 'example.py' + good.is_dir.return_value = False + good.is_file.return_value = True + + class FakeScandirIterator: + def __init__(self, entries): + self._iterator = iter(entries) + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def __next__(self): + return next(self._iterator) + + with self._patch_scandir( + lambda path: FakeScandirIterator([failing, good])): + finder = self.get_finder('dummy') + children = list(finder._find_children()) + self.assertEqual(children, ['example']) + ( Frozen_TestFileFinder, diff --git a/Misc/NEWS.d/next/Library/2026-08-17-06-07-49.gh-issue-155935.lbMPKW.rst b/Misc/NEWS.d/next/Library/2026-08-17-06-07-49.gh-issue-155935.lbMPKW.rst new file mode 100644 index 000000000000000..21d7ba6a4f758c4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-17-06-07-49.gh-issue-155935.lbMPKW.rst @@ -0,0 +1 @@ +Fix :meth:`!importlib.machinery.FileFinder.discover` looping forever when the underlying directory scan keeps failing with :exc:`OSError`. A failing scan now ends the listing instead of being retried. From 1782f62e8f97b4c333c917adaae6a68f7ddec964 Mon Sep 17 00:00:00 2001 From: scrieve <86650480+scrieve@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:30:19 +0000 Subject: [PATCH 226/229] gh-160573: Drop the set once when a shared iterator is exhausted (#160619) Co-authored-by: Kumar Aditya --- Lib/test/test_free_threading/test_set.py | 67 +++++++++++++++++ ...-10-24-19-40-00.gh-issue-160573.sT9kQe.rst | 4 + Objects/setobject.c | 75 ++++++++++++++----- 3 files changed, 127 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-40-00.gh-issue-160573.sT9kQe.rst diff --git a/Lib/test/test_free_threading/test_set.py b/Lib/test/test_free_threading/test_set.py index 9dd3d68d5dad135..639bc729fa31ac6 100644 --- a/Lib/test/test_free_threading/test_set.py +++ b/Lib/test/test_free_threading/test_set.py @@ -159,5 +159,72 @@ class LargeSetTest(RaceTestBase, unittest.TestCase): SET_SIZE = 20 # larger than PySet_MINSIZE +class TestSetIteratorExhaustion(unittest.TestCase): + def test_racing_shared_iterator_exhaustion(self): + # gh-160573: exhausting a set iterator shared between threads must + # not release the iterator's reference to the set more than once. + for tp in (set, frozenset): + cell = [iter(tp(range(8)))] + + def worker(): + for _ in range(5000): + it = cell[0] + try: + next(it) + except StopIteration: + cell[0] = iter(tp(range(8))) + + with self.subTest(tp=tp): + threading_helper.run_concurrently(worker, nthreads=4) + + def test_racing_shared_iterator_exhaustion_contended(self): + # Same as above, but the set's lock is contended: other threads + # keep mutating and iterating the set while its iterator is shared. + s = set(range(8)) + cell = [iter(s)] + done = Event() + + def exhaust(): + for _ in range(5000): + it = cell[0] + try: + next(it) + except StopIteration: + cell[0] = iter(s) + except RuntimeError: + # The set changed size during iteration. + cell[0] = iter(s) + + def mutate(): + while not done.is_set(): + s.add(100) + s.discard(100) + + def iterate(): + while not done.is_set(): + try: + for _ in s: + pass + except RuntimeError: + pass + + def run_exhaust(): + try: + threading_helper.run_concurrently(exhaust, nthreads=4) + finally: + done.set() + + threading_helper.run_concurrently([run_exhaust, mutate, iterate]) + + def test_exhausted_iterator_stays_exhausted(self): + s = {1} + it = iter(s) + self.assertEqual(list(it), [1]) + s.add(2) + self.assertRaises(StopIteration, next, it) + self.assertEqual(it.__length_hint__(), 0) + self.assertEqual(list(it.__reduce__()[1][0]), []) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-40-00.gh-issue-160573.sT9kQe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-40-00.gh-issue-160573.sT9kQe.rst new file mode 100644 index 000000000000000..7a69a4769204857 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-40-00.gh-issue-160573.sT9kQe.rst @@ -0,0 +1,4 @@ +Fix a crash when a :class:`set` iterator that is shared between threads reaches +its end in a free-threaded build: every thread that walked off the end of the +table released the iterator's reference to the set, so the set was over-released. +The iterator now keeps its reference to the set until it is deallocated. diff --git a/Objects/setobject.c b/Objects/setobject.c index c6ceebeb09957c6..6ba6e611d9448dc 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1027,7 +1027,10 @@ frozenset_hash(PyObject *self) typedef struct { PyObject_HEAD - PySetObject *si_set; /* Set to NULL when iterator is exhausted */ + /* Set to NULL when the iterator is exhausted, except in the free-threaded + build, where the reference is kept until the iterator is deallocated + (see setiter_set_exhausted()). */ + PySetObject *si_set; Py_ssize_t si_used; Py_ssize_t si_pos; Py_ssize_t len; @@ -1056,8 +1059,14 @@ setiter_len(PyObject *op, PyObject *Py_UNUSED(ignored)) { setiterobject *si = (setiterobject*)op; Py_ssize_t len = 0; - if (si->si_set != NULL && si->si_used == si->si_set->used) - len = si->len; + PySetObject *so = si->si_set; + if (so != NULL) { + Py_ssize_t so_used = FT_ATOMIC_LOAD_SSIZE_RELAXED(so->used); + Py_ssize_t si_used = FT_ATOMIC_LOAD_SSIZE_RELAXED(si->si_used); + if (si_used == so_used) { + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(si->len); + } + } return PyLong_FromSsize_t(len); } @@ -1089,6 +1098,25 @@ static PyMethodDef setiter_methods[] = { {NULL, NULL} /* sentinel */ }; +/* Mark the iterator as exhausted. In the free-threaded build an iterator + may be advanced by several threads at once, each holding a borrowed + pointer to si_set, so the iterator keeps its reference to the set until + it is deallocated (as list and tuple iterators do) and records exhaustion + by setting si_pos to SETITER_EXHAUSTED. */ +#define SETITER_EXHAUSTED (-1) + +static void +setiter_set_exhausted(setiterobject *si, PySetObject *so) +{ +#ifdef Py_GIL_DISABLED + FT_ATOMIC_STORE_SSIZE_RELAXED(si->si_pos, SETITER_EXHAUSTED); + FT_ATOMIC_STORE_SSIZE_RELAXED(si->len, 0); +#else + si->si_set = NULL; + Py_DECREF(so); +#endif +} + static PyObject *setiter_iternext(PyObject *self) { setiterobject *si = (setiterobject*)self; @@ -1101,34 +1129,43 @@ static PyObject *setiter_iternext(PyObject *self) return NULL; assert (PyAnySet_Check(so)); +#ifdef Py_GIL_DISABLED + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(si->si_pos) == SETITER_EXHAUSTED) { + return NULL; + } +#endif + Py_ssize_t so_used = FT_ATOMIC_LOAD_SSIZE_RELAXED(so->used); Py_ssize_t si_used = FT_ATOMIC_LOAD_SSIZE_RELAXED(si->si_used); if (si_used != so_used) { PyErr_SetString(PyExc_RuntimeError, "Set changed size during iteration"); - si->si_used = -1; /* Make this state sticky */ + /* Make this state sticky */ + FT_ATOMIC_STORE_SSIZE_RELAXED(si->si_used, -1); return NULL; } + /* si_pos and len are only written with the set locked, so that an + iterator marked as exhausted by another thread stays exhausted. */ Py_BEGIN_CRITICAL_SECTION(so); i = si->si_pos; - assert(i>=0); - entry = so->table; - mask = so->mask; - while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) { - i++; - } - if (i <= mask) { - key = Py_NewRef(entry[i].key); + if (i != SETITER_EXHAUSTED) { + assert(i >= 0); + entry = so->table; + mask = so->mask; + while (i <= mask && (entry[i].key == NULL || entry[i].key == dummy)) { + i++; + } + if (i <= mask) { + key = Py_NewRef(entry[i].key); + FT_ATOMIC_STORE_SSIZE_RELAXED(si->si_pos, i + 1); + FT_ATOMIC_STORE_SSIZE_RELAXED(si->len, si->len - 1); + } + else { + setiter_set_exhausted(si, so); + } } Py_END_CRITICAL_SECTION(); - si->si_pos = i+1; - if (key == NULL) { - si->si_set = NULL; - Py_DECREF(so); - return NULL; - } - si->len--; return key; } From e0d0f2e153450cf79b84e2ec5ffbcbad8aea7922 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Fri, 20 Nov 2026 17:46:04 +0200 Subject: [PATCH 227/229] gh-160697: Process the QSBR queue from the eval breaker, not from `_PyMem_FreeDelayed()` (#160698) --- Include/internal/pycore_ceval.h | 3 +- Include/internal/pycore_qsbr.h | 11 --- InternalDocs/qsbr.md | 16 +++-- Lib/test/test_free_threading/test_qsbr.py | 72 +++++++++++++++++++ ...-10-26-10-18-26.gh-issue-160697.qSbR1f.rst | 4 ++ Objects/obmalloc.c | 14 ++-- Python/ceval_gil.c | 2 +- Python/instrumentation.c | 2 +- 8 files changed, 93 insertions(+), 31 deletions(-) create mode 100644 Lib/test/test_free_threading/test_qsbr.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-10-18-26.gh-issue-160697.qSbR1f.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index a0fea389b6502c0..c404fc6e67d1932 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -350,9 +350,10 @@ _PyEval_SpecialMethodCanSuggest(PyObject *self, int oparg); #define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) #define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) #define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7) +#define _PY_EVAL_PROCESS_DELAYED_BIT (1U << 8) /* Reserve a few bits for future use */ -#define _PY_EVAL_EVENTS_BITS 8 +#define _PY_EVAL_EVENTS_BITS 9 #define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1) static inline void diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index 91d3415614b154d..98c23384221d9ca 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -61,9 +61,6 @@ struct _qsbr_thread_state { // and in a different thread. size_t deferred_page_memory; - // True if the deferred memory frees should be processed. - bool should_process; - // Is this thread state allocated? bool allocated; struct _qsbr_thread_state *freelist_next; @@ -142,14 +139,6 @@ _Py_qsbr_advance(struct _qsbr_shared *shared); extern uint64_t _Py_qsbr_shared_next(struct _qsbr_shared *shared); -// Return true if deferred memory frees held by QSBR should be processed to -// determine if they can be safely freed. -static inline bool -_Py_qsbr_should_process(struct _qsbr_thread_state *qsbr) -{ - return qsbr->should_process; -} - // Have the read sequences advanced to the given goal? If this returns true, // it safe to reclaim any memory tagged with the goal (or earlier goal). extern bool diff --git a/InternalDocs/qsbr.md b/InternalDocs/qsbr.md index d511396fdab6452..58532a6045d3fec 100644 --- a/InternalDocs/qsbr.md +++ b/InternalDocs/qsbr.md @@ -116,8 +116,9 @@ its last advance: is larger than `QSBR_FREE_MEM_LIMIT` (1 MiB), when the accumulated deferred memory exceeds that limit, or when the count exceeds `QSBR_DEFERRED_LIMIT` (127, sized so a chunk of work items is processed before it overflows). - Crossing any of these thresholds also sets a per-thread `should_process` - flag, signalling that the deferred-free list should be drained. + Crossing any of these thresholds also sets the thread's + `_PY_EVAL_PROCESS_DELAYED_BIT` eval breaker bit, signalling that the + deferred-free list should be drained. * For mimalloc pages held by QSBR, the thread tracks `deferred_page_memory` and advances `wr_seq` when either the individual page or the accumulated @@ -125,11 +126,12 @@ its last advance: promptly here matters because a held page cannot be reused for a different size class or by a different thread. -Processing of the deferred-free list normally happens from the eval breaker -(rather than from inside `_PyMem_FreeDelayed`), which gives the global -`rd_seq` a better chance to have advanced far enough that items can actually -be freed. `_PyMem_ProcessDelayed` is still called from the free path as a -safety valve when a work-item chunk fills up. +Processing of the deferred-free list happens from the eval breaker rather +than from inside `_PyMem_FreeDelayed`. This gives the global `rd_seq` a better +chance to have advanced far enough that items can actually be freed. It is +also required for correctness: items queued by `_PyObject_XDecRefDelayed` may +run arbitrary code when they are released, and the callers of +`_PyMem_FreeDelayed` may be in the middle of mutating a dict, list or type. This optimization improves runtime speed but may increase peak memory usage by slightly delaying when memory can be reclaimed; the size-based thresholds diff --git a/Lib/test/test_free_threading/test_qsbr.py b/Lib/test/test_free_threading/test_qsbr.py new file mode 100644 index 000000000000000..ec2a7454ca29079 --- /dev/null +++ b/Lib/test/test_free_threading/test_qsbr.py @@ -0,0 +1,72 @@ +import threading +import unittest + +from test.support import threading_helper + + +class Holder: + pass + + +def make_shared(func, *args): + t = threading.Thread(target=func, args=args) + t.start() + t.join() + + +@threading_helper.requires_working_threading() +class TestDelayedFreeReentrancy(unittest.TestCase): + ROUNDS = 1000 + + def queue_finalizers(self, callback, count=ROUNDS): + class Finalizer: + def __del__(self): + callback() + + holders = [Holder() for _ in range(count)] + for h in holders: + h.__dict__ = {"f": Finalizer()} + return holders + + def test_dict_resize(self): + d = {} + make_shared(d.get, 0) + holders = self.queue_finalizers(lambda: d.__setitem__(object(), 0), + count=100) + for h in holders: + h.__dict__ = {} + for _ in range(self.ROUNDS): + for k in list(d): + del d[k] + for i in range(6): + d[i] = i + self.assertEqual(len(d), sum(1 for _ in d)) + + def test_list_resize(self): + L = [] + make_shared(L.append, 0) + holders = self.queue_finalizers(L.clear) + for n, h in enumerate(holders): + h.__dict__ = {} + if n % 2: + h.__dict__ = {} + L.clear() + for i in range(9): + L.insert(len(L), i) + self.assertEqual(len(L), len(list(L))) + + def test_type_cache_invalidation(self): + class C: + x = None + + holders = self.queue_finalizers(lambda: C.x) + for h in holders: + h.__dict__ = {} + C.x + new = object() + C.x = new + self.assertIs(C.x, new) + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-10-18-26.gh-issue-160697.qSbR1f.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-10-18-26.gh-issue-160697.qSbR1f.rst new file mode 100644 index 000000000000000..55649732e8f4b23 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-26-10-18-26.gh-issue-160697.qSbR1f.rst @@ -0,0 +1,4 @@ +Fix crashes and hangs in the :term:`free-threaded build` when a destructor +ran in the middle of a :class:`dict` resize, a :class:`list` resize or a type +cache update. The thread's queue of delayed frees is now processed only from +the eval breaker, instead of synchronously when the queue fills up. diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 3ad485620ede41e..66931dc396fa95a 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,6 +1,7 @@ /* Python's malloc wrappers (see pymem.h) */ #include "Python.h" +#include "pycore_ceval.h" #include "pycore_interp.h" // _PyInterpreterState_HasFeature #include "pycore_mmap.h" // _PyAnnotateMemoryMap() #include "pycore_object.h" // _PyDebugAllocatorStats() definition @@ -1424,7 +1425,7 @@ should_advance_qsbr_for_free(struct _qsbr_thread_state *qsbr, size_t size) if (size > QSBR_FREE_MEM_LIMIT) { qsbr->deferred_count = 0; qsbr->deferred_memory = 0; - qsbr->should_process = true; + _Py_set_eval_breaker_bit(qsbr->tstate, _PY_EVAL_PROCESS_DELAYED_BIT); return true; } qsbr->deferred_count++; @@ -1433,7 +1434,7 @@ should_advance_qsbr_for_free(struct _qsbr_thread_state *qsbr, size_t size) qsbr->deferred_memory > QSBR_FREE_MEM_LIMIT) { qsbr->deferred_count = 0; qsbr->deferred_memory = 0; - qsbr->should_process = true; + _Py_set_eval_breaker_bit(qsbr->tstate, _PY_EVAL_PROCESS_DELAYED_BIT); return true; } return false; @@ -1511,13 +1512,6 @@ free_delayed(uintptr_t ptr, size_t size) buf->array[buf->wr_idx].ptr = ptr; buf->array[buf->wr_idx].qsbr_goal = seq; buf->wr_idx++; - - if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { - // Normally the processing of delayed items is done from the eval - // breaker. Processing here is a safety measure to ensure too much - // work does not accumulate. - _PyMem_ProcessDelayed((PyThreadState *)tstate); - } #endif } @@ -1625,7 +1619,7 @@ _PyMem_ProcessDelayed(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; - tstate_impl->qsbr->should_process = false; + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_PROCESS_DELAYED_BIT); // Process thread-local work process_queue(&tstate_impl->mem_free_queue, tstate_impl, true, NULL, NULL); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 91c75cdd2ed4133..a06eb4f72ece37a 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1387,7 +1387,7 @@ _Py_HandlePending(PyThreadState *tstate) _Py_brc_merge_refcounts(tstate); } /* Process deferred memory frees held by QSBR */ - if (_Py_qsbr_should_process(((_PyThreadStateImpl *)tstate)->qsbr)) { + if ((breaker & _PY_EVAL_PROCESS_DELAYED_BIT) != 0) { _PyMem_ProcessDelayed(tstate); } #endif diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 3c9b2afcde100f3..48daa91259d3c9e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -2041,7 +2041,7 @@ check_tool(PyInterpreterState *interp, int tool_id) } /* We share the eval-breaker with flags, so the monitoring - * version goes in the top 24 bits */ + * version goes in the bits above _PY_EVAL_EVENTS_BITS */ #define MONITORING_VERSION_INCREMENT (1 << _PY_EVAL_EVENTS_BITS) int From ab5b8f87a70b62a22c1ed78b3e26ccd9ba84caaa Mon Sep 17 00:00:00 2001 From: Timofei Ivankov <128279579+deadlovelll@users.noreply.github.com> Date: Sat, 21 Nov 2026 01:16:04 +0300 Subject: [PATCH 228/229] gh-161502: Fix missing tasks in asyncio pstree with intermediate Futures (#161508) Include intermediate Future nodes when the remote asyncio unwinder builds awaited-by graphs. --- Lib/asyncio/tools.py | 13 ++- Lib/test/test_external_inspection.py | 98 +++++++++++++++++++ ...-11-06-20-12-44.gh-issue-161502.rYZtwK.rst | 3 + Modules/_remote_debugging/asyncio.c | 74 +++++++++++--- Modules/_remote_debugging/clinic/module.c.h | 19 ++-- Modules/_remote_debugging/module.c | 24 +++-- 6 files changed, 199 insertions(+), 32 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-06-20-12-44.gh-issue-161502.rYZtwK.rst diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 6bc53f4d7d3e5ff..1ee4a471ba5bb00 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -157,12 +157,13 @@ def get_all_awaited_by(pid): return unwinder.get_all_awaited_by() -def build_async_tree(result, task_emoji="(T)", cor_emoji=""): +def build_async_tree(result, task_emoji="(T)", cor_emoji="", fut_emoji="(F)"): """ Build a list of strings for pretty-print an async call tree. The call tree is produced by `get_all_async_stacks()`, prefixing tasks - with `task_emoji` and coroutine frames with `cor_emoji`. + with `task_emoji`, futures with `fut_emoji`, and coroutine frames with + `cor_emoji`. """ id2name, awaits, task_stacks = _index(result) g = _task_graph(awaits) @@ -170,9 +171,17 @@ def build_async_tree(result, task_emoji="(T)", cor_emoji=""): if cycles: raise CycleFoundException(cycles, id2name) labels, children = _build_tree(id2name, awaits, task_stacks) + future_ids = { + node.task_id + for thread in result + for node in thread.awaited_by + if node.is_future + } def pretty(node): flag = task_emoji if node[0] == NodeType.TASK else cor_emoji + if node[0] == NodeType.TASK and node[1] in future_ids: + flag = fut_emoji return f"{flag} {labels[node]}" def render(root): diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index d9cd7ee72fd2b22..47be0f2286e792c 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -608,6 +608,73 @@ async def main(): stacks = self._extract_coroutine_funcnames(asyncio.run(main())) self.assertEqual(stacks["anext"], ["agen"]) + @skip_if_not_supported + def test_future_between_tasks(self): + # gh-161502 + from asyncio.tools import build_async_tree + + async def worker(method): + return getattr(RemoteUnwinder(os.getpid()), method)() + + async def main(method): + parent = asyncio.current_task() + parent.set_name("main") + future = asyncio.Future() + task = asyncio.create_task(worker(method), name="worker") + asyncio.future_add_to_awaited_by(task, future) + task.add_done_callback(future.set_result) + try: + await future + return task.result(), { + id(parent): [], + id(future): [id(parent)], + id(task): [id(future)], + } + finally: + asyncio.future_discard_from_awaited_by(task, future) + + for method in ("get_all_awaited_by", "get_async_stack_trace"): + with self.subTest(method=method): + snapshot, expected = asyncio.run(main(method)) + tree, = build_async_tree(snapshot) + rendered = "\n".join(tree) + self.assertIn("(T) main", rendered) + self.assertIn("(F) Future-", rendered) + self.assertIn("(T) worker", rendered) + relationships = { + node.task_id: [edge.task_name for edge in node.awaited_by] + for thread in snapshot for node in thread.awaited_by + } + self.assertEqual(relationships, expected) + + @skip_if_not_supported + def test_shared_future_waiter(self): + async def main(): + layers = [[asyncio.current_task()]] + for _ in range(14): + waiters = [asyncio.Future(), asyncio.Future()] + for child in layers[-1]: + for waiter in waiters: + asyncio.future_add_to_awaited_by(child, waiter) + self.addCleanup(asyncio.future_discard_from_awaited_by, + child, waiter) + layers.append(waiters) + + leaf, root = layers[-1][0], layers[0][0] + asyncio.future_add_to_awaited_by(leaf, root) + self.addCleanup(asyncio.future_discard_from_awaited_by, leaf, root) + + expected_ids = [id(node) for layer in layers for node in layer] + unwinder = RemoteUnwinder(os.getpid()) + for method in (unwinder.get_all_awaited_by, + unwinder.get_async_stack_trace): + with self.subTest(method=method.__name__): + ids = [node.task_id for thread in method() + for node in thread.awaited_by] + self.assertCountEqual(ids, expected_ids) + + asyncio.run(main()) + @skip_if_not_supported def test_async_generator_running_task_stack(self): # gh-159620 @@ -4783,6 +4850,37 @@ def _assert_unwinder_limit_error(self, unwind, expected_substring): f"{last_error!r}" ) + @skip_if_not_supported + def test_get_all_awaited_by_at_waiter_limit(self): + # gh-161502 + async def main(): + task = asyncio.current_task() + limit = self.TASK_WAITER_WALK_TASKS - 1 + for _ in range(limit): + future = asyncio.Future() + asyncio.future_add_to_awaited_by(task, future) + self.addCleanup(asyncio.future_discard_from_awaited_by, task, future) + + release = asyncio.Event() + child = asyncio.create_task(release.wait()) + waiter = asyncio.create_task(asyncio.wait([child]), eager_start=True) + unwinder = RemoteUnwinder(os.getpid()) + snapshot = unwinder.get_all_awaited_by() + count = sum(node.is_future for thread in snapshot + for node in thread.awaited_by) + self.assertEqual(count, limit) + + extra = asyncio.Future() + asyncio.future_add_to_awaited_by(task, extra) + self.addCleanup(asyncio.future_discard_from_awaited_by, task, extra) + with self.assertRaisesRegex(RuntimeError, "Too many task waiters"): + unwinder.get_all_awaited_by() + + release.set() + await waiter + + asyncio.run(main()) + @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, diff --git a/Misc/NEWS.d/next/Library/2026-11-06-20-12-44.gh-issue-161502.rYZtwK.rst b/Misc/NEWS.d/next/Library/2026-11-06-20-12-44.gh-issue-161502.rYZtwK.rst new file mode 100644 index 000000000000000..f284c37bd738945 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-06-20-12-44.gh-issue-161502.rYZtwK.rst @@ -0,0 +1,3 @@ +Fix missing tasks in ``python -m asyncio pstree`` output when tasks are +connected through an intermediate :class:`asyncio.Future`. Patch by Timofei +Ivankov diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index 787f7921d7d74aa..04c0e0ba0200f59 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -534,10 +534,11 @@ parse_task_awaited_by( uintptr_t task_address, PyObject *awaited_by ) { - // Read the entire TaskObj at once char task_obj[SIZEOF_TASK_OBJ]; + size_t size = unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by_is_set + + sizeof(char); if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, - (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, + size, task_obj) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); return -1; @@ -571,7 +572,16 @@ process_single_task_node( PyObject *result_item = NULL; PyObject *coroutine_stack = NULL; - tn = parse_task_name(unwinder, task_addr); + char is_task; + // task_is_task = 1 => task + // task_is_task = 0 => future + if (read_char(unwinder, + task_addr + unwinder->async_debug_offsets.asyncio_task_object.task_is_task, + &is_task) < 0) { + goto error; + } + tn = is_task ? parse_task_name(unwinder, task_addr) + : PyUnicode_FromFormat("Future-%p", (void *)task_addr); if (tn == NULL) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task name in single task node"); goto error; @@ -612,6 +622,7 @@ process_single_task_node( PyStructSequence_SetItem(result_item, 1, tn); // steals ref PyStructSequence_SetItem(result_item, 2, coroutine_stack); // steals ref PyStructSequence_SetItem(result_item, 3, current_awaited_by); // steals ref + PyStructSequence_SetItem(result_item, 4, PyBool_FromLong(!is_task)); // References transferred to tuple task_id = NULL; @@ -652,37 +663,68 @@ process_single_task_node( static int process_task_waiters( RemoteUnwinderObject *unwinder, - PyObject *result + PyObject *result, + bool futures_only ) { + Py_ssize_t initial_count = futures_only ? PyList_GET_SIZE(result) : 0; + PyObject *visited = PySet_New(NULL); + if (visited == NULL) { + return -1; + } for (Py_ssize_t i = 0; i < PyList_GET_SIZE(result); i++) { PyObject *task_info = PyList_GET_ITEM(result, i); + if (PySet_Add(visited, PyStructSequence_GET_ITEM(task_info, 0)) < 0) { + goto error; + } PyObject *waiters = PyStructSequence_GET_ITEM(task_info, 3); for (Py_ssize_t j = 0; j < PyList_GET_SIZE(waiters); j++) { - if (PyList_GET_SIZE(result) >= MAX_TASK_WAITER_WALK_TASKS) { - PyErr_SetString(PyExc_RuntimeError, - "Too many task waiters (possible infinite loop)"); - set_exception_cause(unwinder, PyExc_RuntimeError, - "Task waiter walk size limit exceeded"); - return -1; - } PyObject *waiter = PyList_GET_ITEM(waiters, j); - // CoroInfo item 1 holds the waiter task address stored by parse_task(). PyObject *task_id = PyStructSequence_GET_ITEM(waiter, 1); + int seen = PySet_Contains(visited, task_id); + if (seen > 0) { + continue; + } + if (seen < 0 || PySet_Add(visited, task_id) < 0) { + goto error; + } void *task_ptr = PyLong_AsVoidPtr(task_id); if (task_ptr == NULL && PyErr_Occurred()) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse waiter task ID"); - return -1; + goto error; + } + if (futures_only) { + char is_task; + if (read_char(unwinder, + (uintptr_t)task_ptr + unwinder->async_debug_offsets.asyncio_task_object.task_is_task, + &is_task) < 0) { + goto error; + } + if (is_task) { + continue; + } + } + if (PyList_GET_SIZE(result) - initial_count >= MAX_TASK_WAITER_WALK_TASKS) { + PyErr_SetString(PyExc_RuntimeError, + "Too many task waiters (possible infinite loop)"); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Task waiter walk size limit exceeded"); + goto error; } if (process_single_task_node( unwinder, (uintptr_t)task_ptr, NULL, result) < 0) { - return -1; + goto error; } } } + Py_DECREF(visited); return 0; + +error: + Py_DECREF(visited); + return -1; } /* ============================================================================ @@ -927,7 +969,7 @@ append_awaited_by( return -1; } - return 0; + return process_task_waiters(unwinder, task_list, true); } /* ============================================================================ @@ -991,7 +1033,7 @@ process_running_task_chain( } // Now find all tasks that are waiting for this task and process them - if (process_task_waiters(unwinder, result) < 0) { + if (process_task_waiters(unwinder, result, false) < 0) { return -1; } diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index 28e5a3ca32ac803..26e4a3c2c2e84be 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -295,14 +295,17 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__, "\n" " - thread_id (int): Identifier of the thread, or 0 for tasks in the\n" " interpreter\'s fallback task list.\n" -" - awaited_by (list[TaskInfo]): Tasks registered with this thread.\n" +" - awaited_by (list[TaskInfo]): Tasks registered with this thread and\n" +" intermediate futures in their awaited_by chains.\n" "\n" "Each TaskInfo contains:\n" " - task_id (int): Identifier of the task.\n" -" - task_name (str): Name of the task.\n" +" - task_name (str): Name of the task, or a generated name for\n" +" a future.\n" " - coroutine_stack (list[CoroInfo]): Stack of coroutine frames.\n" " - awaited_by (list[CoroInfo]): Coroutine information for tasks or futures\n" " awaiting this task.\n" +" - is_future (bool, named-only): True for a plain Future.\n" "\n" "Each CoroInfo contains:\n" " - call_stack (list[FrameInfo]): Call stack frames for the coroutine.\n" @@ -367,14 +370,16 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, "\n" " - thread_id (int): Identifier of the thread.\n" " - awaited_by (list[TaskInfo]): The running task and all its\n" -" transitive waiters.\n" +" transitive waiters, including intermediate futures.\n" "\n" "Each TaskInfo contains:\n" " - task_id (int): Identifier of the task.\n" -" - task_name (str): Name of the task.\n" +" - task_name (str): Name of the task, or a generated name for\n" +" a future.\n" " - coroutine_stack (list[CoroInfo]): Stack of coroutine frames.\n" " - awaited_by (list[CoroInfo]): Coroutine information for\n" " tasks or futures awaiting this task.\n" +" - is_future (bool, named-only): True for a plain Future.\n" "\n" "Each CoroInfo contains:\n" " - call_stack (list[FrameInfo]): Frames for the coroutine.\n" @@ -408,7 +413,9 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, " coroutine_stack=[\n" " CoroInfo(call_stack=[...], task_name=1)\n" " ],\n" -" awaited_by=[CoroInfo(call_stack=[...], task_name=2)]\n" +" awaited_by=[\n" +" CoroInfo(call_stack=[...], task_name=2)\n" +" ]\n" " ),\n" " TaskInfo(\n" " task_id=2,\n" @@ -1740,4 +1747,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=ef0d1eaac8262c2c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2be4cb3f27f6c8d8 input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index c77ff9ae0abd5b5..2031fe0364f4eb9 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -34,12 +34,13 @@ static PyStructSequence_Field TaskInfo_fields[] = { {"task_name", "Task name"}, {"coroutine_stack", "Coroutine call stack"}, {"awaited_by", "Tasks awaiting this task"}, + {"is_future", "Whether this node is a plain Future rather than a Task"}, {NULL} }; PyStructSequence_Desc TaskInfo_desc = { "_remote_debugging.TaskInfo", - "Information about an asyncio task", + "Information about an asyncio task or future", TaskInfo_fields, 4 }; @@ -887,14 +888,17 @@ Get awaited_by relationships for tasks in the remote process. - thread_id (int): Identifier of the thread, or 0 for tasks in the interpreter's fallback task list. - - awaited_by (list[TaskInfo]): Tasks registered with this thread. + - awaited_by (list[TaskInfo]): Tasks registered with this thread and + intermediate futures in their awaited_by chains. Each TaskInfo contains: - task_id (int): Identifier of the task. - - task_name (str): Name of the task. + - task_name (str): Name of the task, or a generated name for + a future. - coroutine_stack (list[CoroInfo]): Stack of coroutine frames. - awaited_by (list[CoroInfo]): Coroutine information for tasks or futures awaiting this task. + - is_future (bool, named-only): True for a plain Future. Each CoroInfo contains: - call_stack (list[FrameInfo]): Call stack frames for the coroutine. @@ -925,7 +929,7 @@ Example output: static PyObject * _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=6a49cd345e8aec53 input=79e556b3973d21a8]*/ +/*[clinic end generated code: output=6a49cd345e8aec53 input=17fcd3d7bb835624]*/ { if (ensure_async_debug_offsets(self) < 0) { return NULL; @@ -988,14 +992,16 @@ running task, tasks waiting for those tasks, etc.). - thread_id (int): Identifier of the thread. - awaited_by (list[TaskInfo]): The running task and all its - transitive waiters. + transitive waiters, including intermediate futures. Each TaskInfo contains: - task_id (int): Identifier of the task. - - task_name (str): Name of the task. + - task_name (str): Name of the task, or a generated name for + a future. - coroutine_stack (list[CoroInfo]): Stack of coroutine frames. - awaited_by (list[CoroInfo]): Coroutine information for tasks or futures awaiting this task. + - is_future (bool, named-only): True for a plain Future. Each CoroInfo contains: - call_stack (list[FrameInfo]): Frames for the coroutine. @@ -1029,7 +1035,9 @@ Example output: coroutine_stack=[ CoroInfo(call_stack=[...], task_name=1) ], - awaited_by=[CoroInfo(call_stack=[...], task_name=2)] + awaited_by=[ + CoroInfo(call_stack=[...], task_name=2) + ] ), TaskInfo( task_id=2, @@ -1046,7 +1054,7 @@ Example output: static PyObject * _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=6433d52b55e87bbe input=0532031a7c495491]*/ +/*[clinic end generated code: output=6433d52b55e87bbe input=a95958816358476a]*/ { if (ensure_async_debug_offsets(self) < 0) { return NULL; From dc222c8d6c366e3d52d4b0d0053ad7bb8a601fff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maurycy=20Paw=C5=82owski-Wiero=C5=84ski?= Date: Sat, 21 Nov 2026 02:29:36 +0100 Subject: [PATCH 229/229] gh-162246: Fix lost writes on `BufferedRWPair` dealloc (#162295) `socket.makefile("rwb")` passes the same raw stream to both sides of a `BufferedRWPair`. When the pair is deallocated, the reader closes the stream before the writer can flush its buffer. Release the writer first, matching the order in `BufferedRWPair.close()`. --- Lib/test/test_io/test_bufferedio.py | 9 +++++++++ .../2026-11-19-22-01-29.gh-issue-162246.y3L8nP.rst | 2 ++ Modules/_io/bufferedio.c | 3 ++- 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-19-22-01-29.gh-issue-162246.y3L8nP.rst diff --git a/Lib/test/test_io/test_bufferedio.py b/Lib/test/test_io/test_bufferedio.py index 64789821afe59c2..997e335591b483b 100644 --- a/Lib/test/test_io/test_bufferedio.py +++ b/Lib/test/test_io/test_bufferedio.py @@ -1116,6 +1116,15 @@ def test_write(self): pair.flush() self.assertEqual(w._write_stack, [b"abc", b"def"]) + def test_destructor_flushes_shared_stream(self): + raw = self.MockRawIO() + pair = self.tp(raw, raw) + pair.write(b"abc") + self.assertEqual(raw._write_stack, []) + del pair + support.gc_collect() + self.assertEqual(raw._write_stack, [b"abc"]) + def test_peek(self): pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) diff --git a/Misc/NEWS.d/next/Library/2026-11-19-22-01-29.gh-issue-162246.y3L8nP.rst b/Misc/NEWS.d/next/Library/2026-11-19-22-01-29.gh-issue-162246.y3L8nP.rst new file mode 100644 index 000000000000000..702b38215ed6efb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-19-22-01-29.gh-issue-162246.y3L8nP.rst @@ -0,0 +1,2 @@ +Fix :class:`io.BufferedRWPair` losing buffered writes on deallocation when the +reader and writer share a raw stream. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 1e900084b549f4f..f585bd8483370fe 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -2318,8 +2318,9 @@ static int bufferedrwpair_clear(PyObject *op) { rwpair *self = rwpair_CAST(op); - Py_CLEAR(self->reader); + /* Let the writer flush before the reader closes a shared raw stream. */ Py_CLEAR(self->writer); + Py_CLEAR(self->reader); PyObject_ClearManagedDict(op); return 0; }