From fd470f9203dd1c5175cfc973dae0e21d9a4dc746 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 17 Nov 2026 12:05:51 -0800 Subject: [PATCH 1/6] gh-162150: Use full PyLongObject space in freelist The freelist was hardcoded to only allow one digit ints. `ob_digit` is the last member and `sizeof(PyLongObject)` actually has space for two on 64 bit machines. Refactor to add a MAX_DIGITS_FOR_FREELIST which computes what fits and use the freelist for all PyLongObject. For 64 bit platforms that means 2-digit `int` now use the freelist while taking no extra space. Each digit stores 30 bits so this still doesn't reach platform integer length on 64 bit (2 * 30 < 64). Adding another digit would change the allocation class unfortunately so need other approaches. PGO+LTO, upstream/main b90572af8f9 vs this change, mean ns: | statement | main | branch | delta | |-----------------------------------|-------:|-------:|--------:| | `s + s` (1 digit + 1 digit -> 2) | 17.2 | 13.7 | -3.4 | | `s * t` (1 digit * 1 digit -> 2) | 16.8 | 13.6 | -3.2 | | `a - b` (2 digits) | 16.9 | 14.0 | -2.8 | | `a // 7` | 20.8 | 17.7 | -3.1 | | `a & b` | 16.1 | 13.1 | -2.9 | | `a >> 3` | 17.6 | 14.6 | -3.0 | | `id(m)` | 17.2 | 14.5 | -2.7 | | `int('1234567890123')` | 39.2 | 34.6 | -4.6 | | `time.time_ns()` | 35.1 | 30.8 | -4.4 | | `time.perf_counter_ns()` | 34.9 | 29.3 | -5.6 | | `sum(range(2**31, 2**31 + 1000))` | 9189.7 | 6125.9 | -3063.8 | | `str(5)` | 25.1 | 21.5 | -3.5 | | `str(123456789)` | 29.8 | 26.8 | -3.0 | | `f'{n}'` (1 digit) | 30.0 | 25.6 | -4.4 | | `'%d' % n` | 50.2 | 46.0 | -4.3 | | `b'%d' % n` | 38.2 | 35.3 | -2.9 | | `a + b` (allocates 3 digits) | 16.9 | 15.8 | -1.1 | | `a * 3` (allocates 3 digits) | 20.9 | 21.0 | +0.2 | | `a << 1` (allocates 3 digits) | 19.1 | 18.3 | -0.9 | | `str(2**62)` (3-digit scratch) | 35.0 | 35.2 | +0.2 | | `m + k` (1 digit, control) | 9.6 | 9.0 | -0.5 | | `c + d` (3 digits, control) | 16.0 | 16.6 | +0.7 | | `str(10**40)` (control) | 113.6 | 113.0 | -0.6 | pyperformance, one pass per side: | Benchmark | main | branch | |------------------|:-------:|:---------------------:| | crypto_pyaes | 34.5 ms | 31.3 ms: 1.10x faster | | mako | 6.26 ms | 6.05 ms: 1.03x faster | | json_dumps | 4.33 ms | 4.21 ms: 1.03x faster | | pprint_safe_repr | 364 ms | 353 ms: 1.03x faster | | pprint_pformat | 739 ms | 721 ms: 1.02x faster | | deepcopy | 110 us | 109 us: 1.01x faster | | pathlib | 9.12 ms | 9.26 ms: 1.02x slower | | chameleon | 4.60 ms | 4.69 ms: 1.02x slower | | telco (control) | 3.49 ms | 3.38 ms: 1.03x faster | | pyflate (control) | 205 ms | 199 ms: 1.03x faster | | richards (control) | 21.1 ms | 20.8 ms: 1.01x faster | | spectral_norm (control) | 45.1 ms | 44.7 ms: 1.01x faster | | pidigits (control) | 109 ms | 110 ms: 1.01x slower | | tomli_loads (control) | not significant | | Co-Authored-By: Claude Fable 5.2 --- ...1-17-12-00-00.gh-issue-162150.F5c0cFN9.rst | 2 ++ Objects/longobject.c | 32 +++++++++++++------ 2 files changed, 24 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-11-17-12-00-00.gh-issue-162150.F5c0cFN9.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-11-17-12-00-00.gh-issue-162150.F5c0cFN9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-17-12-00-00.gh-issue-162150.F5c0cFN9.rst new file mode 100644 index 000000000000000..4c20f8aee348509 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-11-17-12-00-00.gh-issue-162150.F5c0cFN9.rst @@ -0,0 +1,2 @@ +Improve :class:`int` performance on 64-bit builds by allowing up to 60-bit ints, +which fit in existing spare space, to use the freelist. diff --git a/Objects/longobject.c b/Objects/longobject.c index 09bfe4ad4641281..ee2e57e96e3d24d 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -155,6 +155,14 @@ long_normalize(PyLongObject *v) # define MAX_LONG_DIGITS ((INT64_MAX-1) / PyLong_SHIFT) #endif +/* The freelist holds blocks of sizeof(PyLongObject) bytes: the padding + after ob_digit[0] can make room for more than one digit. */ +#define MAX_DIGITS_FOR_FREELIST \ + ((Py_ssize_t)((sizeof(PyLongObject) - \ + offsetof(PyLongObject, long_value.ob_digit)) / sizeof(digit))) +static_assert(MAX_DIGITS_FOR_FREELIST >= 1, + "int freelist blocks must hold at least one digit"); + static PyLongObject * long_alloc(Py_ssize_t size) { @@ -169,17 +177,19 @@ long_alloc(Py_ssize_t size) * assume that there is always at least one digit present. */ Py_ssize_t ndigits = size ? size : 1; - if (ndigits == 1) { + /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + + sizeof(digit)*size. Previous incarnations of this code used + sizeof() instead of the offsetof, but this risks being + incorrect in the presence of padding between the header + and the digits. */ + size_t nbytes = (offsetof(PyLongObject, long_value.ob_digit) + + ndigits*sizeof(digit)); + if (ndigits <= MAX_DIGITS_FOR_FREELIST) { + nbytes = sizeof(PyLongObject); result = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); } if (result == NULL) { - /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + - sizeof(digit)*size. Previous incarnations of this code used - sizeof() instead of the offsetof, but this risks being - incorrect in the presence of padding between the header - and the digits. */ - result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) + - ndigits*sizeof(digit)); + result = PyObject_Malloc(nbytes); if (!result) { PyErr_NoMemory(); return NULL; @@ -3638,7 +3648,7 @@ _PyLong_ExactDealloc(PyObject *self) _Py_SetImmortal(self); return; } - if (_PyLong_IsCompact((PyLongObject *)self)) { + if (_PyLong_DigitCount((PyLongObject *)self) <= MAX_DIGITS_FOR_FREELIST) { _Py_FREELIST_FREE(ints, self, PyObject_Free); return; } @@ -3658,7 +3668,9 @@ long_dealloc(PyObject *self) _Py_SetImmortal(self); return; } - if (PyLong_CheckExact(self) && _PyLong_IsCompact((PyLongObject *)self)) { + if (PyLong_CheckExact(self) + && _PyLong_DigitCount((PyLongObject *)self) <= MAX_DIGITS_FOR_FREELIST) + { _Py_FREELIST_FREE(ints, self, PyObject_Free); return; } From 26ee13a643dbdc3971278f7f566f0431827f129b Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 17 Nov 2026 12:31:41 -0800 Subject: [PATCH 2/6] Improve phrasing --- Objects/longobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index ee2e57e96e3d24d..3a675b2dbeefa1d 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -156,7 +156,7 @@ long_normalize(PyLongObject *v) #endif /* The freelist holds blocks of sizeof(PyLongObject) bytes: the padding - after ob_digit[0] can make room for more than one digit. */ + after ob_digit[0] can have space for more than one digit. */ #define MAX_DIGITS_FOR_FREELIST \ ((Py_ssize_t)((sizeof(PyLongObject) - \ offsetof(PyLongObject, long_value.ob_digit)) / sizeof(digit))) From 08daafde1fc16592f2a01c6b8c7df607e79cfc67 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 17 Nov 2026 12:48:49 -0800 Subject: [PATCH 3/6] Move static_assert into function to pass checker --- Objects/longobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 3a675b2dbeefa1d..e2b54132cf202de 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -160,12 +160,12 @@ long_normalize(PyLongObject *v) #define MAX_DIGITS_FOR_FREELIST \ ((Py_ssize_t)((sizeof(PyLongObject) - \ offsetof(PyLongObject, long_value.ob_digit)) / sizeof(digit))) -static_assert(MAX_DIGITS_FOR_FREELIST >= 1, - "int freelist blocks must hold at least one digit"); static PyLongObject * long_alloc(Py_ssize_t size) { + static_assert(MAX_DIGITS_FOR_FREELIST >= 1, + "int freelist blocks must hold at least one digit"); assert(size >= 0); PyLongObject *result = NULL; if (size > (Py_ssize_t)MAX_LONG_DIGITS) { From 4d6cddc47b20d2842ac63897e45787310e3b4c45 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 17 Nov 2026 13:31:54 -0800 Subject: [PATCH 4/6] Keep original malloc in long_alloc; explain freelist sizing Co-Authored-By: Claude Opus 5.5 --- Objects/longobject.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index e2b54132cf202de..c7d87ea02d8eff0 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -177,19 +177,22 @@ long_alloc(Py_ssize_t size) * assume that there is always at least one digit present. */ Py_ssize_t ndigits = size ? size : 1; - /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + - sizeof(digit)*size. Previous incarnations of this code used - sizeof() instead of the offsetof, but this risks being - incorrect in the presence of padding between the header - and the digits. */ - size_t nbytes = (offsetof(PyLongObject, long_value.ob_digit) + - ndigits*sizeof(digit)); + Py_ssize_t alloc_digits = ndigits; if (ndigits <= MAX_DIGITS_FOR_FREELIST) { - nbytes = sizeof(PyLongObject); result = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints); + /* The block may later go on the freelist and be reused for any + int with up to MAX_DIGITS_FOR_FREELIST digits, so allocate room + for that many digits, not just ndigits. */ + alloc_digits = MAX_DIGITS_FOR_FREELIST; } if (result == NULL) { - result = PyObject_Malloc(nbytes); + /* Number of bytes needed is: offsetof(PyLongObject, ob_digit) + + sizeof(digit)*size. Previous incarnations of this code used + sizeof() instead of the offsetof, but this risks being + incorrect in the presence of padding between the header + and the digits. */ + result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) + + alloc_digits*sizeof(digit)); if (!result) { PyErr_NoMemory(); return NULL; From 324b0d8b5ced01580305e7ba8c5d4b0abe9026f5 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 19 Nov 2026 14:46:34 -0800 Subject: [PATCH 5/6] Check there is enough space to write MAX_DIGITS_FOR_FREELIST memset the space which will result in python debug allocator and ASAN failures. There's no cheap way without a lot of new code to see exactly how much space was actually allocated. This makes too small of Objects getting in reasonably noisy --- Objects/longobject.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 50cd3fc5ff32533..6f486cc9378d43e 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -166,6 +166,8 @@ long_alloc(Py_ssize_t size) { static_assert(MAX_DIGITS_FOR_FREELIST >= 1, "int freelist blocks must hold at least one digit"); + // ints from tp_alloc must be big enough for the freelist too. + assert(_PyObject_VAR_SIZE(&PyLong_Type, 1) >= sizeof(PyLongObject)); assert(size >= 0); PyLongObject *result = NULL; if (size > (Py_ssize_t)MAX_LONG_DIGITS) { @@ -204,7 +206,7 @@ long_alloc(Py_ssize_t size) #ifdef Py_DEBUG // gh-147988: Fill digits with an invalid pattern to catch usage // of uninitialized digits. - memset(result->long_value.ob_digit, 0xFF, ndigits * sizeof(digit)); + memset(result->long_value.ob_digit, 0xFF, alloc_digits * sizeof(digit)); #endif return result; } @@ -3652,6 +3654,12 @@ _PyLong_ExactDealloc(PyObject *self) return; } if (_PyLong_DigitCount((PyLongObject *)self) <= MAX_DIGITS_FOR_FREELIST) { +#if defined(Py_DEBUG) || defined(_Py_ADDRESS_SANITIZER) + // Validate on debug and ASAN builds we always have + // MAX_DIGITS_FOR_FREELIST space writeable + memset(((PyLongObject *)self)->long_value.ob_digit, 0xFF, + MAX_DIGITS_FOR_FREELIST * sizeof(digit)); +#endif _Py_FREELIST_FREE(ints, self, PyObject_Free); return; } @@ -3674,6 +3682,12 @@ long_dealloc(PyObject *self) if (PyLong_CheckExact(self) && _PyLong_DigitCount((PyLongObject *)self) <= MAX_DIGITS_FOR_FREELIST) { +#if defined(Py_DEBUG) || defined(_Py_ADDRESS_SANITIZER) + // Validate on debug and ASAN builds we always have + // MAX_DIGITS_FOR_FREELIST space writeable + memset(((PyLongObject *)self)->long_value.ob_digit, 0xFF, + MAX_DIGITS_FOR_FREELIST * sizeof(digit)); +#endif _Py_FREELIST_FREE(ints, self, PyObject_Free); return; } From 0ddb67d9052ab806230e64ab4ef93841e676999d Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 19 Nov 2026 15:00:33 -0800 Subject: [PATCH 6/6] Say why the int freelist fill is there Co-Authored-By: Claude Fable 5.2 --- Objects/longobject.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 6f486cc9378d43e..2f8a152b060828a 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3655,8 +3655,7 @@ _PyLong_ExactDealloc(PyObject *self) } if (_PyLong_DigitCount((PyLongObject *)self) <= MAX_DIGITS_FOR_FREELIST) { #if defined(Py_DEBUG) || defined(_Py_ADDRESS_SANITIZER) - // Validate on debug and ASAN builds we always have - // MAX_DIGITS_FOR_FREELIST space writeable + // Use allocator to detect allocations that are too small. memset(((PyLongObject *)self)->long_value.ob_digit, 0xFF, MAX_DIGITS_FOR_FREELIST * sizeof(digit)); #endif @@ -3683,8 +3682,7 @@ long_dealloc(PyObject *self) && _PyLong_DigitCount((PyLongObject *)self) <= MAX_DIGITS_FOR_FREELIST) { #if defined(Py_DEBUG) || defined(_Py_ADDRESS_SANITIZER) - // Validate on debug and ASAN builds we always have - // MAX_DIGITS_FOR_FREELIST space writeable + // Use allocator to detect allocations that are too small. memset(((PyLongObject *)self)->long_value.ob_digit, 0xFF, MAX_DIGITS_FOR_FREELIST * sizeof(digit)); #endif