Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Improve :class:`int` performance on 64-bit builds by allowing up to 60-bit ints,
which fit in existing spare space, to use the freelist.
37 changes: 32 additions & 5 deletions Objects/longobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,19 @@ long_normalize(PyLongObject *v)
# define MAX_LONG_DIGITS ((INT64_MAX-1) / PyLong_SHIFT)
#endif

/* The freelist holds blocks of sizeof(PyLongObject) bytes: the padding
after ob_digit[0] can have space for more than one digit. */
#define MAX_DIGITS_FOR_FREELIST \
((Py_ssize_t)((sizeof(PyLongObject) - \
offsetof(PyLongObject, long_value.ob_digit)) / sizeof(digit)))

static PyLongObject *
long_alloc(Py_ssize_t size)
{
static_assert(MAX_DIGITS_FOR_FREELIST >= 1,
"int freelist blocks must hold at least one digit");
// ints from tp_alloc must be big enough for the freelist too.
assert(_PyObject_VAR_SIZE(&PyLong_Type, 1) >= sizeof(PyLongObject));
assert(size >= 0);
PyLongObject *result = NULL;
if (size > (Py_ssize_t)MAX_LONG_DIGITS) {
Expand All @@ -169,8 +179,13 @@ long_alloc(Py_ssize_t size)
* assume that there is always at least one digit present. */
Py_ssize_t ndigits = size ? size : 1;

if (ndigits == 1) {
Py_ssize_t alloc_digits = ndigits;
if (ndigits <= MAX_DIGITS_FOR_FREELIST) {
result = (PyLongObject *)_Py_FREELIST_POP(PyLongObject, ints);
/* The block may later go on the freelist and be reused for any
int with up to MAX_DIGITS_FOR_FREELIST digits, so allocate room
for that many digits, not just ndigits. */
alloc_digits = MAX_DIGITS_FOR_FREELIST;
}
if (result == NULL) {
/* Number of bytes needed is: offsetof(PyLongObject, ob_digit) +
Expand All @@ -179,7 +194,7 @@ long_alloc(Py_ssize_t size)
incorrect in the presence of padding between the header
and the digits. */
result = PyObject_Malloc(offsetof(PyLongObject, long_value.ob_digit) +
ndigits*sizeof(digit));
alloc_digits*sizeof(digit));
if (!result) {
PyErr_NoMemory();
return NULL;
Expand All @@ -191,7 +206,7 @@ long_alloc(Py_ssize_t size)
#ifdef Py_DEBUG
// gh-147988: Fill digits with an invalid pattern to catch usage
// of uninitialized digits.
memset(result->long_value.ob_digit, 0xFF, ndigits * sizeof(digit));
memset(result->long_value.ob_digit, 0xFF, alloc_digits * sizeof(digit));
#endif
return result;
}
Expand Down Expand Up @@ -3638,7 +3653,12 @@ _PyLong_ExactDealloc(PyObject *self)
_Py_SetImmortal(self);
return;
}
if (_PyLong_IsCompact((PyLongObject *)self)) {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this should actually be ==. While it's unlikely on 64 bit someone appears with a 1-digit PyLongObject after this change keeping to only one size of possible object in the freelist seems good

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Having PyLong_DigitCount((PyLongObject *)self) < MAX_DIGITS_FOR_FREELIST should not indeed not happen (could happen for subclasses, but here we are in _PyLong_ExactDealloc).

Add an assert that _PyLong_DigitCount((PyLongObject *)self) >= MAX_DIGITS_FOR_FREELIST). (the check can then be either <= or ==

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There isn't a good way to get "capacity" unfortunately without adding some significant hooks to the allocator that I don't think are worth it for this safety.

Moved so in debug builds and with ASAN enabled we write to the required freelist capacity which should help.

The check stays <= because PyLongObject ndigits can be shrunk during usage, but we should always have sizeof(PyLongObject) min space.

if (_PyLong_DigitCount((PyLongObject *)self) <= MAX_DIGITS_FOR_FREELIST) {
#if defined(Py_DEBUG) || defined(_Py_ADDRESS_SANITIZER)
// Use allocator to detect allocations that are too small.
memset(((PyLongObject *)self)->long_value.ob_digit, 0xFF,
MAX_DIGITS_FOR_FREELIST * sizeof(digit));
#endif
_Py_FREELIST_FREE(ints, self, PyObject_Free);
return;
}
Expand All @@ -3658,7 +3678,14 @@ long_dealloc(PyObject *self)
_Py_SetImmortal(self);
return;
}
if (PyLong_CheckExact(self) && _PyLong_IsCompact((PyLongObject *)self)) {
if (PyLong_CheckExact(self)
&& _PyLong_DigitCount((PyLongObject *)self) <= MAX_DIGITS_FOR_FREELIST)
{
#if defined(Py_DEBUG) || defined(_Py_ADDRESS_SANITIZER)
// Use allocator to detect allocations that are too small.
memset(((PyLongObject *)self)->long_value.ob_digit, 0xFF,
MAX_DIGITS_FOR_FREELIST * sizeof(digit));
#endif
_Py_FREELIST_FREE(ints, self, PyObject_Free);
return;
}
Expand Down
Loading