Skip to content

Commit 8017acf

Browse files
[3.14] gh-160978: Return the same ctypes array type when created concurrently (GH-161494) (#162313)
gh-160978: Return the same ctypes array type when created concurrently (GH-161494) (cherry picked from commit c0f3fcf) Co-authored-by: Thales Moura <thalesmoura13@gmail.com>
1 parent 7738969 commit 8017acf

3 files changed

Lines changed: 131 additions & 18 deletions

File tree

‎Lib/test/test_ctypes/test_arrays.py‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
import ctypes
22
import sys
3+
import threading
34
import unittest
5+
from concurrent.futures import ThreadPoolExecutor
46
from ctypes import (Structure, Array, ARRAY, sizeof, addressof,
57
create_string_buffer, create_unicode_buffer,
68
c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint,
79
c_long, c_ulonglong, c_float, c_double, c_longdouble)
8-
from test.support import bigmemtest, _2G, threading_helper, Py_GIL_DISABLED
10+
from test.support import (bigmemtest, _2G, threading_helper,
11+
Py_GIL_DISABLED, SHORT_TIMEOUT)
912
from ._support import (_CData, PyCArrayType, Py_TPFLAGS_DISALLOW_INSTANTIATION,
1013
Py_TPFLAGS_IMMUTABLETYPE)
1114

@@ -288,5 +291,49 @@ def run():
288291
raise cm.exc_value
289292

290293

294+
class ArrayTypeCacheTest(unittest.TestCase):
295+
# gh-160978: the array type cache could hand out distinct type objects for
296+
# the same (item type, length) pair when the array type was created from
297+
# more than one thread at a time.
298+
299+
@threading_helper.requires_working_threading()
300+
def test_concurrent_creation_with_set_name(self):
301+
# Creating the array type runs Python code -- ``__set_name__`` here --
302+
# which lets a second thread reach the cache before the first one has
303+
# stored its type. Without the fix this fails on the default build as
304+
# well, so it guards the fix even where threads cannot interleave
305+
# freely.
306+
barrier = threading.Barrier(2)
307+
308+
class Meta(type(c_int)):
309+
def __set_name__(cls, owner, name):
310+
barrier.wait(timeout=SHORT_TIMEOUT)
311+
312+
class Item(c_int, metaclass=Meta):
313+
pass
314+
315+
with ThreadPoolExecutor(2) as pool:
316+
first, second = pool.map(lambda _: Item * 100, range(2))
317+
318+
self.assertIs(first, second)
319+
# Raises TypeError if the two types are distinct.
320+
first.from_param(second())
321+
322+
@threading_helper.requires_working_threading()
323+
def test_concurrent_creation_returns_identical_types(self):
324+
nthreads = 8
325+
barrier = threading.Barrier(nthreads)
326+
327+
def make_type(length):
328+
barrier.wait(timeout=SHORT_TIMEOUT)
329+
return c_int * length
330+
331+
with ThreadPoolExecutor(nthreads) as pool:
332+
for length in range(1, 51):
333+
types = list(pool.map(make_type, [length] * nthreads))
334+
barrier.reset()
335+
self.assertEqual(len(set(types)), 1)
336+
337+
291338
if __name__ == '__main__':
292339
unittest.main()
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fix :mod:`ctypes` handing out distinct array types for the same item type
2+
and length when the array type is created concurrently from several
3+
threads.

‎Modules/_ctypes/_ctypes.c‎

Lines changed: 80 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,23 @@ _DictRemover_call(PyObject *myself, PyObject *args, PyObject *kw)
185185
{
186186
DictRemoverObject *self = _DictRemoverObject_CAST(myself);
187187
if (self->key && self->dict) {
188-
if (-1 == PyDict_DelItem(self->dict, self->key)) {
188+
/* Only remove the entry if it is still the proxy that died: the
189+
key may have been reused for a new object in the meantime. */
190+
PyObject *proxy = PyTuple_GET_SIZE(args) == 1
191+
? PyTuple_GET_ITEM(args, 0) : NULL;
192+
PyObject *current;
193+
int rc;
194+
Py_BEGIN_CRITICAL_SECTION(self->dict);
195+
rc = PyDict_GetItemRef(self->dict, self->key, &current);
196+
if (rc == 1) {
197+
rc = 0;
198+
if (current == proxy) {
199+
rc = PyDict_DelItem(self->dict, self->key);
200+
}
201+
Py_DECREF(current);
202+
}
203+
Py_END_CRITICAL_SECTION();
204+
if (rc < 0) {
189205
PyErr_FormatUnraisable("Exception ignored while "
190206
"calling _ctypes.DictRemover");
191207
}
@@ -214,32 +230,72 @@ static PyType_Spec dictremover_spec = {
214230
.slots = dictremover_slots,
215231
};
216232

217-
int
218-
PyDict_SetItemProxy(ctypes_state *st, PyObject *dict, PyObject *key, PyObject *item)
233+
/* Cache `item` in `dict` under `key` through a weakref proxy, unless a live
234+
entry already exists. On success *presult is a new reference to whatever
235+
is cached for `key` afterwards: `item` itself, or the object another
236+
thread cached first.
237+
Returns 0 on success, -1 with an exception set on failure. */
238+
static int
239+
_PyDict_SetDefaultItemProxy(ctypes_state *st, PyObject *dict, PyObject *key,
240+
PyObject *item, PyObject **presult)
219241
{
220-
PyObject *obj;
221-
DictRemoverObject *remover;
222-
PyObject *proxy;
223-
int result;
224-
225-
obj = _PyObject_CallNoArgs((PyObject *)st->DictRemover_Type);
226-
if (obj == NULL)
242+
PyObject *obj = _PyObject_CallNoArgs((PyObject *)st->DictRemover_Type);
243+
if (obj == NULL) {
227244
return -1;
245+
}
228246

229-
remover = (DictRemoverObject *)obj;
247+
DictRemoverObject *remover = (DictRemoverObject *)obj;
230248
assert(remover->key == NULL);
231249
assert(remover->dict == NULL);
232250
remover->key = Py_NewRef(key);
233251
remover->dict = Py_NewRef(dict);
234252

235-
proxy = PyWeakref_NewProxy(item, obj);
253+
PyObject *proxy = PyWeakref_NewProxy(item, obj);
236254
Py_DECREF(obj);
237-
if (proxy == NULL)
255+
if (proxy == NULL) {
256+
return -1;
257+
}
258+
259+
PyObject *stored;
260+
int present = PyDict_SetDefaultRef(dict, key, proxy, &stored);
261+
if (present < 0) {
262+
Py_DECREF(proxy);
238263
return -1;
264+
}
265+
if (present == 0) {
266+
/* Our proxy was inserted; `item` is the canonical object. */
267+
Py_DECREF(stored);
268+
Py_DECREF(proxy);
269+
*presult = Py_NewRef(item);
270+
return 0;
271+
}
239272

240-
result = PyDict_SetItem(dict, key, proxy);
273+
/* Another thread cached an object for this key first. Prefer it, so that
274+
every caller gets the same object. */
275+
PyObject *existing = NULL;
276+
int rc = 0;
277+
if (PyWeakref_CheckProxy(stored)) {
278+
rc = PyWeakref_GetRef(stored, &existing);
279+
}
280+
Py_DECREF(stored);
281+
if (rc < 0) {
282+
Py_DECREF(proxy);
283+
return -1;
284+
}
285+
if (existing != NULL) {
286+
Py_DECREF(proxy);
287+
*presult = existing;
288+
return 0;
289+
}
290+
291+
/* The cached entry had already been cleared; install ours instead. */
292+
rc = PyDict_SetItem(dict, key, proxy);
241293
Py_DECREF(proxy);
242-
return result;
294+
if (rc < 0) {
295+
return -1;
296+
}
297+
*presult = Py_NewRef(item);
298+
return 0;
243299
}
244300

245301
static int
@@ -5400,13 +5456,20 @@ PyCArrayType_from_ctype(ctypes_state *st, PyObject *itemtype, Py_ssize_t length)
54005456
Py_DECREF(key);
54015457
return NULL;
54025458
}
5403-
if (PyDict_SetItemProxy(st, st->array_cache, key, result) < 0) {
5459+
PyObject *cached;
5460+
int rc;
5461+
Py_BEGIN_CRITICAL_SECTION(st->array_cache);
5462+
rc = _PyDict_SetDefaultItemProxy(st, st->array_cache, key, result,
5463+
&cached);
5464+
Py_END_CRITICAL_SECTION();
5465+
if (rc < 0) {
54045466
Py_DECREF(key);
54055467
Py_DECREF(result);
54065468
return NULL;
54075469
}
54085470
Py_DECREF(key);
5409-
return result;
5471+
Py_DECREF(result);
5472+
return cached;
54105473
}
54115474

54125475

0 commit comments

Comments
 (0)