Skip to content

Commit 40a7b64

Browse files
committed
gh-162273: Acquire strong reference to fillvalue upfront in itertools.zip_longest
zip_longest_new() borrowed fillvalue from kwds via PyDict_GetItemWithError() and only promoted it with Py_NewRef() after calling PyObject_GetIter() on input iterables. Use PyDict_GetItemRef() upfront and decref fillvalue on error paths so clearing or mutating kwds during iterator initialization cannot free fillvalue before ownership is stored on the zip_longest object.
1 parent 4b33a9d commit 40a7b64

3 files changed

Lines changed: 61 additions & 9 deletions

File tree

‎Lib/test/test_itertools.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -908,6 +908,48 @@ def __iter__(self):
908908

909909
self.assertIs(cm.exception, exception)
910910

911+
@threading_helper.requires_working_threading()
912+
def test_zip_longest_reentrant_kwds_clear(self):
913+
# gh-162273: clearing kwds inside __iter__ must not deallocate
914+
# fillvalue before zip_longest_new acquires a strong reference.
915+
import _thread
916+
917+
events = []
918+
cleared = threading.Event()
919+
merged = threading.Event()
920+
finished = threading.Event()
921+
922+
class EphemeralFillValue:
923+
def __del__(self):
924+
events.append("DEALLOCATED")
925+
926+
class Trigger:
927+
def __init__(self, kw):
928+
self.kw = kw
929+
def __iter__(self):
930+
events.append("BEFORE_CLEAR")
931+
self.kw.clear()
932+
cleared.set()
933+
merged.wait(timeout=support.SHORT_TIMEOUT)
934+
events.append("RETURNING_FROM_GETITER")
935+
return iter([1])
936+
937+
class TailIterable:
938+
def __iter__(self):
939+
finished.set()
940+
return iter([2])
941+
942+
kw = {"fillvalue": EphemeralFillValue()}
943+
with threading_helper.wait_threads_exit():
944+
_thread.start_new_thread(zip_longest, (Trigger(kw), TailIterable()), kw)
945+
self.assertTrue(cleared.wait(timeout=support.SHORT_TIMEOUT))
946+
for _ in range(1000):
947+
pass
948+
merged.set()
949+
self.assertTrue(finished.wait(timeout=support.SHORT_TIMEOUT))
950+
951+
self.assertEqual(events[:2], ["BEFORE_CLEAR", "RETURNING_FROM_GETITER"])
952+
911953
def test_bug_7244(self):
912954

913955
class Repeater:
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix a use-after-free in :class:`itertools.zip_longest` when the ``kwds``
2+
dictionary is mutated or cleared while initializing input iterators.

‎Modules/itertoolsmodule.c‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3870,36 +3870,42 @@ zip_longest_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
38703870
Py_ssize_t i;
38713871
PyObject *ittuple; /* tuple of iterators */
38723872
PyObject *result;
3873-
PyObject *fillvalue = Py_None;
3873+
PyObject *fillvalue = NULL;
38743874
Py_ssize_t tuplesize;
38753875

38763876
if (kwds != NULL && PyDict_CheckExact(kwds) && PyDict_GET_SIZE(kwds) > 0) {
3877-
fillvalue = NULL;
38783877
if (PyDict_GET_SIZE(kwds) == 1) {
3879-
fillvalue = PyDict_GetItemWithError(kwds, &_Py_ID(fillvalue));
3878+
int rc = PyDict_GetItemRef(kwds, &_Py_ID(fillvalue), &fillvalue);
3879+
if (rc < 0) {
3880+
return NULL;
3881+
}
38803882
}
38813883
if (fillvalue == NULL) {
3882-
if (!PyErr_Occurred()) {
3883-
PyErr_SetString(PyExc_TypeError,
3884-
"zip_longest() got an unexpected keyword argument");
3885-
}
3884+
PyErr_SetString(PyExc_TypeError,
3885+
"zip_longest() got an unexpected keyword argument");
38863886
return NULL;
38873887
}
38883888
}
3889+
else {
3890+
fillvalue = Py_NewRef(Py_None);
3891+
}
38893892

38903893
/* args must be a tuple */
38913894
assert(PyTuple_Check(args));
38923895
tuplesize = PyTuple_GET_SIZE(args);
38933896

38943897
/* obtain iterators */
38953898
ittuple = PyTuple_New(tuplesize);
3896-
if (ittuple == NULL)
3899+
if (ittuple == NULL) {
3900+
Py_DECREF(fillvalue);
38973901
return NULL;
3902+
}
38983903
for (i=0; i < tuplesize; i++) {
38993904
PyObject *item = PyTuple_GET_ITEM(args, i);
39003905
PyObject *it = PyObject_GetIter(item);
39013906
if (it == NULL) {
39023907
Py_DECREF(ittuple);
3908+
Py_DECREF(fillvalue);
39033909
return NULL;
39043910
}
39053911
PyTuple_SET_ITEM(ittuple, i, it);
@@ -3909,6 +3915,7 @@ zip_longest_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
39093915
result = PyTuple_New(tuplesize);
39103916
if (result == NULL) {
39113917
Py_DECREF(ittuple);
3918+
Py_DECREF(fillvalue);
39123919
return NULL;
39133920
}
39143921
for (i=0 ; i < tuplesize ; i++) {
@@ -3921,13 +3928,14 @@ zip_longest_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
39213928
if (lz == NULL) {
39223929
Py_DECREF(ittuple);
39233930
Py_DECREF(result);
3931+
Py_DECREF(fillvalue);
39243932
return NULL;
39253933
}
39263934
lz->ittuple = ittuple;
39273935
lz->tuplesize = tuplesize;
39283936
lz->numactive = tuplesize;
39293937
lz->result = result;
3930-
lz->fillvalue = Py_NewRef(fillvalue);
3938+
lz->fillvalue = fillvalue;
39313939
return (PyObject *)lz;
39323940
}
39333941

0 commit comments

Comments
 (0)