Skip to content

Commit fadb2db

Browse files
hugovkjelle-openai
andauthored
[3.14] gh-159739: Keep text I/O state valid across reentrant callbacks (GH-160398) (#162096)
A raw read can reinitialize a TextIOWrapper as write-only, and codec callbacks or finalizers can release an encoder, decoder, or decoded chunk while a read or write still needs it. This can crash the interpreter. Check that a decoder is still available after reading, keep codecs alive across calls, and read the chunk length before transferring ownership. Also recheck StringIO after converting the truncate size because __index__() can close the stream. (cherry picked from commit 665bda9) Co-authored-by: Jelle Zijlstra <jelle@openai.com>
1 parent 64be119 commit fadb2db

4 files changed

Lines changed: 45 additions & 11 deletions

File tree

‎Lib/test/test_io.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4396,6 +4396,17 @@ def make_text(buffer):
43964396
wrapper.write('x')
43974397
self.assertRaisesRegex(ValueError, "detached", wrapper.read)
43984398

4399+
def test_read_reinitialized_as_write_only(self):
4400+
class Buffer(self.BytesIO):
4401+
def read(self, size=-1):
4402+
stream.__init__(writer, encoding="ascii")
4403+
return b"a"
4404+
4405+
with (self.BufferedWriter(self.BytesIO()) as writer,
4406+
self.TextIOWrapper(Buffer(), encoding="ascii") as stream):
4407+
with self.assertRaises(self.UnsupportedOperation):
4408+
stream.read()
4409+
43994410
def test_reentrant_detach_during_read(self):
44004411
# gh-157363, gh-157364: The buffer must stay alive until its active
44014412
# read operation returns.

‎Lib/test/test_memoryio.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1005,6 +1005,18 @@ class CStringIOTest(PyStringIOTest):
10051005
ioclass = io.StringIO
10061006
UnsupportedOperation = io.UnsupportedOperation
10071007

1008+
def test_truncate_size_closes_stream(self):
1009+
stream = self.ioclass()
1010+
stream.write("abc")
1011+
1012+
class Size:
1013+
def __index__(self):
1014+
stream.close()
1015+
return 0
1016+
1017+
with self.assertRaises(ValueError):
1018+
stream.truncate(Size())
1019+
10081020
# XXX: For the Python version of io.StringIO, this is highly
10091021
# dependent on the encoding used for the underlying buffer.
10101022
def test_widechar(self):
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Fix crashes in :class:`io.TextIOWrapper` and
2+
:class:`io.IncrementalNewlineDecoder` when a callback changes the active
3+
codec or decoded buffer. :class:`io.StringIO` now raises :exc:`ValueError`
4+
if converting the size passed to ``truncate()`` closes the stream.

‎Modules/_io/textio.c‎

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -336,8 +336,11 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself,
336336

337337
/* decode input (with the eventual \r from a previous pass) */
338338
if (self->decoder != Py_None) {
339-
output = PyObject_CallMethodObjArgs(self->decoder,
339+
/* The wrapper can be reinitialized by a decoding callback. */
340+
PyObject *decoder = Py_NewRef(self->decoder);
341+
output = PyObject_CallMethodObjArgs(decoder,
340342
&_Py_ID(decode), input, final ? Py_True : Py_False, NULL);
343+
Py_DECREF(decoder);
341344
}
342345
else {
343346
output = Py_NewRef(input);
@@ -1001,11 +1004,17 @@ _textiowrapper_decode(_PyIO_State *state, PyObject *decoder, PyObject *bytes,
10011004
{
10021005
PyObject *chars;
10031006

1007+
if (decoder == NULL) {
1008+
return _unsupported(state, "not readable");
1009+
}
1010+
/* A codec callback may replace the wrapper's decoder. */
1011+
Py_INCREF(decoder);
10041012
if (Py_IS_TYPE(decoder, state->PyIncrementalNewlineDecoder_Type))
10051013
chars = _PyIncrementalNewlineDecoder_decode(decoder, bytes, eof);
10061014
else
10071015
chars = PyObject_CallMethodObjArgs(decoder, &_Py_ID(decode), bytes,
10081016
eof ? Py_True : Py_False, NULL);
1017+
Py_DECREF(decoder);
10091018

10101019
if (check_decoded(chars) < 0)
10111020
// check_decoded already decreases refcount
@@ -1805,7 +1814,10 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text)
18051814
self->encoding_start_of_stream = 0;
18061815
}
18071816
else {
1808-
b = PyObject_CallMethodOneArg(self->encoder, &_Py_ID(encode), text);
1817+
/* A codec callback may reconfigure the stream. */
1818+
PyObject *encoder = Py_NewRef(self->encoder);
1819+
b = PyObject_CallMethodOneArg(encoder, &_Py_ID(encode), text);
1820+
Py_DECREF(encoder);
18091821
}
18101822

18111823
Py_DECREF(text);
@@ -2045,8 +2057,9 @@ textiowrapper_read_chunk(textio *self, Py_ssize_t size_hint)
20452057
if (decoded_chars == NULL)
20462058
goto fail;
20472059

2048-
textiowrapper_set_decoded_chars(self, decoded_chars);
2060+
/* The old chunk's finalizer can clear the new chunk. */
20492061
nchars = PyUnicode_GET_LENGTH(decoded_chars);
2062+
textiowrapper_set_decoded_chars(self, decoded_chars);
20502063
if (nchars > 0)
20512064
self->b2cratio = (double) nbytes / nchars;
20522065
else
@@ -2118,15 +2131,9 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n)
21182131
return NULL;
21192132
}
21202133

2121-
_PyIO_State *state = self->state;
2122-
if (Py_IS_TYPE(self->decoder, state->PyIncrementalNewlineDecoder_Type))
2123-
decoded = _PyIncrementalNewlineDecoder_decode(self->decoder,
2124-
bytes, 1);
2125-
else
2126-
decoded = PyObject_CallMethodObjArgs(
2127-
self->decoder, &_Py_ID(decode), bytes, Py_True, NULL);
2134+
decoded = _textiowrapper_decode(self->state, self->decoder, bytes, 1);
21282135
Py_DECREF(bytes);
2129-
if (check_decoded(decoded) < 0)
2136+
if (decoded == NULL)
21302137
goto fail;
21312138

21322139
result = textiowrapper_get_decoded_chars(self, -1);

0 commit comments

Comments
 (0)