Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion Lib/encodings/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,14 @@ def iconv_search_function(encoding):
name = encoding[len('iconv:'):]
else:
name = encoding
if not name:
# glibc's iconv_open() strips trailing '/', ',', '//SUFFIX' options,
# and any non-ASCII-alphanumeric characters outside '_-.,:', treating
# a resulting empty charset name as the current locale encoding.
# Require at least one ASCII alphanumeric before the first '/' or ','.
charset, _, _ = name.partition('/')
charset, _, _ = charset.partition(',')
if not any('0' <= c <= '9' or 'A' <= c <= 'Z' or 'a' <= c <= 'z'
for c in charset):
return None
# Test if the encoding is supported by iconv.
try:
Expand Down
32 changes: 32 additions & 0 deletions Lib/test/test_codecs.py
Original file line number Diff line number Diff line change
Expand Up @@ -3955,6 +3955,38 @@ def test_encode_surrogate_pair(self):
self.assertEqual(pair.encode('iconv:' + latin1, 'backslashreplace'),
rb'\ud83c\udf89')

def test_empty_stripped_charset(self):
# gh-161924: glibc's iconv_open() strips non-ASCII-alphanumeric
# characters and //options, falling back to the locale encoding
# if nothing remains. Ensure these raise LookupError.
ascii_invalid_names = [
'!!!',
'@',
'()',
',',
'/',
'//TRANSLIT',
'//abc',
'iconv://TRANSLIT',
]
for name in ascii_invalid_names:
with self.subTest(name=name):
self.assertRaises(LookupError, codecs.lookup, name)
self.assertRaises(LookupError, codecs.iconv_encode, name, 'a')
self.assertRaises(LookupError, codecs.iconv_decode, name, b'a')

non_ascii_invalid_names = [
'\xe9',
'\ud55c\uae00',
'iconv:\ud55c\uae00',
]
for name in non_ascii_invalid_names:
with self.subTest(name=name):
with self.assertWarns(DeprecationWarning):
self.assertRaises(LookupError, codecs.lookup, name)
self.assertRaises(LookupError, codecs.iconv_encode, name, 'a')
self.assertRaises(LookupError, codecs.iconv_decode, name, b'a')


class ASCIITest(unittest.TestCase):
def test_encode(self):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Reject encoding names in the :mod:`codecs` iconv search function when the
charset portion before ``/`` or ``,`` contains no ASCII alphanumeric
characters, preventing fallback to the current locale encoding on glibc.
36 changes: 36 additions & 0 deletions Modules/_codecsmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -658,6 +658,24 @@ _codecs_iconv_decode_impl(PyObject *module, const char *encoding,
Py_buffer *data, const char *errors, int final)
/*[clinic end generated code: output=6c6145a9decc2ba8 input=d15a04d7d3a3e0cd]*/
{
/* Reject encodings whose charset portion before '/' or ',' lacks any
* ASCII alphanumeric; glibc's iconv_open() normalizes those to "" and
* silently falls back to the current locale encoding. */
int has_alnum = 0;
for (const char *p = encoding; *p != '\0' && *p != '/' && *p != ','; p++) {
unsigned char c = (unsigned char)*p;
if ((c >= '0' && c <= '9') ||
(c >= 'A' && c <= 'Z') ||
(c >= 'a' && c <= 'z')) {
has_alnum = 1;
break;
}
}
if (!has_alnum) {
PyErr_Format(PyExc_LookupError, "unknown encoding: %s", encoding);
return NULL;
}

Py_ssize_t consumed = data->len;
PyObject *decoded = _PyUnicode_DecodeIconv(encoding, data->buf, data->len,
errors,
Expand Down Expand Up @@ -992,6 +1010,24 @@ _codecs_iconv_encode_impl(PyObject *module, const char *encoding,
PyObject *str, const char *errors)
/*[clinic end generated code: output=bda0c5acd9be0f17 input=53a1e2028c9e2b43]*/
{
/* Reject encodings whose charset portion before '/' or ',' lacks any
* ASCII alphanumeric; glibc's iconv_open() normalizes those to "" and
* silently falls back to the current locale encoding. */
int has_alnum = 0;
for (const char *p = encoding; *p != '\0' && *p != '/' && *p != ','; p++) {
unsigned char c = (unsigned char)*p;
if ((c >= '0' && c <= '9') ||
(c >= 'A' && c <= 'Z') ||
(c >= 'a' && c <= 'z')) {
has_alnum = 1;
break;
}
}
if (!has_alnum) {
PyErr_Format(PyExc_LookupError, "unknown encoding: %s", encoding);
return NULL;
}

return codec_tuple(_PyUnicode_EncodeIconv(encoding, str, errors),
PyUnicode_GET_LENGTH(str));
}
Expand Down
Loading