From 5509dd4a8242804e3fc2e4a0a8bcce6d0f7168ff Mon Sep 17 00:00:00 2001 From: Gaurav Kumar Yadav Date: Wed, 18 Nov 2026 04:24:01 +0000 Subject: [PATCH] gh-162080: Fix UnicodeEncodeError for non-ASCII MIME-Version header A MIME-Version header containing undecodable (non-ASCII) bytes parsed without error but could not be serialized again: as_bytes() and as_string() raised UnicodeEncodeError. parse_mime_version() now runs _validate_xtext() on the invalid-text terminals it creates, so an UndecodableBytesDefect is registered and the header is refolded using the unknown-8bit charset, as for other structured headers. Signed-off-by: Gaurav Kumar Yadav --- Lib/email/_header_value_parser.py | 16 +++++-- Lib/test/test_email/test_headerregistry.py | 43 +++++++++++++++++++ ...-11-18-04-23-14.gh-issue-162080.mTGVfn.rst | 5 +++ 3 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-18-04-23-14.gh-issue-162080.mTGVfn.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index f396f4f9b32a9a8..b7f3a98595600af 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -2253,7 +2253,9 @@ def parse_mime_version(value): if not digits.isdigit(): mime_version.defects.append(errors.InvalidHeaderDefect( "Expected MIME major version number but found {!r}".format(digits))) - mime_version.append(ValueTerminal(digits, 'xtext')) + xtext = ValueTerminal(digits, 'xtext') + _validate_xtext(xtext) + mime_version.append(xtext) else: mime_version.major = int(digits) mime_version.append(ValueTerminal(digits, 'digits')) @@ -2265,7 +2267,9 @@ def parse_mime_version(value): mime_version.defects.append(errors.InvalidHeaderDefect( "Incomplete MIME version; found only major number")) if value: - mime_version.append(ValueTerminal(value, 'xtext')) + xtext = ValueTerminal(value, 'xtext') + _validate_xtext(xtext) + mime_version.append(xtext) return mime_version mime_version.append(ValueTerminal('.', 'version-separator')) value = value[1:] @@ -2284,7 +2288,9 @@ def parse_mime_version(value): if not digits.isdigit(): mime_version.defects.append(errors.InvalidHeaderDefect( "Expected MIME minor version number but found {!r}".format(digits))) - mime_version.append(ValueTerminal(digits, 'xtext')) + xtext = ValueTerminal(digits, 'xtext') + _validate_xtext(xtext) + mime_version.append(xtext) else: mime_version.minor = int(digits) mime_version.append(ValueTerminal(digits, 'digits')) @@ -2294,7 +2300,9 @@ def parse_mime_version(value): if value: mime_version.defects.append(errors.InvalidHeaderDefect( "Excess non-CFWS text after MIME version")) - mime_version.append(ValueTerminal(value, 'xtext')) + xtext = ValueTerminal(value, 'xtext') + _validate_xtext(xtext) + mime_version.append(xtext) return mime_version def get_invalid_parameter(value): diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index c118fd5e20d048f..0458f8ac42ec03e 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -966,6 +966,49 @@ def version_string_as_MIME_Version(self, self.assertEqual(h.fold(policy=policy.default), 'MIME-Version:' + source + '\n') + def test_non_ascii_in_minor_version(self): + # gh-162080: a non-ASCII byte in the minor version must be + # refolded as an unknown-8bit encoded word instead of raising + # UnicodeEncodeError on as_bytes()/as_string(). + source = ('1.\xe9'.encode('latin-1') + .decode('us-ascii', 'surrogateescape')) + h = self.make_header('MIME-Version', source) + self.assertEqual(h, '1.\ufffd') + self.assertEqual(h.version, None) + self.assertEqual(h.major, None) + self.assertEqual(h.minor, None) + self.assertDefectsEqual(h.defects, [errors.InvalidHeaderDefect, + errors.UndecodableBytesDefect]) + self.assertEqual(h.fold(policy=policy.default), + 'MIME-Version: 1.=?unknown-8bit?q?=E9?=\n') + + def test_non_ascii_in_major_version(self): + # gh-162080 + source = ('\xe9.0'.encode('latin-1') + .decode('us-ascii', 'surrogateescape')) + h = self.make_header('MIME-Version', source) + self.assertEqual(h, '\ufffd.0') + self.assertEqual(h.major, None) + self.assertEqual(h.minor, 0) + self.assertDefectsEqual(h.defects, [errors.InvalidHeaderDefect, + errors.UndecodableBytesDefect]) + self.assertEqual(h.fold(policy=policy.default), + 'MIME-Version: =?unknown-8bit?q?=E9?=.0\n') + + def test_non_ascii_in_excess_text_after_version(self): + # gh-162080 + source = ('1.0 \xe9'.encode('latin-1') + .decode('us-ascii', 'surrogateescape')) + h = self.make_header('MIME-Version', source) + self.assertEqual(h, '1.0 \ufffd') + self.assertEqual(h.version, '1.0') + self.assertEqual(h.major, 1) + self.assertEqual(h.minor, 0) + self.assertDefectsEqual(h.defects, [errors.InvalidHeaderDefect, + errors.UndecodableBytesDefect]) + self.assertEqual(h.fold(policy=policy.default), + 'MIME-Version: 1.0 =?unknown-8bit?q?=E9?=\n') + version_string_params = { # Examples from the RFC. diff --git a/Misc/NEWS.d/next/Library/2026-11-18-04-23-14.gh-issue-162080.mTGVfn.rst b/Misc/NEWS.d/next/Library/2026-11-18-04-23-14.gh-issue-162080.mTGVfn.rst new file mode 100644 index 000000000000000..fc7d69f53b4ac19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-18-04-23-14.gh-issue-162080.mTGVfn.rst @@ -0,0 +1,5 @@ +Fix :meth:`email.message.Message.as_bytes` and +:meth:`email.message.Message.as_string` raising +:exc:`UnicodeEncodeError` for a ``MIME-Version`` header containing +non-ASCII bytes. Such headers are now refolded using the +``unknown-8bit`` charset, as for other structured headers.