Skip to content

Commit 5509dd4

Browse files
gh-162080: Fix UnicodeEncodeError for non-ASCII MIME-Version header
A MIME-Version header containing undecodable (non-ASCII) bytes parsed without error but could not be serialized again: as_bytes() and as_string() raised UnicodeEncodeError. parse_mime_version() now runs _validate_xtext() on the invalid-text terminals it creates, so an UndecodableBytesDefect is registered and the header is refolded using the unknown-8bit charset, as for other structured headers. Signed-off-by: Gaurav Kumar Yadav <xgauravyaduvanshii@gmail.com>
1 parent 9427640 commit 5509dd4

3 files changed

Lines changed: 60 additions & 4 deletions

File tree

‎Lib/email/_header_value_parser.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2253,7 +2253,9 @@ def parse_mime_version(value):
22532253
if not digits.isdigit():
22542254
mime_version.defects.append(errors.InvalidHeaderDefect(
22552255
"Expected MIME major version number but found {!r}".format(digits)))
2256-
mime_version.append(ValueTerminal(digits, 'xtext'))
2256+
xtext = ValueTerminal(digits, 'xtext')
2257+
_validate_xtext(xtext)
2258+
mime_version.append(xtext)
22572259
else:
22582260
mime_version.major = int(digits)
22592261
mime_version.append(ValueTerminal(digits, 'digits'))
@@ -2265,7 +2267,9 @@ def parse_mime_version(value):
22652267
mime_version.defects.append(errors.InvalidHeaderDefect(
22662268
"Incomplete MIME version; found only major number"))
22672269
if value:
2268-
mime_version.append(ValueTerminal(value, 'xtext'))
2270+
xtext = ValueTerminal(value, 'xtext')
2271+
_validate_xtext(xtext)
2272+
mime_version.append(xtext)
22692273
return mime_version
22702274
mime_version.append(ValueTerminal('.', 'version-separator'))
22712275
value = value[1:]
@@ -2284,7 +2288,9 @@ def parse_mime_version(value):
22842288
if not digits.isdigit():
22852289
mime_version.defects.append(errors.InvalidHeaderDefect(
22862290
"Expected MIME minor version number but found {!r}".format(digits)))
2287-
mime_version.append(ValueTerminal(digits, 'xtext'))
2291+
xtext = ValueTerminal(digits, 'xtext')
2292+
_validate_xtext(xtext)
2293+
mime_version.append(xtext)
22882294
else:
22892295
mime_version.minor = int(digits)
22902296
mime_version.append(ValueTerminal(digits, 'digits'))
@@ -2294,7 +2300,9 @@ def parse_mime_version(value):
22942300
if value:
22952301
mime_version.defects.append(errors.InvalidHeaderDefect(
22962302
"Excess non-CFWS text after MIME version"))
2297-
mime_version.append(ValueTerminal(value, 'xtext'))
2303+
xtext = ValueTerminal(value, 'xtext')
2304+
_validate_xtext(xtext)
2305+
mime_version.append(xtext)
22982306
return mime_version
22992307

23002308
def get_invalid_parameter(value):

‎Lib/test/test_email/test_headerregistry.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -966,6 +966,49 @@ def version_string_as_MIME_Version(self,
966966
self.assertEqual(h.fold(policy=policy.default),
967967
'MIME-Version:' + source + '\n')
968968

969+
def test_non_ascii_in_minor_version(self):
970+
# gh-162080: a non-ASCII byte in the minor version must be
971+
# refolded as an unknown-8bit encoded word instead of raising
972+
# UnicodeEncodeError on as_bytes()/as_string().
973+
source = ('1.\xe9'.encode('latin-1')
974+
.decode('us-ascii', 'surrogateescape'))
975+
h = self.make_header('MIME-Version', source)
976+
self.assertEqual(h, '1.\ufffd')
977+
self.assertEqual(h.version, None)
978+
self.assertEqual(h.major, None)
979+
self.assertEqual(h.minor, None)
980+
self.assertDefectsEqual(h.defects, [errors.InvalidHeaderDefect,
981+
errors.UndecodableBytesDefect])
982+
self.assertEqual(h.fold(policy=policy.default),
983+
'MIME-Version: 1.=?unknown-8bit?q?=E9?=\n')
984+
985+
def test_non_ascii_in_major_version(self):
986+
# gh-162080
987+
source = ('\xe9.0'.encode('latin-1')
988+
.decode('us-ascii', 'surrogateescape'))
989+
h = self.make_header('MIME-Version', source)
990+
self.assertEqual(h, '\ufffd.0')
991+
self.assertEqual(h.major, None)
992+
self.assertEqual(h.minor, 0)
993+
self.assertDefectsEqual(h.defects, [errors.InvalidHeaderDefect,
994+
errors.UndecodableBytesDefect])
995+
self.assertEqual(h.fold(policy=policy.default),
996+
'MIME-Version: =?unknown-8bit?q?=E9?=.0\n')
997+
998+
def test_non_ascii_in_excess_text_after_version(self):
999+
# gh-162080
1000+
source = ('1.0 \xe9'.encode('latin-1')
1001+
.decode('us-ascii', 'surrogateescape'))
1002+
h = self.make_header('MIME-Version', source)
1003+
self.assertEqual(h, '1.0 \ufffd')
1004+
self.assertEqual(h.version, '1.0')
1005+
self.assertEqual(h.major, 1)
1006+
self.assertEqual(h.minor, 0)
1007+
self.assertDefectsEqual(h.defects, [errors.InvalidHeaderDefect,
1008+
errors.UndecodableBytesDefect])
1009+
self.assertEqual(h.fold(policy=policy.default),
1010+
'MIME-Version: 1.0 =?unknown-8bit?q?=E9?=\n')
1011+
9691012
version_string_params = {
9701013

9711014
# Examples from the RFC.
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Fix :meth:`email.message.Message.as_bytes` and
2+
:meth:`email.message.Message.as_string` raising
3+
:exc:`UnicodeEncodeError` for a ``MIME-Version`` header containing
4+
non-ASCII bytes. Such headers are now refolded using the
5+
``unknown-8bit`` charset, as for other structured headers.

0 commit comments

Comments
 (0)