From 4fb1a67d0f2d46ff22d42f370a2bd9e43c0cf9ea Mon Sep 17 00:00:00 2001 From: Octopus136 <1361227324@qq.com> Date: Wed, 18 Nov 2026 01:34:52 +0800 Subject: [PATCH] gh-162125: Preserve parentheses when folding email comments Keep comment delimiters outside encoded words and stop encoded-word merging at actual comment boundaries. Add reproducible generated round-trip tests for Unicode comments, quoted pairs, display names, and folding limits. --- Lib/email/_header_value_parser.py | 60 +++++++++++-- .../test_email/test__header_value_parser.py | 8 +- Lib/test/test_email/test_generator.py | 84 +++++++++++++++++++ ...-11-18-00-00-00.gh-issue-162125.K7zPq2.rst | 2 + 4 files changed, 142 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-11-18-00-00-00.gh-issue-162125.K7zPq2.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index f396f4f9b32a9a..2c7f5e83f626c6 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -2911,12 +2911,24 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy): last_ew = None # if there is an encoded word in the last line of lines, # points to the encoded word's first character last_charset = None + last_comment_end = None # (line count, column) of the last emitted comment wrap_as_ew_blocked = 0 want_encoding = False # This is set to True if we need to encode this part end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') parts = list(parse_tree) while parts: part = parts.pop(0) + if part.token_type == 'cfws': + # Visit comments themselves before changing encoded-word state. + parts = list(part) + parts + continue + if part.token_type == 'comment': + # Encoded words must not combine across comment boundaries. + if (last_word_is_ew and parse_tree.token_type != 'comment' and + not lines[-1].endswith((' ', '\t'))): + lines[-1] += ' ' + last_ew = None + last_word_is_ew = False if part is end_ew_not_allowed: wrap_as_ew_blocked -= 1 continue @@ -2946,6 +2958,24 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy): charset = 'utf-8' want_encoding = True + if (part.token_type == 'comment' and not wrap_as_ew_blocked and + (want_encoding or + (parse_tree.token_type == 'comment' and + len(tstr) > maxlen - len(lines[-1])))): + # Comment.__str__ supplies the parentheses. Keep them outside + # encoded words, including those in nested comments. + if len(lines[-1]) >= maxlen: + newline = _steal_trailing_WSP_if_exists(lines) + lines.append(newline or ' ') + lines[-1] += '(' + _refold_with_ew(part, lines, maxlen, encoding, policy=policy) + if len(lines[-1]) >= maxlen: + lines.append(' ') + lines[-1] += ')' + last_comment_end = (len(lines), len(lines[-1])) + want_encoding = False + continue + if want_encoding and not wrap_as_ew_blocked: if any( not x.as_ew_allowed for x in part @@ -2955,7 +2985,10 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy): last_ew = None if part.syntactic_break: encoded_part = part.fold(policy=policy)[:-len(policy.linesep)] - if policy.linesep not in encoded_part: + if (policy.linesep not in encoded_part and + (not part.contains_token_type('comment') or + len(encoded_part) <= maxlen - len(lines[-1]) or + len(encoded_part) + 1 <= maxlen)): # It fits on a single line if len(encoded_part) > maxlen - len(lines[-1]): # But not on this one, so start a new one. @@ -2977,6 +3010,9 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy): elif part.as_ew_allowed: # It's a terminal, wrap it as an encoded word, possibly # combining it with previously encoded words if allowed. + if (parse_tree.token_type != 'comment' and + last_comment_end == (len(lines), len(lines[-1]))): + lines[-1] += ' ' if (last_ew is not None and charset != last_charset and (last_charset == 'unknown-8bit' or @@ -3004,6 +3040,8 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy): if len(tstr) <= maxlen - len(lines[-1]): lines[-1] += tstr + if part.token_type == 'comment': + last_comment_end = (len(lines), len(lines[-1])) last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP)) continue @@ -3015,6 +3053,8 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy): newline = _steal_trailing_WSP_if_exists(lines) if newline or part.startswith_fws(): lines.append(newline + tstr) + if part.token_type == 'comment': + last_comment_end = (len(lines), len(lines[-1])) last_word_is_ew = (last_word_is_ew and not bool(lines[-1].strip(_WSP))) last_ew = None @@ -3116,21 +3156,23 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, while to_encode: remaining_space = maxlen - len(lines[-1]) text_space = remaining_space - chrome_len - if text_space <= 0: - newline = _steal_trailing_WSP_if_exists(lines) - lines.append(newline or ' ') - new_last_ew = len(lines[-1]) - continue - - to_encode_word = to_encode[:text_space] + to_encode_word = to_encode[:max(1, text_space)] encoded_word = _ew.encode(to_encode_word, charset=encode_as) excess = len(encoded_word) - remaining_space - while excess > 0: + while excess > 0 and len(to_encode_word) > 1: # Since the chunk to encode is guaranteed to fit into less than 100 characters, # shrinking it by one at a time shouldn't take long. to_encode_word = to_encode_word[:-1] encoded_word = _ew.encode(to_encode_word, charset=encode_as) excess = len(encoded_word) - remaining_space + if excess > 0: + if not lines[-1].strip(_WSP): + raise errors.HeaderParseError( + "max_line_length is too small to fit an encoded word") + newline = _steal_trailing_WSP_if_exists(lines) + lines.append(newline or ' ') + new_last_ew = len(lines[-1]) + continue lines[-1] += encoded_word to_encode = to_encode[len(to_encode_word):] diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 26e9249ba9e948..625d5a8438795f 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -3409,9 +3409,11 @@ def test_address_list_with_long_unwrapable_comment(self): with self.subTest(to=to): self._test(parser.get_address_list(to)[0], folded, policy=policy) - # XXX Need tests with comments on various sides of a unicode token, - # and with unicode tokens in the comments. Spaces inside the quotes - # currently don't do the right thing. + def test_address_list_with_unicode_comment(self): + # Regression for gh-162125: parentheses are syntax, not part of + # the encoded word containing the comment text. + self._test(parser.get_address_list('Ana (ñ) ')[0], + 'Ana (=?utf-8?q?=C3=B1?=) \n') def test_split_at_whitespace_after_header_before_long_token(self): body = parser.get_unstructured(' ' + 'x'*77) diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index 8d912738029f78..2320ab2f205585 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -6,6 +6,7 @@ import sys from email import message_from_string, message_from_bytes from email.message import EmailMessage +from email.header import decode_header, make_header from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.generator import Generator, BytesGenerator @@ -296,6 +297,89 @@ def test_keep_long_encoded_newlines(self): g.flatten(msg) self.assertEqual(s.getvalue(), self.typ(expected)) + def test_non_ascii_address_comments(self): + cases = [ + # (value, max_line_length, number of comment delimiters) + ('Ana (ñ) ', 20, 1), + ('Ana (ñ) ', 27, 1), + ('Ana (((é))) ', 20, 3), + ] + for value, maxlen, comment_count in cases: + p = self.policy.clone(max_line_length=maxlen) + with self.subTest(value=value, maxlen=maxlen): + msg = EmailMessage(policy=p) + msg['To'] = value + s = self.ioclass() + self.genclass(s, policy=p).flatten(msg) + folded = s.getvalue() + self.assertEqual(folded.count(self.typ('(')), comment_count) + self.assertEqual(folded.count(self.typ(')')), comment_count) + self.assertNotIn(self.typ('=?utf-8?q??='), folded) + for line in folded.splitlines(): + self.assertLessEqual(len(line), maxlen) + reparsed = self.msgmaker(folded, policy=p) + self.assertEqual(reparsed['To'].defects, ()) + self.assertEqual(reparsed['To'].addresses, + (Address('Ana', 'a', 'example.com'),)) + + def test_generated_non_ascii_address_comments(self): + for case in range(32): + words = [''.join(random.choices('abcdeéñΩЖ日本', + k=random.randint(1, 8))) + for _ in range(random.randint(1, 3))] + comment = random.choice('éñΩ日') + ''.join( + random.choices('abcéñΩ日()\\ ', k=random.randrange(24))) + comment = ' '.join(comment.split()) + escaped = (comment.replace('\\', '\\\\') + .replace('(', r'\(').replace(')', r'\)')) + position = random.randrange(len(words) + 1) + left = ' '.join(words[:position]) + right = ' '.join(words[position:]) + gap = random.choice(('', ' ', '\t')) + value = f'{left}{gap}({escaped}){gap}{right} ' + maxlen = random.choice((26, 40, 78)) + p = self.policy.clone(max_line_length=maxlen) + + with self.subTest(case=case, value=value, maxlen=maxlen): + msg = EmailMessage(policy=p) + msg['To'] = value + s = self.ioclass() + self.genclass(s, policy=p).flatten(msg) + folded = s.getvalue() + self.assertNotRegex(folded, self.typ(r'=\?[^?]+\?[bqBQ]\?\?=')) + for line in folded.splitlines(): + self.assertLessEqual(len(line), maxlen) + + header = self.msgmaker(folded, policy=p)['To'] + self.assertEqual(header.defects, ()) + self.assertEqual(header.addresses, ( + Address(' '.join(words), 'a', 'example.com'),)) + comments = header._parse_tree.comments + self.assertEqual(len(comments), 1) + decoded = str(make_header(decode_header(comments[0]))) + self.assertEqual(' '.join(decoded.split()), comment) + + def test_encoded_word_after_literal_parenthesis(self): + msg = EmailMessage(policy=self.policy) + msg['Subject'] = ')=?utf-8?q?=C3=A9?=' + s = self.ioclass() + self.genclass(s, policy=self.policy).flatten(msg) + reparsed = self.msgmaker(s.getvalue()) + self.assertEqual(str(reparsed['Subject']), ')é') + + def test_non_ascii_content_disposition_comment(self): + msg = EmailMessage(policy=self.policy) + msg['Content-Disposition'] = 'attachment (é)' + s = self.ioclass() + self.genclass(s, policy=self.policy).flatten(msg) + header = self.msgmaker(s.getvalue(), policy=self.policy)[ + 'Content-Disposition'] + self.assertEqual(header.defects, ()) + self.assertEqual(header.content_disposition, 'attachment') + comments = header._parse_tree.comments + self.assertEqual(len(comments), 1) + self.assertEqual(str(make_header(decode_header(comments[0]))), 'é') + def test_non_ascii_addr_spec_raises(self): # non-ascii is not permitted in any part of an addr-spec. If the # programmer generated it, it's an error. (See also diff --git a/Misc/NEWS.d/next/Library/2026-11-18-00-00-00.gh-issue-162125.K7zPq2.rst b/Misc/NEWS.d/next/Library/2026-11-18-00-00-00.gh-issue-162125.K7zPq2.rst new file mode 100644 index 00000000000000..601f3ce9aa76bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-11-18-00-00-00.gh-issue-162125.K7zPq2.rst @@ -0,0 +1,2 @@ +Preserve parentheses around non-ASCII comments when folding :mod:`email` +headers with encoded words.