From 4fb1a67d0f2d46ff22d42f370a2bd9e43c0cf9ea Mon Sep 17 00:00:00 2001
From: Octopus136 <1361227324@qq.com>
Date: Wed, 18 Nov 2026 01:34:52 +0800
Subject: [PATCH] gh-162125: Preserve parentheses when folding email comments
Keep comment delimiters outside encoded words and stop encoded-word
merging at actual comment boundaries.
Add reproducible generated round-trip tests for Unicode comments,
quoted pairs, display names, and folding limits.
---
Lib/email/_header_value_parser.py | 60 +++++++++++--
.../test_email/test__header_value_parser.py | 8 +-
Lib/test/test_email/test_generator.py | 84 +++++++++++++++++++
...-11-18-00-00-00.gh-issue-162125.K7zPq2.rst | 2 +
4 files changed, 142 insertions(+), 12 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2026-11-18-00-00-00.gh-issue-162125.K7zPq2.rst
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index f396f4f9b32a9a..2c7f5e83f626c6 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -2911,12 +2911,24 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
last_ew = None # if there is an encoded word in the last line of lines,
# points to the encoded word's first character
last_charset = None
+ last_comment_end = None # (line count, column) of the last emitted comment
wrap_as_ew_blocked = 0
want_encoding = False # This is set to True if we need to encode this part
end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked')
parts = list(parse_tree)
while parts:
part = parts.pop(0)
+ if part.token_type == 'cfws':
+ # Visit comments themselves before changing encoded-word state.
+ parts = list(part) + parts
+ continue
+ if part.token_type == 'comment':
+ # Encoded words must not combine across comment boundaries.
+ if (last_word_is_ew and parse_tree.token_type != 'comment' and
+ not lines[-1].endswith((' ', '\t'))):
+ lines[-1] += ' '
+ last_ew = None
+ last_word_is_ew = False
if part is end_ew_not_allowed:
wrap_as_ew_blocked -= 1
continue
@@ -2946,6 +2958,24 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
charset = 'utf-8'
want_encoding = True
+ if (part.token_type == 'comment' and not wrap_as_ew_blocked and
+ (want_encoding or
+ (parse_tree.token_type == 'comment' and
+ len(tstr) > maxlen - len(lines[-1])))):
+ # Comment.__str__ supplies the parentheses. Keep them outside
+ # encoded words, including those in nested comments.
+ if len(lines[-1]) >= maxlen:
+ newline = _steal_trailing_WSP_if_exists(lines)
+ lines.append(newline or ' ')
+ lines[-1] += '('
+ _refold_with_ew(part, lines, maxlen, encoding, policy=policy)
+ if len(lines[-1]) >= maxlen:
+ lines.append(' ')
+ lines[-1] += ')'
+ last_comment_end = (len(lines), len(lines[-1]))
+ want_encoding = False
+ continue
+
if want_encoding and not wrap_as_ew_blocked:
if any(
not x.as_ew_allowed for x in part
@@ -2955,7 +2985,10 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
last_ew = None
if part.syntactic_break:
encoded_part = part.fold(policy=policy)[:-len(policy.linesep)]
- if policy.linesep not in encoded_part:
+ if (policy.linesep not in encoded_part and
+ (not part.contains_token_type('comment') or
+ len(encoded_part) <= maxlen - len(lines[-1]) or
+ len(encoded_part) + 1 <= maxlen)):
# It fits on a single line
if len(encoded_part) > maxlen - len(lines[-1]):
# But not on this one, so start a new one.
@@ -2977,6 +3010,9 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
elif part.as_ew_allowed:
# It's a terminal, wrap it as an encoded word, possibly
# combining it with previously encoded words if allowed.
+ if (parse_tree.token_type != 'comment' and
+ last_comment_end == (len(lines), len(lines[-1]))):
+ lines[-1] += ' '
if (last_ew is not None and
charset != last_charset and
(last_charset == 'unknown-8bit' or
@@ -3004,6 +3040,8 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
if len(tstr) <= maxlen - len(lines[-1]):
lines[-1] += tstr
+ if part.token_type == 'comment':
+ last_comment_end = (len(lines), len(lines[-1]))
last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP))
continue
@@ -3015,6 +3053,8 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
newline = _steal_trailing_WSP_if_exists(lines)
if newline or part.startswith_fws():
lines.append(newline + tstr)
+ if part.token_type == 'comment':
+ last_comment_end = (len(lines), len(lines[-1]))
last_word_is_ew = (last_word_is_ew
and not bool(lines[-1].strip(_WSP)))
last_ew = None
@@ -3116,21 +3156,23 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset,
while to_encode:
remaining_space = maxlen - len(lines[-1])
text_space = remaining_space - chrome_len
- if text_space <= 0:
- newline = _steal_trailing_WSP_if_exists(lines)
- lines.append(newline or ' ')
- new_last_ew = len(lines[-1])
- continue
-
- to_encode_word = to_encode[:text_space]
+ to_encode_word = to_encode[:max(1, text_space)]
encoded_word = _ew.encode(to_encode_word, charset=encode_as)
excess = len(encoded_word) - remaining_space
- while excess > 0:
+ while excess > 0 and len(to_encode_word) > 1:
# Since the chunk to encode is guaranteed to fit into less than 100 characters,
# shrinking it by one at a time shouldn't take long.
to_encode_word = to_encode_word[:-1]
encoded_word = _ew.encode(to_encode_word, charset=encode_as)
excess = len(encoded_word) - remaining_space
+ if excess > 0:
+ if not lines[-1].strip(_WSP):
+ raise errors.HeaderParseError(
+ "max_line_length is too small to fit an encoded word")
+ newline = _steal_trailing_WSP_if_exists(lines)
+ lines.append(newline or ' ')
+ new_last_ew = len(lines[-1])
+ continue
lines[-1] += encoded_word
to_encode = to_encode[len(to_encode_word):]
diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py
index 26e9249ba9e948..625d5a8438795f 100644
--- a/Lib/test/test_email/test__header_value_parser.py
+++ b/Lib/test/test_email/test__header_value_parser.py
@@ -3409,9 +3409,11 @@ def test_address_list_with_long_unwrapable_comment(self):
with self.subTest(to=to):
self._test(parser.get_address_list(to)[0], folded, policy=policy)
- # XXX Need tests with comments on various sides of a unicode token,
- # and with unicode tokens in the comments. Spaces inside the quotes
- # currently don't do the right thing.
+ def test_address_list_with_unicode_comment(self):
+ # Regression for gh-162125: parentheses are syntax, not part of
+ # the encoded word containing the comment text.
+ self._test(parser.get_address_list('Ana (ñ) ')[0],
+ 'Ana (=?utf-8?q?=C3=B1?=) \n')
def test_split_at_whitespace_after_header_before_long_token(self):
body = parser.get_unstructured(' ' + 'x'*77)
diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py
index 8d912738029f78..2320ab2f205585 100644
--- a/Lib/test/test_email/test_generator.py
+++ b/Lib/test/test_email/test_generator.py
@@ -6,6 +6,7 @@
import sys
from email import message_from_string, message_from_bytes
from email.message import EmailMessage
+from email.header import decode_header, make_header
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.generator import Generator, BytesGenerator
@@ -296,6 +297,89 @@ def test_keep_long_encoded_newlines(self):
g.flatten(msg)
self.assertEqual(s.getvalue(), self.typ(expected))
+ def test_non_ascii_address_comments(self):
+ cases = [
+ # (value, max_line_length, number of comment delimiters)
+ ('Ana (ñ) ', 20, 1),
+ ('Ana (ñ) ', 27, 1),
+ ('Ana (((é))) ', 20, 3),
+ ]
+ for value, maxlen, comment_count in cases:
+ p = self.policy.clone(max_line_length=maxlen)
+ with self.subTest(value=value, maxlen=maxlen):
+ msg = EmailMessage(policy=p)
+ msg['To'] = value
+ s = self.ioclass()
+ self.genclass(s, policy=p).flatten(msg)
+ folded = s.getvalue()
+ self.assertEqual(folded.count(self.typ('(')), comment_count)
+ self.assertEqual(folded.count(self.typ(')')), comment_count)
+ self.assertNotIn(self.typ('=?utf-8?q??='), folded)
+ for line in folded.splitlines():
+ self.assertLessEqual(len(line), maxlen)
+ reparsed = self.msgmaker(folded, policy=p)
+ self.assertEqual(reparsed['To'].defects, ())
+ self.assertEqual(reparsed['To'].addresses,
+ (Address('Ana', 'a', 'example.com'),))
+
+ def test_generated_non_ascii_address_comments(self):
+ for case in range(32):
+ words = [''.join(random.choices('abcdeéñΩЖ日本',
+ k=random.randint(1, 8)))
+ for _ in range(random.randint(1, 3))]
+ comment = random.choice('éñΩ日') + ''.join(
+ random.choices('abcéñΩ日()\\ ', k=random.randrange(24)))
+ comment = ' '.join(comment.split())
+ escaped = (comment.replace('\\', '\\\\')
+ .replace('(', r'\(').replace(')', r'\)'))
+ position = random.randrange(len(words) + 1)
+ left = ' '.join(words[:position])
+ right = ' '.join(words[position:])
+ gap = random.choice(('', ' ', '\t'))
+ value = f'{left}{gap}({escaped}){gap}{right} '
+ maxlen = random.choice((26, 40, 78))
+ p = self.policy.clone(max_line_length=maxlen)
+
+ with self.subTest(case=case, value=value, maxlen=maxlen):
+ msg = EmailMessage(policy=p)
+ msg['To'] = value
+ s = self.ioclass()
+ self.genclass(s, policy=p).flatten(msg)
+ folded = s.getvalue()
+ self.assertNotRegex(folded, self.typ(r'=\?[^?]+\?[bqBQ]\?\?='))
+ for line in folded.splitlines():
+ self.assertLessEqual(len(line), maxlen)
+
+ header = self.msgmaker(folded, policy=p)['To']
+ self.assertEqual(header.defects, ())
+ self.assertEqual(header.addresses, (
+ Address(' '.join(words), 'a', 'example.com'),))
+ comments = header._parse_tree.comments
+ self.assertEqual(len(comments), 1)
+ decoded = str(make_header(decode_header(comments[0])))
+ self.assertEqual(' '.join(decoded.split()), comment)
+
+ def test_encoded_word_after_literal_parenthesis(self):
+ msg = EmailMessage(policy=self.policy)
+ msg['Subject'] = ')=?utf-8?q?=C3=A9?='
+ s = self.ioclass()
+ self.genclass(s, policy=self.policy).flatten(msg)
+ reparsed = self.msgmaker(s.getvalue())
+ self.assertEqual(str(reparsed['Subject']), ')é')
+
+ def test_non_ascii_content_disposition_comment(self):
+ msg = EmailMessage(policy=self.policy)
+ msg['Content-Disposition'] = 'attachment (é)'
+ s = self.ioclass()
+ self.genclass(s, policy=self.policy).flatten(msg)
+ header = self.msgmaker(s.getvalue(), policy=self.policy)[
+ 'Content-Disposition']
+ self.assertEqual(header.defects, ())
+ self.assertEqual(header.content_disposition, 'attachment')
+ comments = header._parse_tree.comments
+ self.assertEqual(len(comments), 1)
+ self.assertEqual(str(make_header(decode_header(comments[0]))), 'é')
+
def test_non_ascii_addr_spec_raises(self):
# non-ascii is not permitted in any part of an addr-spec. If the
# programmer generated it, it's an error. (See also
diff --git a/Misc/NEWS.d/next/Library/2026-11-18-00-00-00.gh-issue-162125.K7zPq2.rst b/Misc/NEWS.d/next/Library/2026-11-18-00-00-00.gh-issue-162125.K7zPq2.rst
new file mode 100644
index 00000000000000..601f3ce9aa76bf
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2026-11-18-00-00-00.gh-issue-162125.K7zPq2.rst
@@ -0,0 +1,2 @@
+Preserve parentheses around non-ASCII comments when folding :mod:`email`
+headers with encoded words.