Skip to content

Commit 4fb1a67

Browse files
committed
gh-162125: Preserve parentheses when folding email comments
Keep comment delimiters outside encoded words and stop encoded-word merging at actual comment boundaries. Add reproducible generated round-trip tests for Unicode comments, quoted pairs, display names, and folding limits.
1 parent 4497fe9 commit 4fb1a67

4 files changed

Lines changed: 142 additions & 12 deletions

File tree

‎Lib/email/_header_value_parser.py‎

Lines changed: 51 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2911,12 +2911,24 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
29112911
last_ew = None # if there is an encoded word in the last line of lines,
29122912
# points to the encoded word's first character
29132913
last_charset = None
2914+
last_comment_end = None # (line count, column) of the last emitted comment
29142915
wrap_as_ew_blocked = 0
29152916
want_encoding = False # This is set to True if we need to encode this part
29162917
end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked')
29172918
parts = list(parse_tree)
29182919
while parts:
29192920
part = parts.pop(0)
2921+
if part.token_type == 'cfws':
2922+
# Visit comments themselves before changing encoded-word state.
2923+
parts = list(part) + parts
2924+
continue
2925+
if part.token_type == 'comment':
2926+
# Encoded words must not combine across comment boundaries.
2927+
if (last_word_is_ew and parse_tree.token_type != 'comment' and
2928+
not lines[-1].endswith((' ', '\t'))):
2929+
lines[-1] += ' '
2930+
last_ew = None
2931+
last_word_is_ew = False
29202932
if part is end_ew_not_allowed:
29212933
wrap_as_ew_blocked -= 1
29222934
continue
@@ -2946,6 +2958,24 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
29462958
charset = 'utf-8'
29472959
want_encoding = True
29482960

2961+
if (part.token_type == 'comment' and not wrap_as_ew_blocked and
2962+
(want_encoding or
2963+
(parse_tree.token_type == 'comment' and
2964+
len(tstr) > maxlen - len(lines[-1])))):
2965+
# Comment.__str__ supplies the parentheses. Keep them outside
2966+
# encoded words, including those in nested comments.
2967+
if len(lines[-1]) >= maxlen:
2968+
newline = _steal_trailing_WSP_if_exists(lines)
2969+
lines.append(newline or ' ')
2970+
lines[-1] += '('
2971+
_refold_with_ew(part, lines, maxlen, encoding, policy=policy)
2972+
if len(lines[-1]) >= maxlen:
2973+
lines.append(' ')
2974+
lines[-1] += ')'
2975+
last_comment_end = (len(lines), len(lines[-1]))
2976+
want_encoding = False
2977+
continue
2978+
29492979
if want_encoding and not wrap_as_ew_blocked:
29502980
if any(
29512981
not x.as_ew_allowed for x in part
@@ -2955,7 +2985,10 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
29552985
last_ew = None
29562986
if part.syntactic_break:
29572987
encoded_part = part.fold(policy=policy)[:-len(policy.linesep)]
2958-
if policy.linesep not in encoded_part:
2988+
if (policy.linesep not in encoded_part and
2989+
(not part.contains_token_type('comment') or
2990+
len(encoded_part) <= maxlen - len(lines[-1]) or
2991+
len(encoded_part) + 1 <= maxlen)):
29592992
# It fits on a single line
29602993
if len(encoded_part) > maxlen - len(lines[-1]):
29612994
# But not on this one, so start a new one.
@@ -2977,6 +3010,9 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
29773010
elif part.as_ew_allowed:
29783011
# It's a terminal, wrap it as an encoded word, possibly
29793012
# combining it with previously encoded words if allowed.
3013+
if (parse_tree.token_type != 'comment' and
3014+
last_comment_end == (len(lines), len(lines[-1]))):
3015+
lines[-1] += ' '
29803016
if (last_ew is not None and
29813017
charset != last_charset and
29823018
(last_charset == 'unknown-8bit' or
@@ -3004,6 +3040,8 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
30043040

30053041
if len(tstr) <= maxlen - len(lines[-1]):
30063042
lines[-1] += tstr
3043+
if part.token_type == 'comment':
3044+
last_comment_end = (len(lines), len(lines[-1]))
30073045
last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP))
30083046
continue
30093047

@@ -3015,6 +3053,8 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
30153053
newline = _steal_trailing_WSP_if_exists(lines)
30163054
if newline or part.startswith_fws():
30173055
lines.append(newline + tstr)
3056+
if part.token_type == 'comment':
3057+
last_comment_end = (len(lines), len(lines[-1]))
30183058
last_word_is_ew = (last_word_is_ew
30193059
and not bool(lines[-1].strip(_WSP)))
30203060
last_ew = None
@@ -3116,21 +3156,23 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset,
31163156
while to_encode:
31173157
remaining_space = maxlen - len(lines[-1])
31183158
text_space = remaining_space - chrome_len
3119-
if text_space <= 0:
3120-
newline = _steal_trailing_WSP_if_exists(lines)
3121-
lines.append(newline or ' ')
3122-
new_last_ew = len(lines[-1])
3123-
continue
3124-
3125-
to_encode_word = to_encode[:text_space]
3159+
to_encode_word = to_encode[:max(1, text_space)]
31263160
encoded_word = _ew.encode(to_encode_word, charset=encode_as)
31273161
excess = len(encoded_word) - remaining_space
3128-
while excess > 0:
3162+
while excess > 0 and len(to_encode_word) > 1:
31293163
# Since the chunk to encode is guaranteed to fit into less than 100 characters,
31303164
# shrinking it by one at a time shouldn't take long.
31313165
to_encode_word = to_encode_word[:-1]
31323166
encoded_word = _ew.encode(to_encode_word, charset=encode_as)
31333167
excess = len(encoded_word) - remaining_space
3168+
if excess > 0:
3169+
if not lines[-1].strip(_WSP):
3170+
raise errors.HeaderParseError(
3171+
"max_line_length is too small to fit an encoded word")
3172+
newline = _steal_trailing_WSP_if_exists(lines)
3173+
lines.append(newline or ' ')
3174+
new_last_ew = len(lines[-1])
3175+
continue
31343176
lines[-1] += encoded_word
31353177
to_encode = to_encode[len(to_encode_word):]
31363178

‎Lib/test/test_email/test__header_value_parser.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3409,9 +3409,11 @@ def test_address_list_with_long_unwrapable_comment(self):
34093409
with self.subTest(to=to):
34103410
self._test(parser.get_address_list(to)[0], folded, policy=policy)
34113411

3412-
# XXX Need tests with comments on various sides of a unicode token,
3413-
# and with unicode tokens in the comments. Spaces inside the quotes
3414-
# currently don't do the right thing.
3412+
def test_address_list_with_unicode_comment(self):
3413+
# Regression for gh-162125: parentheses are syntax, not part of
3414+
# the encoded word containing the comment text.
3415+
self._test(parser.get_address_list('Ana (ñ) <a@example.com>')[0],
3416+
'Ana (=?utf-8?q?=C3=B1?=) <a@example.com>\n')
34153417

34163418
def test_split_at_whitespace_after_header_before_long_token(self):
34173419
body = parser.get_unstructured(' ' + 'x'*77)

‎Lib/test/test_email/test_generator.py‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import sys
77
from email import message_from_string, message_from_bytes
88
from email.message import EmailMessage
9+
from email.header import decode_header, make_header
910
from email.mime.multipart import MIMEMultipart
1011
from email.mime.text import MIMEText
1112
from email.generator import Generator, BytesGenerator
@@ -296,6 +297,89 @@ def test_keep_long_encoded_newlines(self):
296297
g.flatten(msg)
297298
self.assertEqual(s.getvalue(), self.typ(expected))
298299

300+
def test_non_ascii_address_comments(self):
301+
cases = [
302+
# (value, max_line_length, number of comment delimiters)
303+
('Ana (ñ) <a@example.com>', 20, 1),
304+
('Ana (ñ) <a@example.com>', 27, 1),
305+
('Ana (((é))) <a@example.com>', 20, 3),
306+
]
307+
for value, maxlen, comment_count in cases:
308+
p = self.policy.clone(max_line_length=maxlen)
309+
with self.subTest(value=value, maxlen=maxlen):
310+
msg = EmailMessage(policy=p)
311+
msg['To'] = value
312+
s = self.ioclass()
313+
self.genclass(s, policy=p).flatten(msg)
314+
folded = s.getvalue()
315+
self.assertEqual(folded.count(self.typ('(')), comment_count)
316+
self.assertEqual(folded.count(self.typ(')')), comment_count)
317+
self.assertNotIn(self.typ('=?utf-8?q??='), folded)
318+
for line in folded.splitlines():
319+
self.assertLessEqual(len(line), maxlen)
320+
reparsed = self.msgmaker(folded, policy=p)
321+
self.assertEqual(reparsed['To'].defects, ())
322+
self.assertEqual(reparsed['To'].addresses,
323+
(Address('Ana', 'a', 'example.com'),))
324+
325+
def test_generated_non_ascii_address_comments(self):
326+
for case in range(32):
327+
words = [''.join(random.choices('abcdeéñΩЖ日本',
328+
k=random.randint(1, 8)))
329+
for _ in range(random.randint(1, 3))]
330+
comment = random.choice('éñΩ日') + ''.join(
331+
random.choices('abcéñΩ日()\\ ', k=random.randrange(24)))
332+
comment = ' '.join(comment.split())
333+
escaped = (comment.replace('\\', '\\\\')
334+
.replace('(', r'\(').replace(')', r'\)'))
335+
position = random.randrange(len(words) + 1)
336+
left = ' '.join(words[:position])
337+
right = ' '.join(words[position:])
338+
gap = random.choice(('', ' ', '\t'))
339+
value = f'{left}{gap}({escaped}){gap}{right} <a@example.com>'
340+
maxlen = random.choice((26, 40, 78))
341+
p = self.policy.clone(max_line_length=maxlen)
342+
343+
with self.subTest(case=case, value=value, maxlen=maxlen):
344+
msg = EmailMessage(policy=p)
345+
msg['To'] = value
346+
s = self.ioclass()
347+
self.genclass(s, policy=p).flatten(msg)
348+
folded = s.getvalue()
349+
self.assertNotRegex(folded, self.typ(r'=\?[^?]+\?[bqBQ]\?\?='))
350+
for line in folded.splitlines():
351+
self.assertLessEqual(len(line), maxlen)
352+
353+
header = self.msgmaker(folded, policy=p)['To']
354+
self.assertEqual(header.defects, ())
355+
self.assertEqual(header.addresses, (
356+
Address(' '.join(words), 'a', 'example.com'),))
357+
comments = header._parse_tree.comments
358+
self.assertEqual(len(comments), 1)
359+
decoded = str(make_header(decode_header(comments[0])))
360+
self.assertEqual(' '.join(decoded.split()), comment)
361+
362+
def test_encoded_word_after_literal_parenthesis(self):
363+
msg = EmailMessage(policy=self.policy)
364+
msg['Subject'] = ')=?utf-8?q?=C3=A9?='
365+
s = self.ioclass()
366+
self.genclass(s, policy=self.policy).flatten(msg)
367+
reparsed = self.msgmaker(s.getvalue())
368+
self.assertEqual(str(reparsed['Subject']), ')é')
369+
370+
def test_non_ascii_content_disposition_comment(self):
371+
msg = EmailMessage(policy=self.policy)
372+
msg['Content-Disposition'] = 'attachment (é)'
373+
s = self.ioclass()
374+
self.genclass(s, policy=self.policy).flatten(msg)
375+
header = self.msgmaker(s.getvalue(), policy=self.policy)[
376+
'Content-Disposition']
377+
self.assertEqual(header.defects, ())
378+
self.assertEqual(header.content_disposition, 'attachment')
379+
comments = header._parse_tree.comments
380+
self.assertEqual(len(comments), 1)
381+
self.assertEqual(str(make_header(decode_header(comments[0]))), 'é')
382+
299383
def test_non_ascii_addr_spec_raises(self):
300384
# non-ascii is not permitted in any part of an addr-spec. If the
301385
# programmer generated it, it's an error. (See also
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Preserve parentheses around non-ASCII comments when folding :mod:`email`
2+
headers with encoded words.

0 commit comments

Comments
 (0)