Fix JavaScript line continuation decoding in extracted messages

babel/messages/jslexer.py

@@ -29,7 +29,7 @@
 division_re = re.compile(r'/=?')
 regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*', re.DOTALL)
 line_re = re.compile(r'(\r\n|\n|\r)')
-line_join_re = re.compile(r'\\' + line_re.pattern)
+line_join_re = re.compile(r'\\(?:\r\n|[\n\r\u2028\u2029])')
 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')
 hex_escape_re = re.compile(r'[a-fA-F0-9]{1,2}')
 
@@ -55,7 +55,7 @@ class Token(NamedTuple):
     )''', re.VERBOSE)),
     ('jsx_tag', re.compile(r'(?:</?[^>\s]+|/>)', re.I)),  # May be mangled in `get_rules`
     ('operator', re.compile(r'(%s)' % '|'.join(re.escape(op) for op in operators))),
-    ('template_string', re.compile(r'''`(?:[^`\\]*(?:\\.[^`\\]*)*)`''', re.UNICODE)),
+    ('template_string', re.compile(r'''`(?:[^`\\]*(?:\\.[^`\\]*)*)`''', re.UNICODE | re.DOTALL)),
     ('string', re.compile(r'''(
         '(?:[^'\\]*(?:\\.[^'\\]*)*)'  |
         "(?:[^"\\]*(?:\\.[^"\\]*)*)"
@@ -103,7 +103,7 @@ def unquote_string(string: str) -> str:
     assert string and string[0] == string[-1] and string[0] in '"\'`', (
         'string provided is not properly delimited'
     )
-    string = line_join_re.sub('\\1', string[1:-1])
+    string = string[1:-1]
     result: list[str] = []
     add = result.append
     pos = 0
@@ -117,6 +117,11 @@ def unquote_string(string: str) -> str:
 
         # check which character is escaped
         next_char = string[escape_pos + 1]
+        continuation = line_join_re.match(string, escape_pos)
+        if continuation is not None:
+            pos = continuation.end()
+            continue
+
         if next_char in escapes:
             add(escapes[next_char])
 

tests/messages/test_js_line_continuations.py

@@ -0,0 +1,34 @@
+from io import BytesIO
+
+import pytest
+
+from babel.messages import extract, jslexer
+
+
+@pytest.mark.parametrize('quote', ["'", '"', '`'])
+@pytest.mark.parametrize('newline', ['\n', '\r', '\r\n', '\u2028', '\u2029'])
+def test_unquote_line_continuation(quote, newline):
+    source = f'{quote}hello\\{newline}world{quote}'
+    assert jslexer.unquote_string(source) == 'helloworld'
+
+
+@pytest.mark.parametrize('quote', ["'", '"', '`'])
+@pytest.mark.parametrize('newline', ['\n', '\r', '\r\n'])
+def test_extract_line_continuation_and_keep_source_line_numbers(quote, newline):
+    source = f'gettext({quote}hello\\{newline}world{quote});\ngettext("after");'
+    messages = list(extract.extract('javascript', BytesIO(source.encode('utf-8'))))
+    assert messages == [(1, 'helloworld', [], None), (3, 'after', [], None)]
+
+
+@pytest.mark.parametrize('quote', ["'", '"', '`'])
+@pytest.mark.parametrize(('content', 'expected'), [
+    (r'hello\nworld', 'hello\nworld'),
+    (r'hello\\nworld', r'hello\nworld'),
+])
+def test_ordinary_escapes_are_preserved(quote, content, expected):
+    assert jslexer.unquote_string(quote + content + quote) == expected
+
+
+def test_escaped_backslash_does_not_start_template_line_continuation():
+    source = '`hello' + '\\\\' + '\nworld`'
+    assert jslexer.unquote_string(source) == 'hello\\\nworld'