Hoist top-level function declarations out of the wombat block

CHANGELOG.md

@@ -24,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - Rewrite `save_large_file` to stream and resume the download itself instead of running `wget` once, whose `-t` counter is a budget for the whole download: a flaky connection now keeps resuming and the retry budget only counts attempts that make no progress, so long downloads are no longer abandoned mid-way (#348)
-- Carry a script's top-level `const`, `let` and `class` declarations across the wombat block, so other scripts on the page can still see them (#329)
+- Carry a script's top-level `const`, `let`, `class` and `function` declarations across the wombat block, so other scripts on the page can still see them, re-declaring classes with `let` so they can be reassigned (#329, webrecorder/wabac.js#380)
 - Keep a rewritten `<script>`'s original `src` in `__wb_orig_src`, so wombat can return it from `getAttribute("src")` and bundlers that identify chunks by it (Next.js/Turbopack, Vite) still work (openzim/warc2zim#473)
 - Keep the original of every rewritten `src`, `href` and `xlink:href` in `__wb_orig_<attr>`, not just a script's `src`, so wombat can return it instead of a broken URL (openzim/warc2zim#413, #338)
 - Keep URL-encoded `#` characters of the path or querystring when rewriting document URIs, instead of dropping everything after them (#341)

rules/rules.yaml

@@ -5,9 +5,9 @@
 # scraperlib we are not really fuzzy matching (searching the best entry among existing
 # ones) but just rewriting to proper path.
 #
-# This file is in sync with content at commit
-# https://github.com/webrecorder/wabac.js/commit/067214314f86249ecc7610bf8ac14b7f54c5f285
-# from Apr 29, 2026
+# This file is in sync with content at commit (release 2.27.4)
+# https://github.com/webrecorder/wabac.js/commit/f62b924492339cf2ee069f51319ab6603f63500d
+# from Oct 22, 2026
 #
 # This file should be updated at every release of scraperlib
 #

src/zimscraperlib/rewriting/js.py

@@ -13,8 +13,8 @@
 ZIM at `_zim_static/__wb_module_decl.js`
 
 This code is based on https://github.com/webrecorder/wabac.js/blob/main/src/rewrite/jsrewriter.ts
-Last backport of upstream changes is from wabac.js commit:
-Jul 30, 2026 - 0564e36993f4044f17119e71dd7b2892512ee59c
+Last backport of upstream changes is from wabac.js commit (release 2.27.4):
+Oct 22, 2026 - f62b924492339cf2ee069f51319ab6603f63500d
 """
 
 import re
@@ -356,89 +356,97 @@
 
         return new_text
 
     def _wrap(self, new_text: str, overrides: list[str]) -> str:
         """Put the script inside the wombat block, and put its globals back.
 
-        The block is a scope, so `const`, `let` and `class` declared at the top
-        level of the script stop being reachable from any other script on the
-        page — which is how a page that declares its data in one <script> and
-        reads it from another comes out broken but silent (#329).
+        The block is a scope, so `const`, `let`, `class` and `function`
+        declared at the top level of the script stop being reachable from any
+        other script on the page — which is how a page that declares its data in
+        one <script> and reads it from another comes out broken but silent
+        (#329, webrecorder/wabac.js#379).
 
         So the declarations are carried across the block boundary, exactly as
         wabac.js does it:
 
           * `let x` is declared before the block and the keyword removed
             inside it, so the assignment inside writes the outer binding
-          * `const x` and `class X` cannot be split that way, so their value
-            is handed out through `self.___WB_const_x` and re-declared as a
-            const after the block, and the carrier deleted
+          * `const x`, `class X` and `function f` cannot be split that way, so
+            their value is hoisted out through `self.___WB_hoist_x` and
+            re-declared after the block, and the carrier deleted: a const as a
+            `const`, a class as a `let` (its binding can be reassigned) and a
+            function as a `var` (as a function on the global scope is)
           * a name that shadows one of the wombat globals is left alone, and
             that global is dropped from the wrapper instead
           * a top-level `document.write()` gets its `document.close()`
 
         If the script cannot be parsed, none of this happens and the wrapper is
         exactly what it was before: a script Zimi cannot read is still a script
         it must not corrupt."""
         first_buff = self.first_buff
         last_buff = self.last_buff
         pre_scope_globals = ""
         in_scope_globals = ""
         post_scope_globals = ""
 
         parsed = parse_top_level(new_text) if new_text else None
         if parsed is not None:
-            names: list[tuple[str, str]] = []
+            # (name, kind it is re-declared with, hoisted through a carrier)
+            names: list[tuple[str, str, bool]] = []
             exclude_overrides: set[str] = set()
             let_offsets: list[int] = []
             last_start = -1
             for decl in parsed.declarations:
                 if decl.name in overrides:
                     exclude_overrides.add(decl.name)
                     continue
-                if decl.kind == "class":
-                    names.append((decl.name, "const"))
-                elif decl.kind in ("const", "let"):
-                    names.append((decl.name, decl.kind))
-                    if decl.kind == "let" and last_start != decl.start:
+                if decl.kind == "const":
+                    names.append((decl.name, "const", True))
+                elif decl.kind == "let":
+                    names.append((decl.name, "let", False))
+                    if last_start != decl.start:
                         let_offsets.insert(0, decl.start)
                         last_start = decl.start
+                elif decl.kind == "class":
+                    names.append((decl.name, "let", True))
+                elif decl.kind == "function":
+                    names.append((decl.name, "var", True))
 
             if exclude_overrides:
                 first_buff = self._init_local_declaration(
                     [name for name in overrides if name not in exclude_overrides]
                 )
             if parsed.has_document_write:
                 last_buff = ";document.close();" + self.last_buff
 
             # Offsets are byte offsets into the source, and descending, so each
             # removal leaves the ones still to come valid.
             data = new_text.encode("utf-8")
             for offset in let_offsets:
                 data = data[:offset] + data[offset + len("let") :]
             new_text = data.decode("utf-8", errors="replace")
 
-            for name, kind in names:
-                if kind == "const":
-                    varname = f"self.___WB_const_{name}"
+            for name, kind, hoist in names:
+                if hoist:
+                    varname = f"self.___WB_hoist_{name}"
                     in_scope_globals += f"{varname} = {name};\n"
                     post_scope_globals += (
                         f"{kind} {name} = {varname}; delete {varname};\n"
                     )
                 else:
                     pre_scope_globals += f"let {name};\n"
             if in_scope_globals:
                 in_scope_globals = "\n;" + in_scope_globals
             if post_scope_globals:
                 post_scope_globals = "\n" + post_scope_globals
 
         return (
             pre_scope_globals
             + first_buff
             + new_text
             + in_scope_globals
             + last_buff
             + post_scope_globals
         )
 
     def _get_esm_import_rule(self) -> TransformationRule:
         # Capture plain local values instead of closing over `self`: a closure that

src/zimscraperlib/rewriting/js_ast.py

@@ -3,15 +3,16 @@
 `js.py` wraps a script in a block so wombat can shadow `window`, `document`
 and friends. A block is a scope, so every top-level `const`, `let` and `class`
 in the script becomes block-scoped too, and stops being visible to any other
-script on the page. wabac.js solves this by parsing the script and hoisting
+script on the page; so, in practice, does a top-level `function`
+(webrecorder/wabac.js#379). wabac.js solves this by parsing the script and hoisting
 those names back out; this module is the parsing half of that, kept behind one
 function so the choice of parser is one import to change.
 
 Only the top level matters. Nothing nested can leak a global, so this never
 walks into a function body, and it answers three questions:
 
-  * which `const`, `let`, `var` and `class` names the script declares at the
-    top level, and of what kind
+  * which `const`, `let`, `var`, `class` and `function` names the script
+    declares at the top level, and of what kind
   * where each declaration starts, so a `let` keyword can be removed
   * whether the script calls `document.write()` at the top level
 
@@ -40,7 +41,7 @@ class Declaration:
     """One name a script declares at its top level."""
 
     name: str
-    kind: str  # "const" | "let" | "var" | "class"
+    kind: str  # "const" | "let" | "var" | "class" | "function"
     start: int  # byte offset of the statement that declares it
 
 
@@ -111,11 +112,17 @@ def parse_top_level(text: str) -> TopLevel | None:
             elif node.type == "variable_declaration":
                 for name in _identifiers(node, source):
                     declarations.append(Declaration(name, "var", node.start_byte))
-            elif node.type == "class_declaration":
-                name_node = node.child_by_field_name("name")
-                declarations.append(
-                    Declaration(node_text(name_node, source), "class", node.start_byte)
-                )
+            elif node.type in (
+                "class_declaration",
+                "function_declaration",
+                "generator_function_declaration",
+            ):
+                # `async function` is a function_declaration too, and
+                # `async function*` a generator one. A declaration always has a
+                # name: a nameless `function () {}` parses as an expression.
+                name = node_text(node.child_by_field_name("name"), source)
+                kind = "class" if node.type == "class_declaration" else "function"
+                declarations.append(Declaration(name, kind, node.start_byte))
             elif not has_document_write and _is_document_write(node, source):
                 has_document_write = True
         return TopLevel(

tests/rewriting/test_js_globals.py

@@ -3,7 +3,8 @@
 The rewriter wraps a script in `{ ... }` so wombat can shadow `window` and
 friends. A block is a scope, so a `const`, `let` or `class` declared at the top
 level of that script stops being visible to every other script on the page —
-silently, because the script itself still runs.
+silently, because the script itself still runs. In practice, so does a top-level
+`function` (webrecorder/wabac.js#379).
 
 These cases are ported from wabac.js's own suite (test/rewriteJS.ts), which is
 the reference implementation this module tracks, with its expected strings kept
@@ -36,12 +37,12 @@ def test_a_const_is_readable_after_the_block(js_rewriter: JsRewriter):
     # The bug that started #329: nerdfonts.com declares its glyph table as a
     # top-level const in one inline script and reads it from another.
     out = js_rewriter.rewrite("const glyphs = {a: 1};\nwindow.x = 1;")
-    assert ";self.___WB_const_glyphs = glyphs;\n" in out
+    assert ";self.___WB_hoist_glyphs = glyphs;\n" in out
     assert (
-        "const glyphs = self.___WB_const_glyphs; delete self.___WB_const_glyphs;" in out
+        "const glyphs = self.___WB_hoist_glyphs; delete self.___WB_hoist_glyphs;" in out
     )
-    assert out.index("self.___WB_const_glyphs = glyphs") < out.index("\n\n}")
-    assert out.index("\n\n}") < out.index("const glyphs = self.___WB_const_glyphs")
+    assert out.index("self.___WB_hoist_glyphs = glyphs") < out.index("\n\n}")
+    assert out.index("\n\n}") < out.index("const glyphs = self.___WB_hoist_glyphs")
 
 
 def test_a_let_is_declared_before_the_block_and_assigned_inside(
@@ -55,17 +56,54 @@ def test_a_let_is_declared_before_the_block_and_assigned_inside(
     assert "let counter = 4" not in out
 
 
-def test_a_class_travels_as_a_const(js_rewriter: JsRewriter):
+def test_a_class_travels_as_a_let(js_rewriter: JsRewriter):
+    # A class binding can be reassigned, so it must not come back as a const.
     out = js_rewriter.rewrite("class Thing {}\nwindow.x = 1;")
-    assert ";self.___WB_const_Thing = Thing;\n" in out
-    assert "const Thing = self.___WB_const_Thing; delete self.___WB_const_Thing;" in out
+    assert ";self.___WB_hoist_Thing = Thing;\n" in out
+    assert "let Thing = self.___WB_hoist_Thing; delete self.___WB_hoist_Thing;" in out
+
+
+def test_a_function_travels_as_a_var(js_rewriter: JsRewriter):
+    # webrecorder/wabac.js#379: a page calling, from one script, a function
+    # declared at the top level of another. It comes back as a `var`, which is
+    # what a function declared on the global scope is.
+    out = js_rewriter.rewrite("function helper() { return 1; }\nwindow.x = 1;")
+    assert "function helper() { return 1; }" in out
+    assert ";self.___WB_hoist_helper = helper;\n" in out
+    assert (
+        "var helper = self.___WB_hoist_helper; delete self.___WB_hoist_helper;" in out
+    )
+    assert out.index("self.___WB_hoist_helper = helper") < out.index("\n\n}")
+    assert out.index("\n\n}") < out.index("var helper = self.___WB_hoist_helper")
+
+
+@pytest.mark.parametrize(
+    "declaration",
+    [
+        "async function f() {}",
+        "function* f() {}",
+        "async function* f() {}",
+    ],
+)
+def test_every_kind_of_function_travels(js_rewriter: JsRewriter, declaration: str):
+    out = js_rewriter.rewrite(f"{declaration}\nwindow.x = 1;")
+    assert ";self.___WB_hoist_f = f;\n" in out
+    assert "var f = self.___WB_hoist_f; delete self.___WB_hoist_f;" in out
+
+
+def test_a_function_expression_is_left_alone(js_rewriter: JsRewriter):
+    # Only a declaration binds a name in the script's scope; a nameless
+    # `function () {}` is an expression, so there is nothing to carry.
+    out = js_rewriter.rewrite("function() { return 1; }")
+    assert "___WB_hoist_" not in out
+    assert "function() { return 1; }" in out
 
 
 def test_var_is_left_alone(js_rewriter: JsRewriter):
     # `var` is function-scoped, so the block never captured it and there is
     # nothing to carry.
     out = js_rewriter.rewrite("var legacy = 3;\nwindow.x = 1;")
-    assert "___WB_const_legacy" not in out
+    assert "___WB_hoist_legacy" not in out
     assert "var legacy = 3;" in out
 
 
@@ -75,7 +113,7 @@ def test_a_name_that_shadows_a_wombat_global_wins(js_rewriter: JsRewriter):
     out = js_rewriter.rewrite("const location = 'here';\nwindow.x = 1;")
     assert 'let location = _____WB$wombat$assign$function_____("location");' not in out
     assert 'let window = _____WB$wombat$assign$function_____("window");' in out
-    assert "___WB_const_location" not in out
+    assert "___WB_hoist_location" not in out
 
 
 def test_a_top_level_document_write_gets_its_close(js_rewriter: JsRewriter):
@@ -90,14 +128,24 @@ class A {}
   const B = 5;
   let C = 4;
   var D = 3;
+  function E() {}
+  async function* F() {}
 
   location = "http://example.com/2\"""")
     assert out.startswith("let C;\n")
-    assert "  class A {}\n  const B = 5;\n   C = 4;\n  var D = 3;" in out
-    assert ";self.___WB_const_A = A;\nself.___WB_const_B = B;\n" in out
     assert (
-        "const A = self.___WB_const_A; delete self.___WB_const_A;\n"
-        "const B = self.___WB_const_B; delete self.___WB_const_B;\n"
+        "  class A {}\n  const B = 5;\n   C = 4;\n  var D = 3;\n"
+        "  function E() {}\n  async function* F() {}"
+    ) in out
+    assert (
+        ";self.___WB_hoist_A = A;\nself.___WB_hoist_B = B;\n"
+        "self.___WB_hoist_E = E;\nself.___WB_hoist_F = F;\n"
+    ) in out
+    assert (
+        "let A = self.___WB_hoist_A; delete self.___WB_hoist_A;\n"
+        "const B = self.___WB_hoist_B; delete self.___WB_hoist_B;\n"
+        "var E = self.___WB_hoist_E; delete self.___WB_hoist_E;\n"
+        "var F = self.___WB_hoist_F; delete self.___WB_hoist_F;\n"
     ) in out
 
 
@@ -109,19 +157,19 @@ def test_several_declarators_on_one_line(js_rewriter: JsRewriter):
     )
     assert out.startswith("let a;\nlet b;\nlet c;\n")
     assert "\n a = document.location.href, b = 1, c = 2;" in out
-    assert ";self.___WB_const_foo = foo;\nself.___WB_const_bar = bar;\n" in out
+    assert ";self.___WB_hoist_foo = foo;\nself.___WB_hoist_bar = bar;\n" in out
     assert (
-        "const foo = self.___WB_const_foo; delete self.___WB_const_foo;\n"
-        "const bar = self.___WB_const_bar; delete self.___WB_const_bar;\n"
+        "const foo = self.___WB_hoist_foo; delete self.___WB_hoist_foo;\n"
+        "const bar = self.___WB_hoist_bar; delete self.___WB_hoist_bar;\n"
     ) in out
 
 
 def test_a_carried_const_and_a_document_write_together(js_rewriter: JsRewriter):
     # wabac.js: "global + document.close append". The close goes after the
     # carrier assignment and still inside the block.
     out = js_rewriter.rewrite("\n\nconst y = document.location;\ndocument.write(x);")
-    assert ";self.___WB_const_y = y;\n;document.close();" in out
-    assert "const y = self.___WB_const_y; delete self.___WB_const_y;\n" in out
+    assert ";self.___WB_hoist_y = y;\n;document.close();" in out
+    assert "const y = self.___WB_hoist_y; delete self.___WB_hoist_y;\n" in out
 
 
 def test_let_var_and_const_in_one_script(js_rewriter: JsRewriter):
@@ -131,14 +179,14 @@ def test_let_var_and_const_in_one_script(js_rewriter: JsRewriter):
     )
     assert out.startswith("let a;\n")
     assert "\n a = document.location.href; var b = 5; const foo = 4" in out
-    assert ";self.___WB_const_foo = foo;\n" in out
+    assert ";self.___WB_hoist_foo = foo;\n" in out
 
 
 def test_nothing_is_carried_out_of_a_nested_scope(js_rewriter: JsRewriter):
     # Only the top level can leak a global; a const inside a function is the
     # function's business and must be left exactly as written.
     out = js_rewriter.rewrite("function f() { const inner = 1; }\nwindow.x = 1;")
-    assert "___WB_const_inner" not in out
+    assert "___WB_hoist_inner" not in out
     assert "const inner = 1;" in out
 
 
@@ -150,14 +198,14 @@ def test_modern_syntax_is_still_parsed(js_rewriter: JsRewriter):
         "const config = window?.settings ?? {};\nclass P { #id = 1; static all = []; }"
     )
     # Only the first carried name takes the leading ";".
-    assert ";self.___WB_const_config = config;\n" in out
-    assert "self.___WB_const_P = P;\n" in out
+    assert ";self.___WB_hoist_config = config;\n" in out
+    assert "self.___WB_hoist_P = P;\n" in out
 
 
 def test_an_unparsable_script_is_wrapped_but_not_altered(js_rewriter: JsRewriter):
     # A script Zimi cannot read is still a script it must not corrupt.
     out = js_rewriter.rewrite("window.x = 1; const = = =;")
-    assert "___WB_const_" not in out
+    assert "___WB_hoist_" not in out
     assert "const = = =;" in out
 
 
@@ -166,15 +214,15 @@ def test_a_script_with_no_globals_is_unchanged_around_the_block(
 ):
     out = js_rewriter.rewrite("window.x = 1;")
     assert not out.startswith("let ")
-    assert "___WB_const_" not in out
+    assert "___WB_hoist_" not in out
     assert out.endswith("\n\n}")
 
 
 def test_a_destructured_declaration_is_left_alone(js_rewriter: JsRewriter):
     # Carrying `const {a, b} = x` across the block would mean rebuilding the
     # pattern; wabac.js skips these for the same reason.
     out = js_rewriter.rewrite("const {a, b} = window.data;\nwindow.x = 1;")
-    assert "___WB_const_" not in out
+    assert "___WB_hoist_" not in out
     assert "const {a, b} = window.data;" in out
 
 
@@ -183,7 +231,7 @@ def test_an_inline_script_stays_on_one_line(js_rewriter: JsRewriter):
     # end it. The hoisting adds lines, so the collapse happens after it too.
     out = js_rewriter.rewrite("const a = 1;\nwindow.x = 2;", opts={"inline": True})
     assert "\n" not in out
-    assert "self.___WB_const_a = a;" in out
+    assert "self.___WB_hoist_a = a;" in out
 
 
 def test_a_script_the_parser_cannot_read_is_wrapped_unchanged(
@@ -196,7 +244,7 @@ def cannot_read(_text: str) -> TopLevel | None:
 
     monkeypatch.setattr("zimscraperlib.rewriting.js.parse_top_level", cannot_read)
     out = js_rewriter.rewrite("const glyphs = {a: 1};\nwindow.x = 1;")
-    assert "___WB_const_" not in out
+    assert "___WB_hoist_" not in out
     assert "const glyphs = {a: 1};" in out
     assert out.endswith("\n\n}")