From 9082d987e24f4c8e4d0b1eb268f1209adcac53a4 Mon Sep 17 00:00:00 2001 From: Farhan Date: Fri, 20 Nov 2026 21:04:00 +0000 Subject: [PATCH 01/16] Import JavaScript modules from Python and call their exports by name `from my_app.scripts.party import celebrate` now finds my_app/scripts/party.js (or .ts) when no Python module of that name exists. Each named export becomes a JsFunction; calling it returns a `_call_js` event naming the module and the function, with the arguments as JSON. It works in event triggers, returned or yielded from handlers, and uncalled as a trigger (it takes the trigger's arguments, like a handler). An event handler argument arrives as its full name, for `env.addEvents([env.event(name, payload)])`. The compiler copies each imported module, with the .js/.ts files beside it so relative imports resolve, under utils/script_modules, and utils/scripts.js re-exports it under a key `$` makes unique. The frontend imports that index on the first call, so nothing is evaluated and an app that never calls a module does not fetch the chunk: it works under a CSP without unsafe-eval. npm packages the modules import are installed with the app. A stub (party.pyi) with untyped arguments is written next to the module, so type checkers resolve the import; typed stubs come next. The finder is stdlib-only and reflex_base.js is lazy, so `import reflex`, which installs it, still does not load the event system. The function's return value is not sent back: results will come from awaiting the event, a separate design. Part of ENG-9127. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- docs/api-reference/browser_javascript.md | 58 +++++ news/+js-modules.feature.md | 7 + .../reflex-base/news/+js-modules.feature.md | 1 + .../src/reflex_base/constants/base.py | 4 + .../src/reflex_base/js/__init__.py | 24 ++- .../reflex_base/js/{env.py => environment.py} | 0 .../reflex-base/src/reflex_base/js/finder.py | 89 ++++++++ .../reflex-base/src/reflex_base/js/modules.py | 204 ++++++++++++++++++ .../reflex-base/src/reflex_base/registry.py | 5 + packages/reflex-frontend/src/utils/scripts.js | 3 + packages/reflex-frontend/src/utils/state.js | 33 +++ .../reflex-frontend/tests/call_js.test.js | 75 +++++++ .../reflex-frontend/tests/public_api.test.js | 1 + pyi_hashes.json | 2 +- reflex/__init__.py | 5 + reflex/compiler/compiler.py | 6 +- reflex/compiler/utils.py | 56 +++++ .../tests_playwright/test_call_script.py | 55 +++++ tests/units/compiler/test_compiler_utils.py | 94 ++++++++ tests/units/reflex_base/js/test_modules.py | 184 ++++++++++++++++ 20 files changed, 902 insertions(+), 4 deletions(-) create mode 100644 news/+js-modules.feature.md create mode 100644 packages/reflex-base/news/+js-modules.feature.md rename packages/reflex-base/src/reflex_base/js/{env.py => environment.py} (100%) create mode 100644 packages/reflex-base/src/reflex_base/js/finder.py create mode 100644 packages/reflex-base/src/reflex_base/js/modules.py create mode 100644 packages/reflex-frontend/src/utils/scripts.js create mode 100644 packages/reflex-frontend/tests/call_js.test.js create mode 100644 tests/units/reflex_base/js/test_modules.py diff --git a/docs/api-reference/browser_javascript.md b/docs/api-reference/browser_javascript.md index 1490f3bbf4b..ac2431a9d6a 100644 --- a/docs/api-reference/browser_javascript.md +++ b/docs/api-reference/browser_javascript.md @@ -176,6 +176,64 @@ Prefer `env` over the other names a script can see, such as `refs` or `ReflexEvent`: those are internals of the frontend and may change between Reflex versions. +## JavaScript Modules + +For more than a line of Javascript, write it in a `.js` (or `.ts`) file in your +app package and import its exported functions in Python. You get your editor's +Javascript support in the file, and nothing is evaluated: the module is bundled +with the app, and calling one of its functions sends only the function's name +and its arguments as JSON. Apps whose Content Security Policy forbids +`unsafe-eval` can use them. + +```javascript +// my_app/scripts/party.js +import confetti from "canvas-confetti"; +import { env } from "$/utils/script_env"; + +export function celebrate(count, elementId) { + confetti({ particleCount: count }); + document.getElementById(elementId).scrollIntoView(); +} + +export function report(handler, value) { + env.addEvents([env.event(handler, { value })]); +} +``` + +```python +from my_app.scripts.party import celebrate, report + + +class PartyState(rx.State): + count: int = 200 + + @rx.event + def finish(self): + return celebrate(self.count, "results") + + @rx.event + def done(self, value: str): ... + + +rx.button("🎉", on_click=celebrate(PartyState.count, "results")) +rx.button("Report", on_click=report(PartyState.done, "clicked")) +``` + +- Each named export becomes a function returning an event. Use it in an event + trigger, or return or yield it from an event handler. Passed to a trigger + uncalled, like `rx.input(on_change=set_title)`, it receives the trigger's + arguments. +- Arguments are sent as JSON. An event handler arrives as its full name, which + the function can dispatch with `env.addEvents([env.event(name, payload)])`. +- The function's return value is not sent back to the backend; report results + through `env.addEvents`. +- Packages the module imports, such as `canvas-confetti`, are installed with + the app. Pin a version with `frontend_packages` in `rxconfig.py`. +- The `.js` and `.ts` files beside the module are bundled with it, so it can + import them with relative paths such as `./helpers.js`. +- Reflex writes a type stub (`party.pyi`) next to the module when it compiles + the app, so type checkers resolve the import. + ## Using React Hooks To use React Hooks directly in a Reflex app, you must subclass `rx.Component`, diff --git a/news/+js-modules.feature.md b/news/+js-modules.feature.md new file mode 100644 index 00000000000..4552c6ecd9b --- /dev/null +++ b/news/+js-modules.feature.md @@ -0,0 +1,7 @@ +Import functions from a `.js` or `.ts` file in your app package like a Python module, and call them from event triggers and event handlers. The module is bundled with the app and its npm imports are installed; see [JavaScript Modules](https://reflex.dev/docs/api-reference/browser-javascript/#javascript-modules). + +```python +from my_app.scripts.party import celebrate + +rx.button("🎉", on_click=celebrate(200, "results")) +``` diff --git a/packages/reflex-base/news/+js-modules.feature.md b/packages/reflex-base/news/+js-modules.feature.md new file mode 100644 index 00000000000..cc1bb78f888 --- /dev/null +++ b/packages/reflex-base/news/+js-modules.feature.md @@ -0,0 +1 @@ +Call exported functions of the app's own JavaScript modules from Python without evaluating any code: the `_call_js` event runs a function bundled from `utils/scripts.js` by name, with JSON arguments, so it works under a Content Security Policy without `unsafe-eval`. diff --git a/packages/reflex-base/src/reflex_base/constants/base.py b/packages/reflex-base/src/reflex_base/constants/base.py index d8f8eebdb4f..a6c2cbc2581 100644 --- a/packages/reflex-base/src/reflex_base/constants/base.py +++ b/packages/reflex-base/src/reflex_base/constants/base.py @@ -42,6 +42,10 @@ class Dirs(SimpleNamespace): CLIENT_STATE_PATH = UTILS + "/client_state" # The name of the script environment file. SCRIPT_ENV_PATH = UTILS + "/script_env" + # The index of the JavaScript modules an app imports from Python. + SCRIPTS_PATH = UTILS + "/scripts" + # The directory those modules are copied to. + SCRIPT_MODULES_PATH = UTILS + "/script_modules" # The name of the output directory. BUILD_DIR = "build" # The name of the static files directory. diff --git a/packages/reflex-base/src/reflex_base/js/__init__.py b/packages/reflex-base/src/reflex_base/js/__init__.py index 040549e7c6a..c19ffa47831 100644 --- a/packages/reflex-base/src/reflex_base/js/__init__.py +++ b/packages/reflex-base/src/reflex_base/js/__init__.py @@ -1,5 +1,25 @@ """Run JavaScript in the browser through a defined API.""" -from reflex_base.js.env import ScriptEnvVar, env +from typing import TYPE_CHECKING -__all__ = ["ScriptEnvVar", "env"] +from reflex_base.utils import lazy_loader + +if TYPE_CHECKING: + from .environment import ScriptEnvVar, env + from .modules import JsFunction + + __all__ = ["JsFunction", "ScriptEnvVar", "env"] + +# Lazy, so `import reflex`, which installs the module finder, does not load the +# event and var systems. +_SUBMODULES: set[str] = {"environment", "finder", "modules"} +_SUBMOD_ATTRS: lazy_loader.SubmodAttrsType = { + "environment": ["ScriptEnvVar", "env"], + "modules": ["JsFunction"], +} + +__getattr__, __dir__, __all__ = lazy_loader.attach( + __name__, + submodules=_SUBMODULES, + submod_attrs=_SUBMOD_ATTRS, +) diff --git a/packages/reflex-base/src/reflex_base/js/env.py b/packages/reflex-base/src/reflex_base/js/environment.py similarity index 100% rename from packages/reflex-base/src/reflex_base/js/env.py rename to packages/reflex-base/src/reflex_base/js/environment.py diff --git a/packages/reflex-base/src/reflex_base/js/finder.py b/packages/reflex-base/src/reflex_base/js/finder.py new file mode 100644 index 00000000000..25adaa84703 --- /dev/null +++ b/packages/reflex-base/src/reflex_base/js/finder.py @@ -0,0 +1,89 @@ +"""Import JavaScript modules from Python. + +``from my_app.scripts.party import celebrate`` finds ``my_app/scripts/party.js`` +(or ``.ts``) when no Python module of that name exists. This module only uses +the standard library: ``import reflex`` installs the finder, and loading the +event system is left to the first JavaScript module imported. +""" + +from __future__ import annotations + +import importlib.abc +import importlib.machinery +import sys +from collections.abc import Sequence +from pathlib import Path +from types import ModuleType + +# The extensions of the files a JavaScript module may be written in. +SCRIPT_EXTENSIONS = (".js", ".ts") + + +class _ScriptLoader(importlib.abc.Loader): + """Execute a JavaScript module by exposing its exports as functions.""" + + def __init__(self, path: Path): + """Create the loader. + + Args: + path: The JavaScript file. + """ + self.path = path + + def create_module(self, spec: importlib.machinery.ModuleSpec) -> None: + """Use the default module creation. + + Args: + spec: The module spec. + """ + + def exec_module(self, module: ModuleType) -> None: + """Populate the module with one function per export. + + Args: + module: The module to populate. + """ + # Imported here: the finder loads with `import reflex`, which must not + # pay for the event system until a JavaScript module is imported. + from reflex_base.js.modules import load_module + + load_module(module, self.path) + + +class ScriptFinder(importlib.abc.MetaPathFinder): + """Find a JavaScript file where no Python module of the name exists.""" + + def find_spec( + self, + fullname: str, + path: Sequence[str] | None, + target: ModuleType | None = None, + ) -> importlib.machinery.ModuleSpec | None: + """Find the JavaScript file for a submodule. + + Args: + fullname: The dotted module name. + path: The search path of the parent package; None for a top-level + module, which is never a JavaScript module. + target: Unused. + + Returns: + The spec of the JavaScript module, or None. + """ + if path is None: + return None + name = fullname.rpartition(".")[2] + for entry in path: + for extension in SCRIPT_EXTENSIONS: + file = Path(entry, name + extension) + if file.is_file(): + return importlib.machinery.ModuleSpec( + fullname, _ScriptLoader(file), origin=str(file) + ) + return None + + +def install() -> None: + """Let ``import`` find JavaScript modules, after every other finder.""" + if not any(isinstance(finder, ScriptFinder) for finder in sys.meta_path): + sys.meta_path.append(ScriptFinder()) diff --git a/packages/reflex-base/src/reflex_base/js/modules.py b/packages/reflex-base/src/reflex_base/js/modules.py new file mode 100644 index 00000000000..433c860db60 --- /dev/null +++ b/packages/reflex-base/src/reflex_base/js/modules.py @@ -0,0 +1,204 @@ +"""JavaScript modules imported from Python, and calls to their exports. + +Each exported function of an imported module becomes a ``JsFunction``. Calling +it returns an event that runs the function in the browser by name: the compiler +copies the module into ``.web`` and re-exports it from ``utils/scripts.js``, so +only the module, the function name and the JSON arguments are sent, and nothing +is evaluated. +""" + +from __future__ import annotations + +import dataclasses +import inspect +import keyword +import re +from pathlib import Path +from types import ModuleType +from typing import Any + +from reflex_base.event import EventHandler, EventSpec, server_side +from reflex_base.registry import RegistrationContext +from reflex_base.utils.format import format_event_handler + +# The name of the frontend event that calls a function of a module. +CALL_JS = "_call_js" + +_COMMENT_RE = re.compile(r"/\*.*?\*/|//[^\n]*", re.DOTALL) +_DECLARED_EXPORT_RE = re.compile( + r"^[ \t]*export[ \t]+(?:async[ \t]+)?" + r"(?:function\b[ \t]*\*?[ \t]*|(?:const|let|var)[ \t]+)([\w$]+)", + re.MULTILINE, +) +_EXPORT_LIST_RE = re.compile(r"^[ \t]*export[ \t]*\{([^}]*)\}", re.MULTILINE) +_SPECIFIER_RE = re.compile(r"""(?:\bfrom|\bimport)[ \t]*\(?[ \t]*["']([^"'\n]+)["']""") +# Specifiers that do not name an npm package. +_LOCAL_PREFIXES = (".", "/", "$/", "node:", "http://", "https://") + + +def _strip_comments(source: str) -> str: + """Remove the comments of a JavaScript source. + + Args: + source: The source. + + Returns: + The source without comments. + """ + return _COMMENT_RE.sub("", source) + + +def find_exports(source: str) -> list[str]: + """Find the named exports of a JavaScript module. + + Covers declared exports (``export function f``, ``export async function``, + ``export const f``) and export lists (``export { a, b as c }``, also when + re-exported ``from`` another module). A default export has no name to + import in Python and is left out. + + Args: + source: The module source. + + Returns: + The export names, in the order they appear, without duplicates. + """ + source = _strip_comments(source) + found: dict[int, list[str]] = {} + for match in _DECLARED_EXPORT_RE.finditer(source): + found[match.start()] = [match.group(1)] + for match in _EXPORT_LIST_RE.finditer(source): + found[match.start()] = [ + spec.split(" as ")[-1].strip() + for spec in match.group(1).split(",") + if spec.strip() + ] + return list( + dict.fromkeys( + name + for start in sorted(found) + for name in found[start] + if name != "default" + ) + ) + + +def find_package_imports(source: str) -> list[str]: + """Find the npm packages a JavaScript module imports. + + Args: + source: The module source. + + Returns: + The import specifiers that name a package, such as ``canvas-confetti`` + or ``@scope/pkg/sub``, without duplicates. + """ + return list( + dict.fromkeys( + specifier + for specifier in _SPECIFIER_RE.findall(_strip_comments(source)) + if not specifier.startswith(_LOCAL_PREFIXES) + ) + ) + + +def module_key(name: str) -> str: + """Get the name ``utils/scripts.js`` exports a module under. + + ``$`` cannot occur in a Python module name, so two modules never share one. + + Args: + name: The dotted Python module name. + + Returns: + A JavaScript identifier. + """ + return name.replace(".", "$") + + +def _call_js(module: str, function: str, args: list) -> None: + """The signature of the event that calls a function of a module. + + Args: + module: The module key. + function: The exported name. + args: The arguments. + """ + + +_CALL_JS_SIGNATURE = inspect.signature(_call_js) + + +@dataclasses.dataclass(frozen=True, slots=True) +class JsFunction: + """An exported function of a JavaScript module imported from Python.""" + + # The dotted name of the module. + module: str + # The exported name. + name: str + + def __call__(self, *args: Any) -> EventSpec: + """Build the event that calls the function in the browser. + + An event handler argument arrives as its full name, which the + function can dispatch with ``env.addEvents([env.event(name, payload)])``. + + Args: + *args: The arguments: values, Vars or event handlers. Values are + sent as JSON. + + Returns: + The event. + """ + return server_side( + CALL_JS, + _CALL_JS_SIGNATURE, + module=module_key(self.module), + function=self.name, + args=[ + format_event_handler(arg) if isinstance(arg, EventHandler) else arg + for arg in args + ], + ) + + +def load_module(module: ModuleType, path: Path) -> None: + """Expose the exports of a JavaScript file on a Python module. + + Args: + module: The module being imported. + path: The JavaScript file. + """ + exports = find_exports(path.read_text(encoding="utf-8")) + RegistrationContext.ensure_context().script_modules[module.__name__] = path + for name in exports: + setattr(module, name, JsFunction(module.__name__, name)) + setattr(module, "__all__", exports) # noqa: B010 + + +def render_stub(path: Path, exports: list[str]) -> str: + """Render a type stub for a JavaScript module, so type checkers see it. + + Exports that are not Python identifiers cannot be imported and are left + out. The arguments are untyped. + + Args: + path: The JavaScript file. + exports: Its export names. + + Returns: + The ``.pyi`` source. + """ + lines = [ + f"# Generated by Reflex from {path.name}; rewritten when it changes.", + "from typing import Any", + "", + "from reflex_base.event import EventSpec", + "", + ] + lines.extend( + f"def {name}(*args: Any) -> EventSpec: ..." + for name in exports + if name.isidentifier() and not keyword.iskeyword(name) + ) + return "\n".join(lines) + "\n" diff --git a/packages/reflex-base/src/reflex_base/registry.py b/packages/reflex-base/src/reflex_base/registry.py index 05fb5c694da..97d3eb014f5 100644 --- a/packages/reflex-base/src/reflex_base/registry.py +++ b/packages/reflex-base/src/reflex_base/registry.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from collections.abc import Callable, Sequence + from pathlib import Path from reflex.app import App from reflex.state import BaseState @@ -73,6 +74,10 @@ class RegistrationContext(BaseContext): _explicit_bundled_libraries: dict[str, None] = dataclasses.field( default_factory=dict, repr=False ) + # JavaScript modules imported from Python, by module name. + script_modules: dict[str, Path] = dataclasses.field( + default_factory=dict, repr=False + ) _app: App | None = dataclasses.field(default=None, repr=False) _memoized_event_triggers: dict[tuple[str, int], tuple[Any, Var]] = ( dataclasses.field(default_factory=dict, repr=False) diff --git a/packages/reflex-frontend/src/utils/scripts.js b/packages/reflex-frontend/src/utils/scripts.js new file mode 100644 index 00000000000..844e9c509fc --- /dev/null +++ b/packages/reflex-frontend/src/utils/scripts.js @@ -0,0 +1,3 @@ +// The JavaScript modules an app imports from Python, by module key. The +// compiler rewrites this file; an app that imports none keeps it empty. +export {}; diff --git a/packages/reflex-frontend/src/utils/state.js b/packages/reflex-frontend/src/utils/state.js index ce6c9be6d27..b58bc11646c 100644 --- a/packages/reflex-frontend/src/utils/state.js +++ b/packages/reflex-frontend/src/utils/state.js @@ -562,6 +562,27 @@ const runScript = (code, addEvents, socket, navigate, params) => { ); }; +/** + * Call an exported function of a JavaScript module the app imports from + * Python. The modules are bundled, so nothing is evaluated: the event names the + * module and the function, and carries the arguments as JSON. + * + * Loaded on first use, so an app that never calls one does not fetch the chunk. + * @param {{module: string, function: string, args: any[]}} payload The call. + * @returns {Promise} The function's result. + */ +const callModuleFunction = async ({ module, function: name, args }) => { + /** @type {Record>} */ + const modules = await import("$/utils/scripts"); + const fn = Object.hasOwn(modules, module) ? modules[module][name] : undefined; + if (typeof fn !== "function") { + throw new TypeError( + `${module.replaceAll("$", ".")}.${name} is not a function exported by a JavaScript module this app imports.`, + ); + } + return await fn(...args); +}; + /** * Invoke an event's result callback, if it declared one. * @@ -829,6 +850,18 @@ export const applyEvent = async (event, socket, navigate, params) => { return; } + if (event.name === "_call_js") { + try { + await callModuleFunction(event.payload); + } catch (e) { + console.log("_call_js", e); + if (window?.onerror) { + window.onerror(e.message, null, null, null, e); + } + } + return; + } + if (event.name === "_call_script" || event.name === "_call_function") { try { const eval_result = diff --git a/packages/reflex-frontend/tests/call_js.test.js b/packages/reflex-frontend/tests/call_js.test.js new file mode 100644 index 00000000000..c2ccf6a5919 --- /dev/null +++ b/packages/reflex-frontend/tests/call_js.test.js @@ -0,0 +1,75 @@ +/** + * `_call_js` calls an exported function of a JavaScript module an app imports + * from Python. The modules are bundled and re-exported by utils/scripts.js, so + * the event carries only names and JSON arguments, and nothing is evaluated. + */ +import { afterEach, expect, test, vi } from "vitest"; + +import { applyEvent } from "$/utils/state"; + +const seen = []; + +vi.mock("$/utils/scripts", () => ({ + app$scripts$party: { + celebrate: (count, id) => seen.push(["celebrate", count, id]), + later: async (value) => { + await Promise.resolve(); + seen.push(["later", value]); + }, + notAFunction: 3, + }, +})); + +afterEach(() => { + seen.length = 0; + window.onerror = null; +}); + +const callJs = (module, fn, args) => + applyEvent( + { name: "_call_js", payload: { module, function: fn, args } }, + null, + null, + null, + ); + +test("calls the named function with the arguments", async () => { + await callJs("app$scripts$party", "celebrate", [200, "results"]); + expect(seen).toEqual([["celebrate", 200, "results"]]); +}); + +test("waits for an async function before the next event", async () => { + await callJs("app$scripts$party", "later", ["done"]); + expect(seen).toEqual([["later", "done"]]); +}); + +test("reports a missing module, function or non-function export", async () => { + const onerror = vi.fn(); + window.onerror = onerror; + await callJs("app$scripts$party", "missing", []); + await callJs("app$scripts$party", "notAFunction", []); + await callJs("app$other", "celebrate", []); + expect(onerror.mock.calls.map((call) => [call[0], call[4].name])).toEqual([ + [ + "app.scripts.party.missing is not a function exported by a JavaScript module this app imports.", + "TypeError", + ], + [ + "app.scripts.party.notAFunction is not a function exported by a JavaScript module this app imports.", + "TypeError", + ], + [ + "app.other.celebrate is not a function exported by a JavaScript module this app imports.", + "TypeError", + ], + ]); + expect(seen).toEqual([]); +}); + +test("a function that throws is reported, not raised", async () => { + const onerror = vi.fn(); + window.onerror = onerror; + await callJs("app$scripts$party", "celebrate", null); + expect(onerror).toHaveBeenCalledTimes(1); + expect(onerror.mock.calls[0][4]).toBeInstanceOf(TypeError); +}); diff --git a/packages/reflex-frontend/tests/public_api.test.js b/packages/reflex-frontend/tests/public_api.test.js index 40838591a03..03ff0aad439 100644 --- a/packages/reflex-frontend/tests/public_api.test.js +++ b/packages/reflex-frontend/tests/public_api.test.js @@ -83,6 +83,7 @@ const PUBLIC_API = { ], "$/utils/react-theme": ["ThemeProvider", "useTheme"], "$/utils/script_env": ["env"], + "$/utils/scripts": [], "$/utils/helpers/array": ["default"], "$/utils/helpers/datetime": ["default"], "$/utils/helpers/debounce": ["default"], diff --git a/pyi_hashes.json b/pyi_hashes.json index 3538f32191a..6543e41e1fd 100644 --- a/pyi_hashes.json +++ b/pyi_hashes.json @@ -118,7 +118,7 @@ "packages/reflex-components-recharts/src/reflex_components_recharts/polar.pyi": "adc4db1a3ca7cd292b099ec4d97a4346", "packages/reflex-components-recharts/src/reflex_components_recharts/recharts.pyi": "f2a9cf6db58d169289fbc76de6813300", "packages/reflex-components-sonner/src/reflex_components_sonner/toast.pyi": "f170ac685b6ba5892370166c80684db3", - "reflex/__init__.pyi": "9e15759df6266f047709e2a1325c2cb4", + "reflex/__init__.pyi": "060fa790e6c7b1acc4ecae5804ebfe73", "reflex/components/__init__.pyi": "9facd05a776d0641432696bbf8e34388", "reflex/experimental/memo.pyi": "0fc8eae24cd0def5ea9d8d66ce87206e" } diff --git a/reflex/__init__.py b/reflex/__init__.py index 035dda21e53..127a3cbd34a 100644 --- a/reflex/__init__.py +++ b/reflex/__init__.py @@ -84,6 +84,7 @@ from __future__ import annotations +from reflex_base.js import finder as _js_finder from reflex_base.utils import lazy_loader from reflex_components_radix.mappings import RADIX_MAPPING @@ -257,5 +258,9 @@ ) +# Let app code import the JavaScript modules beside it. +_js_finder.install() + + def __getattr__(name: str): return getattr(name) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index 83d72e5dd12..bbe645ed8e5 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1590,6 +1590,8 @@ def add_save_task( ), utils._compile_bundled_libraries(), ]) + script_module_outputs, script_module_imports = utils.compile_script_modules() + compile_results.extend(script_module_outputs) progress.advance(task) compile_results.append(compile_app_root(app_root, hydrate_fallback_export)) @@ -1611,7 +1613,9 @@ def add_save_task( log.timing(logger, "Install Frontend Packages"), otel.span("reflex.compile.install_frontend_packages"), ): - app._get_frontend_packages(all_imports) + app._get_frontend_packages( + utils.merge_imports(all_imports, script_module_imports) + ) frontend_skeleton.update_react_router_config( prerender_routes=prerender_routes, diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index e0b490938b3..36531ebbd1d 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -32,6 +32,13 @@ MemoParamKind, ) from reflex_base.constants.state import FIELD_MARKER +from reflex_base.js.finder import SCRIPT_EXTENSIONS +from reflex_base.js.modules import ( + find_exports, + find_package_imports, + module_key, + render_stub, +) from reflex_base.registry import RegistrationContext from reflex_base.style import Style from reflex_base.utils import format, imports, memo_paths, serializers, types @@ -300,6 +307,55 @@ def _compile_bundled_libraries() -> tuple[str, str]: return constants.Dirs.BUNDLED_LIBRARIES, format.json_dumps(bundled_libraries) +def compile_script_modules() -> tuple[ + list[tuple[str, str]], dict[str, list[ImportVar]] +]: + """Compile the JavaScript modules the app imports from Python. + + Each module is copied under ``utils/script_modules`` at the path of its + Python name, with the script files beside it so its relative imports + resolve, and ``utils/scripts.js`` re-exports it under its module key. A + type stub is written next to the module, for type checkers. + + Returns: + The (path, code) outputs, and the npm packages the modules import, to + install. + """ + web_dir = get_web_dir() + outputs: list[tuple[str, str]] = [] + package_imports: dict[str, list[ImportVar]] = {} + index: list[str] = [] + copied: set[Path] = set() + modules = RegistrationContext.ensure_context().script_modules + for name, path in sorted(modules.items()): + package_dir = Path(*name.split(".")[:-1]) + for file in sorted(path.parent.iterdir()): + if file.suffix not in SCRIPT_EXTENSIONS or file in copied: + continue + copied.add(file) + source = file.read_text(encoding="utf-8") + outputs.append(( + str( + web_dir + / constants.Dirs.SCRIPT_MODULES_PATH + / package_dir + / file.name + ), + source, + )) + for specifier in find_package_imports(source): + package_imports[specifier] = [ImportVar(tag=None, render=False)] + target = f"$/{constants.Dirs.SCRIPT_MODULES_PATH}/{package_dir.as_posix()}/{path.name}" + index.append(f'export * as {module_key(name)} from "{target}";') + exports = find_exports(path.read_text(encoding="utf-8")) + outputs.append((str(path.with_suffix(".pyi")), render_stub(path, exports))) + outputs.append(( + str(web_dir / (constants.Dirs.SCRIPTS_PATH + constants.Ext.JS)), + "\n".join(index) + "\n" if index else "export {};\n", + )) + return outputs, package_imports + + def _restore_bundled_libraries() -> bool: """Restore the registry saved by the most recent compile. diff --git a/tests/integration/tests_playwright/test_call_script.py b/tests/integration/tests_playwright/test_call_script.py index 7c7995bb9dc..0af78422f7f 100644 --- a/tests/integration/tests_playwright/test_call_script.py +++ b/tests/integration/tests_playwright/test_call_script.py @@ -38,6 +38,21 @@ def CallScript(): external_scripts = inline_scripts.replace("inline", "external") + import importlib + + # A JavaScript module beside the app, imported from Python. + Path(__file__).with_name("page_script.js").write_text( + 'import { env } from "$/utils/script_env";\n' + "export function setText(id, text) {\n" + " document.getElementById(id).textContent = text;\n" + "}\n" + "export async function report(handler, value) {\n" + " await Promise.resolve();\n" + " env.addEvents([env.event(handler, { value })]);\n" + "}\n" + ) + page_script = importlib.import_module(__name__.rpartition(".")[0] + ".page_script") + class CallScriptState(rx.State): results: rx.Field[list[str | dict | list | None]] = rx.field([]) inline_counter: rx.Field[int] = rx.field(0) @@ -200,6 +215,10 @@ def set_value(self, value: str): def set_env_result(self, value: str): self.env_result = value + @rx.event + def js_module_backend(self): + return page_script.setText("js_module_text", "from the backend") + @rx.event def env_var_backend(self): return rx.run_script( @@ -389,6 +408,24 @@ def index(): on_click=CallScriptState.env_var_backend, id="env_var_backend", ), + rx.text("", id="js_module_text"), + rx.button( + "js_module_frontend", + on_click=page_script.setText("js_module_text", "from a trigger"), + id="js_module_frontend", + ), + rx.button( + "js_module_backend", + on_click=CallScriptState.js_module_backend, + id="js_module_backend", + ), + rx.button( + "js_module_report", + on_click=page_script.report( + CallScriptState.set_env_result, "from a module" + ), + id="js_module_report", + ), ) @@ -578,3 +615,21 @@ def test_script_env(call_script_page: Page): call_script_page.locator("#env_var_backend").click() expect(env_result).to_have_value("env var backend") + + +def test_js_module(call_script_page: Page): + """Functions of a JavaScript module imported from Python run by name. + + Args: + call_script_page: Playwright page. + """ + text = call_script_page.locator("#js_module_text") + + call_script_page.locator("#js_module_frontend").click() + expect(text).to_have_text("from a trigger") + + call_script_page.locator("#js_module_backend").click() + expect(text).to_have_text("from the backend") + + call_script_page.locator("#js_module_report").click() + expect(call_script_page.locator("#env_result")).to_have_value("from a module") diff --git a/tests/units/compiler/test_compiler_utils.py b/tests/units/compiler/test_compiler_utils.py index 5e2b83b4d17..44f3da2d05c 100644 --- a/tests/units/compiler/test_compiler_utils.py +++ b/tests/units/compiler/test_compiler_utils.py @@ -808,3 +808,97 @@ def test_document_root_controls_preserve_no_id_and_page_refs(): assert not document_root._get_all_hooks() assert page_script._get_all_hooks() + + +def test_compile_script_modules( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + clean_registration_context: RegistrationContext, +): + """Imported JavaScript modules are copied with their siblings and indexed. + + Args: + tmp_path: pytest tmp_path fixture. + monkeypatch: pytest monkeypatch fixture. + clean_registration_context: a fresh registry of imported modules. + """ + web_dir = tmp_path / ".web" + monkeypatch.setattr(utils, "get_web_dir", lambda: web_dir) + scripts = tmp_path / "app" / "scripts" + scripts.mkdir(parents=True) + party = scripts / "party.js" + party.write_text( + 'import confetti from "canvas-confetti";\n' + 'import { describe } from "./helpers.js";\n' + "export function celebrate(count) {}\n" + ) + helpers = scripts / "helpers.js" + helpers.write_text('import dayjs from "dayjs";\nexport const describe = 1;\n') + (scripts / "notes.txt").write_text("not a script") + typed = tmp_path / "app" / "typed.ts" + typed.write_text("export function typed(n: number) {}\n") + clean_registration_context.script_modules.update({ + "app.typed": typed, + "app.scripts.party": party, + }) + + outputs, package_imports = utils.compile_script_modules() + + modules_dir = web_dir / "utils" / "script_modules" + assert outputs == [ + (str(modules_dir / "app" / "scripts" / "helpers.js"), helpers.read_text()), + (str(modules_dir / "app" / "scripts" / "party.js"), party.read_text()), + ( + str(scripts / "party.pyi"), + ( + "# Generated by Reflex from party.js; rewritten when it changes.\n" + "from typing import Any\n" + "\n" + "from reflex_base.event import EventSpec\n" + "\n" + "def celebrate(*args: Any) -> EventSpec: ...\n" + ), + ), + (str(modules_dir / "app" / "typed.ts"), typed.read_text()), + ( + str(tmp_path / "app" / "typed.pyi"), + ( + "# Generated by Reflex from typed.ts; rewritten when it changes.\n" + "from typing import Any\n" + "\n" + "from reflex_base.event import EventSpec\n" + "\n" + "def typed(*args: Any) -> EventSpec: ...\n" + ), + ), + ( + str(web_dir / "utils" / "scripts.js"), + ( + 'export * as app$scripts$party from "$/utils/script_modules/app/scripts/party.js";\n' + 'export * as app$typed from "$/utils/script_modules/app/typed.ts";\n' + ), + ), + ] + assert package_imports == { + "canvas-confetti": [rx.ImportVar(tag=None, render=False)], + "dayjs": [rx.ImportVar(tag=None, render=False)], + } + + +def test_compile_script_modules_without_modules( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + clean_registration_context: RegistrationContext, +): + """An app that imports no JavaScript module gets an empty index. + + Args: + tmp_path: pytest tmp_path fixture. + monkeypatch: pytest monkeypatch fixture. + clean_registration_context: an empty registry. + """ + monkeypatch.setattr(utils, "get_web_dir", lambda: tmp_path) + assert utils.compile_script_modules() == ( + [(str(tmp_path / "utils" / "scripts.js"), "export {};\n")], + {}, + ) diff --git a/tests/units/reflex_base/js/test_modules.py b/tests/units/reflex_base/js/test_modules.py new file mode 100644 index 00000000000..51ad9362c16 --- /dev/null +++ b/tests/units/reflex_base/js/test_modules.py @@ -0,0 +1,184 @@ +"""Tests for importing JavaScript modules from Python.""" + +import importlib +import sys +from collections.abc import Generator +from pathlib import Path + +import pytest +from reflex_base.event import EventSpec +from reflex_base.js.finder import ScriptFinder +from reflex_base.js.modules import ( + JsFunction, + find_exports, + find_package_imports, + module_key, + render_stub, +) +from reflex_base.registry import RegistrationContext +from reflex_base.vars.base import LiteralVar, Var + +import reflex as rx + +SOURCE = """ +import confetti from "canvas-confetti"; +import { describe } from "./helpers.js"; +import { env } from "$/utils/script_env"; +import "@scope/pkg/style.css"; +// export function commentedOut() {} +/* export const alsoCommentedOut = 1; */ +export function celebrate(count) {} +export async function load() {} +export function* steps() {} +export const arrow = () => 1; +function hidden() {} +export { hidden, describe as describeIt }; +export { other as reexported } from "lodash/fp"; +export default function () {} +const later = await import("dayjs"); +""" + + +def test_find_exports(): + assert find_exports(SOURCE) == [ + "celebrate", + "load", + "steps", + "arrow", + "hidden", + "describeIt", + "reexported", + ] + + +def test_find_package_imports(): + assert find_package_imports(SOURCE) == [ + "canvas-confetti", + "@scope/pkg/style.css", + "lodash/fp", + "dayjs", + ] + + +def test_module_key_cannot_collide(): + assert module_key("app.scripts.party") == "app$scripts$party" + assert module_key("app.scripts__party") != module_key("app__scripts.party") + + +def test_calling_a_function_builds_the_call_js_event(): + spec = JsFunction("app.scripts.party", "celebrate")(200, "results") + assert isinstance(spec, EventSpec) + assert str(LiteralVar.create(spec)) == ( + '(ReflexEvent("_call_js", ({ ["module"] : "app$scripts$party", ' + '["function"] : "celebrate", ["args"] : [200, "results"] }), ({ })))' + ) + + +def test_an_event_handler_argument_is_sent_as_its_name(): + class JsHandlerState(rx.State): + @rx.event + def done(self, value: str): + """Receive a value. + + Args: + value: The value. + """ + + spec = JsFunction("app.scripts.party", "report")(JsHandlerState.done, "x") + name = f"{JsHandlerState.get_full_name()}.done" + assert str(LiteralVar.create(spec)) == ( + '(ReflexEvent("_call_js", ({ ["module"] : "app$scripts$party", ' + f'["function"] : "report", ["args"] : ["{name}", "x"] }}), ({{ }})))' + ) + + +def test_var_arguments_are_compiled_into_the_event(): + count = Var(_js_expr="count").to(int) + spec = JsFunction("app.scripts.party", "celebrate")(count) + literal = LiteralVar.create(spec) + assert str(literal) == ( + '(ReflexEvent("_call_js", ({ ["module"] : "app$scripts$party", ' + '["function"] : "celebrate", ["args"] : [count] }), ({ })))' + ) + + +def test_render_stub_skips_names_python_cannot_import(): + assert render_stub(Path("party.js"), ["celebrate", "$private", "class"]) == ( + "# Generated by Reflex from party.js; rewritten when it changes.\n" + "from typing import Any\n" + "\n" + "from reflex_base.event import EventSpec\n" + "\n" + "def celebrate(*args: Any) -> EventSpec: ...\n" + ) + + +@pytest.fixture +def script_package( + tmp_path: Path, clean_registration_context: RegistrationContext +) -> Generator[Path, None, None]: + """Put a package with a JavaScript module on sys.path. + + Args: + tmp_path: pytest tmp_path fixture. + clean_registration_context: a fresh registry for the imported module. + + Yields: + The package directory. + """ + package = tmp_path / "js_pkg" + package.mkdir() + (package / "__init__.py").write_text("") + (package / "party.js").write_text(SOURCE) + (package / "typed.ts").write_text("export function typed(n: number) {}\n") + sys.path.insert(0, str(tmp_path)) + yield package + sys.path.remove(str(tmp_path)) + for name in [n for n in sys.modules if n == "js_pkg" or n.startswith("js_pkg.")]: + del sys.modules[name] + + +def test_importing_a_module_exposes_its_exports( + script_package: Path, clean_registration_context: RegistrationContext +): + assert any(isinstance(finder, ScriptFinder) for finder in sys.meta_path) + party = importlib.import_module("js_pkg.party") + assert party.__all__ == find_exports(SOURCE) + assert party.celebrate == JsFunction("js_pkg.party", "celebrate") + assert party.__spec__ is not None + assert party.__spec__.origin == str(script_package / "party.js") + typed = importlib.import_module("js_pkg.typed") + assert typed.typed == JsFunction("js_pkg.typed", "typed") + assert clean_registration_context.script_modules == { + "js_pkg.party": script_package / "party.js", + "js_pkg.typed": script_package / "typed.ts", + } + + +def test_a_missing_module_still_fails_to_import(script_package: Path): + with pytest.raises(ModuleNotFoundError): + importlib.import_module("js_pkg.missing") + + +def test_a_python_module_wins_over_a_script_of_the_same_name(script_package: Path): + (script_package / "both.py").write_text("VALUE = 'python'\n") + (script_package / "both.js").write_text("export function VALUE() {}\n") + assert importlib.import_module("js_pkg.both").VALUE == "python" + + +def test_a_function_is_an_event_trigger(script_package: Path): + from js_pkg.party import celebrate # pyright: ignore[reportMissingImports] + + called = rx.button("go", on_click=celebrate(1, "out")) + assert str(LiteralVar.create(called.event_triggers["on_click"])) == ( + '((_e) => (addEvents([(ReflexEvent("_call_js", ({ ["module"] : ' + '"js_pkg$party", ["function"] : "celebrate", ["args"] : [1, "out"] }), ' + "({ })))], [_e], ({ }))))" + ) + # Like an event handler, an uncalled function takes the trigger's arguments. + uncalled = rx.input(on_change=celebrate) + assert str(LiteralVar.create(uncalled.event_triggers["on_change"])) == ( + '((_e) => (addEvents([(ReflexEvent("_call_js", ({ ["module"] : ' + '"js_pkg$party", ["function"] : "celebrate", ' + '["args"] : [_e?.["target"]?.["value"]] }), ({ })))], [_e], ({ }))))' + ) From c0ebd9c759f099e5b40c0a1396e45238789449a5 Mon Sep 17 00:00:00 2001 From: Farhan Date: Fri, 20 Nov 2026 21:35:21 +0000 Subject: [PATCH 02/16] Read JS modules with a tokenizer, copy their import graph, keep import reflex cheap Review fixes for the JavaScript module import: - Exports and imports are read from tokens (reflex_base/js/source.py) instead of regular expressions over the source. Text inside strings, template literals, regular expressions and comments is never read as code, so a string such as "/api/*" no longer hides the exports after it. Every name of `export const a = ..., b = ...` is found, as are destructured exports, TS annotations with commas, and `"string" as name` export lists. Imports are found wherever line breaks fall, including dynamic `import("x")`. - The compiler copies the module and the local files it imports, followed transitively and resolved as the bundler does (extensionless, index files, `.js` naming a `.ts`), at their paths from the directory of the module's package, instead of the script files beside it. Nested helpers and imported CSS now resolve, and unused siblings are no longer bundled or installed. A file outside the package's directory is a clear error. - RegistrationContext.fork() copies script_modules, so an app built in a fork still compiles the modules imported before it. - The finder no longer imports pathlib, importlib.abc or typing, which `import reflex` does not load otherwise (CodSpeed: test_import_reflex +1.6ms); a test pins it. - Test docstrings. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- docs/api-reference/browser_javascript.md | 5 +- .../src/reflex_base/js/__init__.py | 4 +- .../reflex-base/src/reflex_base/js/finder.py | 49 +- .../reflex-base/src/reflex_base/js/modules.py | 68 +-- .../reflex-base/src/reflex_base/js/source.py | 522 ++++++++++++++++++ .../reflex-base/src/reflex_base/registry.py | 1 + reflex/compiler/utils.py | 57 +- tests/units/compiler/test_compiler_utils.py | 61 +- tests/units/reflex_base/js/test_modules.py | 62 ++- tests/units/reflex_base/js/test_source.py | 111 ++++ tests/units/reflex_base/test_registry.py | 16 + 11 files changed, 822 insertions(+), 134 deletions(-) create mode 100644 packages/reflex-base/src/reflex_base/js/source.py create mode 100644 tests/units/reflex_base/js/test_source.py diff --git a/docs/api-reference/browser_javascript.md b/docs/api-reference/browser_javascript.md index ac2431a9d6a..d82e8d499a7 100644 --- a/docs/api-reference/browser_javascript.md +++ b/docs/api-reference/browser_javascript.md @@ -229,8 +229,9 @@ rx.button("Report", on_click=report(PartyState.done, "clicked")) through `env.addEvents`. - Packages the module imports, such as `canvas-confetti`, are installed with the app. Pin a version with `frontend_packages` in `rxconfig.py`. -- The `.js` and `.ts` files beside the module are bundled with it, so it can - import them with relative paths such as `./helpers.js`. +- Local files the module imports with relative paths, such as `./helpers.js` + or `./party.css`, are bundled with it, and so are the files they import. They + must be inside the directory of your app's package. - Reflex writes a type stub (`party.pyi`) next to the module when it compiles the app, so type checkers resolve the import. diff --git a/packages/reflex-base/src/reflex_base/js/__init__.py b/packages/reflex-base/src/reflex_base/js/__init__.py index c19ffa47831..788c2b521b1 100644 --- a/packages/reflex-base/src/reflex_base/js/__init__.py +++ b/packages/reflex-base/src/reflex_base/js/__init__.py @@ -1,9 +1,9 @@ """Run JavaScript in the browser through a defined API.""" -from typing import TYPE_CHECKING - from reflex_base.utils import lazy_loader +# Not imported from typing, which `import reflex` does not load otherwise. +TYPE_CHECKING = False if TYPE_CHECKING: from .environment import ScriptEnvVar, env from .modules import JsFunction diff --git a/packages/reflex-base/src/reflex_base/js/finder.py b/packages/reflex-base/src/reflex_base/js/finder.py index 25adaa84703..7cee3ff91a2 100644 --- a/packages/reflex-base/src/reflex_base/js/finder.py +++ b/packages/reflex-base/src/reflex_base/js/finder.py @@ -1,28 +1,38 @@ """Import JavaScript modules from Python. ``from my_app.scripts.party import celebrate`` finds ``my_app/scripts/party.js`` -(or ``.ts``) when no Python module of that name exists. This module only uses -the standard library: ``import reflex`` installs the finder, and loading the -event system is left to the first JavaScript module imported. +(or ``.ts``) when no Python module of that name exists. + +``import reflex`` installs the finder, so this module stays as cheap to import +as it can: ``os.path`` instead of ``pathlib``, and a finder and loader that +implement the import protocol without the ``importlib.abc`` base classes, and +no ``typing``. The event system loads with the first JavaScript module +imported. """ from __future__ import annotations -import importlib.abc -import importlib.machinery +import os.path import sys -from collections.abc import Sequence -from pathlib import Path -from types import ModuleType +from importlib.machinery import ModuleSpec + +# Not imported from typing, which `import reflex` does not load otherwise. +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Sequence + from importlib.abc import Loader as _LoaderBase + from types import ModuleType +else: + _LoaderBase = object # The extensions of the files a JavaScript module may be written in. SCRIPT_EXTENSIONS = (".js", ".ts") -class _ScriptLoader(importlib.abc.Loader): +class _ScriptLoader(_LoaderBase): """Execute a JavaScript module by exposing its exports as functions.""" - def __init__(self, path: Path): + def __init__(self, path: str): """Create the loader. Args: @@ -30,7 +40,7 @@ def __init__(self, path: Path): """ self.path = path - def create_module(self, spec: importlib.machinery.ModuleSpec) -> None: + def create_module(self, spec: ModuleSpec) -> None: """Use the default module creation. Args: @@ -45,12 +55,14 @@ def exec_module(self, module: ModuleType) -> None: """ # Imported here: the finder loads with `import reflex`, which must not # pay for the event system until a JavaScript module is imported. + from pathlib import Path + from reflex_base.js.modules import load_module - load_module(module, self.path) + load_module(module, Path(self.path)) -class ScriptFinder(importlib.abc.MetaPathFinder): +class ScriptFinder: """Find a JavaScript file where no Python module of the name exists.""" def find_spec( @@ -58,7 +70,7 @@ def find_spec( fullname: str, path: Sequence[str] | None, target: ModuleType | None = None, - ) -> importlib.machinery.ModuleSpec | None: + ) -> ModuleSpec | None: """Find the JavaScript file for a submodule. Args: @@ -75,11 +87,10 @@ def find_spec( name = fullname.rpartition(".")[2] for entry in path: for extension in SCRIPT_EXTENSIONS: - file = Path(entry, name + extension) - if file.is_file(): - return importlib.machinery.ModuleSpec( - fullname, _ScriptLoader(file), origin=str(file) - ) + # Not pathlib: see the module docstring. + file = os.path.join(entry, name + extension) # noqa: PTH118 + if os.path.isfile(file): # noqa: PTH113 + return ModuleSpec(fullname, _ScriptLoader(file), origin=file) return None diff --git a/packages/reflex-base/src/reflex_base/js/modules.py b/packages/reflex-base/src/reflex_base/js/modules.py index 433c860db60..377b9f35ded 100644 --- a/packages/reflex-base/src/reflex_base/js/modules.py +++ b/packages/reflex-base/src/reflex_base/js/modules.py @@ -12,76 +12,22 @@ import dataclasses import inspect import keyword -import re from pathlib import Path from types import ModuleType from typing import Any from reflex_base.event import EventHandler, EventSpec, server_side +from reflex_base.js.source import find_exports, find_imports from reflex_base.registry import RegistrationContext from reflex_base.utils.format import format_event_handler # The name of the frontend event that calls a function of a module. CALL_JS = "_call_js" -_COMMENT_RE = re.compile(r"/\*.*?\*/|//[^\n]*", re.DOTALL) -_DECLARED_EXPORT_RE = re.compile( - r"^[ \t]*export[ \t]+(?:async[ \t]+)?" - r"(?:function\b[ \t]*\*?[ \t]*|(?:const|let|var)[ \t]+)([\w$]+)", - re.MULTILINE, -) -_EXPORT_LIST_RE = re.compile(r"^[ \t]*export[ \t]*\{([^}]*)\}", re.MULTILINE) -_SPECIFIER_RE = re.compile(r"""(?:\bfrom|\bimport)[ \t]*\(?[ \t]*["']([^"'\n]+)["']""") # Specifiers that do not name an npm package. _LOCAL_PREFIXES = (".", "/", "$/", "node:", "http://", "https://") -def _strip_comments(source: str) -> str: - """Remove the comments of a JavaScript source. - - Args: - source: The source. - - Returns: - The source without comments. - """ - return _COMMENT_RE.sub("", source) - - -def find_exports(source: str) -> list[str]: - """Find the named exports of a JavaScript module. - - Covers declared exports (``export function f``, ``export async function``, - ``export const f``) and export lists (``export { a, b as c }``, also when - re-exported ``from`` another module). A default export has no name to - import in Python and is left out. - - Args: - source: The module source. - - Returns: - The export names, in the order they appear, without duplicates. - """ - source = _strip_comments(source) - found: dict[int, list[str]] = {} - for match in _DECLARED_EXPORT_RE.finditer(source): - found[match.start()] = [match.group(1)] - for match in _EXPORT_LIST_RE.finditer(source): - found[match.start()] = [ - spec.split(" as ")[-1].strip() - for spec in match.group(1).split(",") - if spec.strip() - ] - return list( - dict.fromkeys( - name - for start in sorted(found) - for name in found[start] - if name != "default" - ) - ) - - def find_package_imports(source: str) -> list[str]: """Find the npm packages a JavaScript module imports. @@ -92,13 +38,11 @@ def find_package_imports(source: str) -> list[str]: The import specifiers that name a package, such as ``canvas-confetti`` or ``@scope/pkg/sub``, without duplicates. """ - return list( - dict.fromkeys( - specifier - for specifier in _SPECIFIER_RE.findall(_strip_comments(source)) - if not specifier.startswith(_LOCAL_PREFIXES) - ) - ) + return [ + specifier + for specifier in find_imports(source) + if not specifier.startswith(_LOCAL_PREFIXES) + ] def module_key(name: str) -> str: diff --git a/packages/reflex-base/src/reflex_base/js/source.py b/packages/reflex-base/src/reflex_base/js/source.py new file mode 100644 index 00000000000..64a13e6dc79 --- /dev/null +++ b/packages/reflex-base/src/reflex_base/js/source.py @@ -0,0 +1,522 @@ +"""Read the exports and imports of a JavaScript or TypeScript module. + +A small tokenizer, not a parser: it knows enough of the language (strings, +template literals, regular expressions and comments) that text inside them is +never read as code, which is what a regular expression over the source gets +wrong. Exports and imports are then read from the tokens. +""" + +from __future__ import annotations + +import dataclasses +import re +from pathlib import Path + +# Files whose imports are followed; other local files (CSS, JSON) are copied. +SCRIPT_SOURCE_EXTENSIONS = (".js", ".ts", ".mjs", ".jsx", ".tsx") +# Tried in order for an import without an extension, as the bundler does. +_RESOLVE_EXTENSIONS = (".ts", ".tsx", ".js", ".jsx", ".mjs") + +_IDENTIFIER_RE = re.compile(r"[A-Za-z_$\u0080-￿][\w$\u0080-￿]*") +# A `/` after one of these starts a regular expression, not a division. +_PUNCT_BEFORE_REGEX = frozenset("(,=:[!&|?{};+-*%<>~^") +_KEYWORDS_BEFORE_REGEX = frozenset({ + "return", + "typeof", + "instanceof", + "in", + "of", + "new", + "delete", + "void", + "throw", + "case", + "do", + "else", + "yield", + "await", +}) +# A line break before a token ends a declaration unless the line before ends +# in, or the next one starts with, one of these. +_CONTINUES_EXPRESSION = frozenset("=,+-*/%&|^!~?:<>.([{") +_STARTS_CONTINUATION = frozenset(".,?:+-*/%&|^=<>)]}") + + +@dataclasses.dataclass(frozen=True, slots=True) +class Token: + """A token: a name (identifiers, keywords, numbers), a string, or punctuation.""" + + # "name", "string", "punct", or "literal" for templates and regexes. + kind: str + # The text; the contents for a string. + value: str + # Whether a line break precedes the token. + newline: bool + + +def _read_string(source: str, start: int) -> tuple[str, int]: + """Read a quoted string. + + Args: + source: The source. + start: The index of the opening quote. + + Returns: + The contents, and the index after the closing quote. + """ + quote = source[start] + index = start + 1 + while index < len(source): + char = source[index] + if char == "\\": + index += 2 + continue + if char in (quote, "\n"): + return source[start + 1 : index], index + 1 + index += 1 + return source[start + 1 :], len(source) + + +def _skip_template(source: str, start: int) -> int: + """Skip a template literal, including the expressions in it. + + Args: + source: The source. + start: The index of the opening backtick. + + Returns: + The index after the closing backtick. + """ + index = start + 1 + while index < len(source): + char = source[index] + if char == "\\": + index += 2 + elif char == "`": + return index + 1 + elif source.startswith("${", index): + index = _skip_braces(source, index + 2) + else: + index += 1 + return len(source) + + +def _skip_braces(source: str, index: int) -> int: + """Skip to the brace that closes one already open. + + Args: + source: The source. + index: The index after the opening brace. + + Returns: + The index after the closing brace. + """ + depth = 1 + while index < len(source): + char = source[index] + if char in "'\"": + index = _read_string(source, index)[1] + elif char == "`": + index = _skip_template(source, index) + elif source.startswith("//", index): + index = _line_end(source, index) + elif source.startswith("/*", index): + index = _comment_end(source, index) + else: + if char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + return index + 1 + index += 1 + return len(source) + + +def _skip_regex(source: str, start: int) -> int: + """Skip a regular expression literal and its flags. + + Args: + source: The source. + start: The index of the opening slash. + + Returns: + The index after it. + """ + index = start + 1 + in_class = False + while index < len(source): + char = source[index] + if char == "\\": + index += 2 + continue + if char == "\n": + return index + if char == "[": + in_class = True + elif char == "]": + in_class = False + elif char == "/" and not in_class: + index += 1 + while index < len(source) and ( + source[index].isalnum() or source[index] == "_" + ): + index += 1 + return index + index += 1 + return len(source) + + +def _line_end(source: str, index: int) -> int: + end = source.find("\n", index) + return len(source) if end < 0 else end + + +def _comment_end(source: str, index: int) -> int: + end = source.find("*/", index + 2) + return len(source) if end < 0 else end + 2 + + +def _regex_allowed(previous: Token | None) -> bool: + """Whether a slash after a token starts a regular expression. + + Args: + previous: The token before the slash. + + Returns: + Whether it starts a regular expression rather than a division. + """ + if previous is None: + return True + if previous.kind == "punct": + return previous.value in _PUNCT_BEFORE_REGEX or previous.value == "}" + return previous.kind == "name" and previous.value in _KEYWORDS_BEFORE_REGEX + + +def tokenize(source: str) -> list[Token]: + """Split a JavaScript or TypeScript source into tokens, without comments. + + Args: + source: The source. + + Returns: + The tokens. + """ + tokens: list[Token] = [] + newline = False + index = 0 + length = len(source) + while index < length: + char = source[index] + if char == "\n": + newline = True + index += 1 + continue + if char.isspace(): + index += 1 + continue + if source.startswith("//", index): + index = _line_end(source, index) + continue + if source.startswith("/*", index): + end = _comment_end(source, index) + newline = newline or "\n" in source[index:end] + index = end + continue + if char in "'\"": + value, index = _read_string(source, index) + tokens.append(Token("string", value, newline)) + elif char == "`": + index = _skip_template(source, index) + tokens.append(Token("literal", "", newline)) + elif char == "/" and _regex_allowed(tokens[-1] if tokens else None): + index = _skip_regex(source, index) + tokens.append(Token("literal", "", newline)) + elif char.isalnum() or char in "_$" or ord(char) > 127: + end = index + 1 + while end < length and ( + source[end].isalnum() or source[end] in "_$" or ord(source[end]) > 127 + ): + end += 1 + tokens.append(Token("name", source[index:end], newline)) + index = end + else: + tokens.append(Token("punct", char, newline)) + index += 1 + newline = False + return tokens + + +def _is(token: Token | None, kind: str, value: str | None = None) -> bool: + return ( + token is not None + and token.kind == kind + and (value is None or token.value == value) + ) + + +def _at(tokens: list[Token], index: int) -> Token | None: + return tokens[index] if 0 <= index < len(tokens) else None + + +def _pattern_names(tokens: list[Token], index: int) -> tuple[list[str], int]: + """Read the names a destructuring pattern declares. + + Args: + tokens: The tokens. + index: The index of the pattern's opening brace or bracket. + + Returns: + The names, and the index after the pattern. + """ + names: list[str] = [] + depth = 0 + skip_depth = 0 + while index < len(tokens): + token = tokens[index] + if token.kind == "punct" and token.value in "{[(": + depth += 1 + elif token.kind == "punct" and token.value in "}])": + if skip_depth == depth: + skip_depth = 0 + depth -= 1 + if depth == 0: + return names, index + 1 + elif _is(token, "punct", ",") and skip_depth == depth: + skip_depth = 0 + elif _is(token, "punct", "="): + # A default value, up to the next element of this pattern. + skip_depth = skip_depth or depth + elif ( + token.kind == "name" + and not skip_depth + and _IDENTIFIER_RE.fullmatch(token.value) + and not _is(_at(tokens, index + 1), "punct", ":") + ): + names.append(token.value) + index += 1 + return names, index + + +def _declared_names(tokens: list[Token], index: int) -> list[str]: + """Read the names of an exported ``const``, ``let`` or ``var`` declaration. + + Args: + tokens: The tokens. + index: The index after the keyword. + + Returns: + Every declared name, ``export const a = 1, b = 2`` declaring two. + """ + names: list[str] = [] + while index < len(tokens): + token = tokens[index] + if token.kind == "name" and _IDENTIFIER_RE.fullmatch(token.value): + names.append(token.value) + index += 1 + elif token.kind == "punct" and token.value in "{[": + pattern, index = _pattern_names(tokens, index) + names.extend(pattern) + else: + return names + # Skip a type annotation and the initializer, up to the next declarator. + depth = 0 + angle = 0 + annotation = _is(_at(tokens, index), "punct", ":") + while index < len(tokens): + token = tokens[index] + previous = tokens[index - 1] + if token.kind == "punct": + if token.value in "([{": + depth += 1 + elif token.value in ")]}": + if depth == 0: + return names + depth -= 1 + elif annotation and depth == 0 and token.value in "<>": + angle += 1 if token.value == "<" else -1 + elif depth == 0 and angle == 0 and token.value == "=": + annotation = False + elif depth == 0 and angle == 0 and token.value == ",": + index += 1 + break + elif depth == 0 and token.value == ";": + return names + if ( + depth == 0 + and token.newline + and not ( + previous.kind == "punct" and previous.value in _CONTINUES_EXPRESSION + ) + and not (token.kind == "punct" and token.value in _STARTS_CONTINUATION) + ): + return names + index += 1 + else: + return names + return names + + +def _list_names(tokens: list[Token], index: int) -> list[str]: + """Read the names an export list exports: ``{ a, b as c, "d" as e }``. + + Args: + tokens: The tokens. + index: The index after the opening brace. + + Returns: + The exported names. + """ + names: list[str] = [] + current: str | None = None + while index < len(tokens): + token = tokens[index] + if _is(token, "punct", "}") or _is(token, "punct", ","): + if current is not None: + names.append(current) + current = None + if token.value == "}": + break + elif token.kind in ("name", "string") and token.value != "as": + # The last name of an item is its exported name: `a as b` exports b. + current = token.value + index += 1 + return names + + +def _export_names(tokens: list[Token], index: int) -> list[str]: + """Read the names one ``export`` statement exports. + + Args: + tokens: The tokens. + index: The index after ``export``. + + Returns: + The names; none for a default export, a class, a namespace re-export or + a type. + """ + token = _at(tokens, index) + if _is(token, "name", "async"): + index += 1 + token = _at(tokens, index) + if _is(token, "name", "function"): + index += 1 + if _is(_at(tokens, index), "punct", "*"): + index += 1 + name = _at(tokens, index) + return [name.value] if name is not None and name.kind == "name" else [] + if ( + token is not None + and token.kind == "name" + and token.value in ("const", "let", "var") + ): + return _declared_names(tokens, index + 1) + if _is(token, "punct", "{"): + return _list_names(tokens, index + 1) + return [] + + +def find_exports(source: str) -> list[str]: + """Find the names of the functions a module may export. + + Covers declared functions and variables (``export function f``, + ``export const f = ...``, also several or destructured) and export lists + (``export { a, b as c }``, also re-exported ``from`` another module). A + default export, a class, ``export *`` and TypeScript types have no function + name to import in Python and are left out. + + Args: + source: The module source. + + Returns: + The names, in the order they appear, without duplicates. + """ + tokens = tokenize(source) + names: list[str] = [] + for index, token in enumerate(tokens): + if _is(token, "name", "export") and not _is( + _at(tokens, index - 1), "punct", "." + ): + names.extend(_export_names(tokens, index + 1)) + return [name for name in dict.fromkeys(names) if name != "default"] + + +def find_imports(source: str) -> list[str]: + """Find the specifiers a module imports or re-exports from. + + Covers ``import ... from "x"``, ``import "x"``, ``import("x")`` and + ``export ... from "x"``, wherever line breaks fall. + + Args: + source: The module source. + + Returns: + The specifiers, without duplicates. + """ + tokens = tokenize(source) + specifiers: list[str] = [] + for index, token in enumerate(tokens): + if token.kind != "string": + continue + keyword_index = index - 1 + if _is(_at(tokens, keyword_index), "punct", "("): + # A dynamic import: `import("x")`. + keyword_index -= 1 + if not _is(_at(tokens, keyword_index), "name", "import"): + continue + keyword = _at(tokens, keyword_index) + if ( + keyword is not None + and keyword.kind == "name" + and keyword.value in ("from", "import") + and not _is(_at(tokens, keyword_index - 1), "punct", ".") + ): + specifiers.append(token.value) + return list(dict.fromkeys(specifiers)) + + +def _resolve_local(directory: Path, specifier: str) -> Path | None: + """Find the file a relative import names, as the bundler would. + + Args: + directory: The importing file's directory. + specifier: The relative specifier, such as ``./helpers`` or ``./x.js``. + + Returns: + The file, or None if there is none. + """ + target = directory / specifier + candidates = [ + target, + *(target.with_name(target.name + ext) for ext in _RESOLVE_EXTENSIONS), + ] + if target.suffix == ".js": + # TypeScript imports a .ts file by the .js name it compiles to. + candidates.append(target.with_suffix(".ts")) + candidates.extend(target / f"index{ext}" for ext in _RESOLVE_EXTENSIONS) + return next((candidate for candidate in candidates if candidate.is_file()), None) + + +def module_files(path: Path) -> list[Path]: + """Find a module and every local file it imports, directly or not. + + Args: + path: The module. + + Returns: + The resolved files, the module first. + """ + files = {path.resolve(): None} + pending = [path.resolve()] + while pending: + file = pending.pop() + if file.suffix not in SCRIPT_SOURCE_EXTENSIONS: + continue + for specifier in find_imports(file.read_text(encoding="utf-8")): + if not specifier.startswith("."): + continue + target = _resolve_local(file.parent, specifier) + if target is not None and (target := target.resolve()) not in files: + files[target] = None + pending.append(target) + return list(files) diff --git a/packages/reflex-base/src/reflex_base/registry.py b/packages/reflex-base/src/reflex_base/registry.py index 97d3eb014f5..0315be0a8a3 100644 --- a/packages/reflex-base/src/reflex_base/registry.py +++ b/packages/reflex-base/src/reflex_base/registry.py @@ -169,6 +169,7 @@ def fork(self) -> Self: decorated_pages=list(self.decorated_pages), bundled_libraries=list(self.bundled_libraries), _explicit_bundled_libraries=dict(self._explicit_bundled_libraries), + script_modules=dict(self.script_modules), ) def _set_config(self, config: Config) -> None: diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index 36531ebbd1d..5520a67e4da 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -32,17 +32,12 @@ MemoParamKind, ) from reflex_base.constants.state import FIELD_MARKER -from reflex_base.js.finder import SCRIPT_EXTENSIONS -from reflex_base.js.modules import ( - find_exports, - find_package_imports, - module_key, - render_stub, -) +from reflex_base.js.modules import find_package_imports, module_key, render_stub +from reflex_base.js.source import SCRIPT_SOURCE_EXTENSIONS, find_exports, module_files from reflex_base.registry import RegistrationContext from reflex_base.style import Style from reflex_base.utils import format, imports, memo_paths, serializers, types -from reflex_base.utils.exceptions import VarTypeError +from reflex_base.utils.exceptions import ReflexRuntimeError, VarTypeError from reflex_base.utils.imports import ImportVar, ParsedImportDict from reflex_base.vars.base import Field, Var, VarData from reflex_base.vars.function import DestructuredArg @@ -312,41 +307,51 @@ def compile_script_modules() -> tuple[ ]: """Compile the JavaScript modules the app imports from Python. - Each module is copied under ``utils/script_modules`` at the path of its - Python name, with the script files beside it so its relative imports - resolve, and ``utils/scripts.js`` re-exports it under its module key. A - type stub is written next to the module, for type checkers. + Each module and the local files it imports, directly or not, are copied + under ``utils/script_modules`` at their paths from the directory that holds + the module's top-level package, so relative imports resolve as they do in + the source tree. ``utils/scripts.js`` re-exports each module under its + module key, and a type stub is written next to it, for type checkers. Returns: The (path, code) outputs, and the npm packages the modules import, to install. + + Raises: + ReflexRuntimeError: If a module imports a file outside its top-level + package's directory. """ web_dir = get_web_dir() + modules_dir = web_dir / constants.Dirs.SCRIPT_MODULES_PATH outputs: list[tuple[str, str]] = [] package_imports: dict[str, list[ImportVar]] = {} index: list[str] = [] copied: set[Path] = set() modules = RegistrationContext.ensure_context().script_modules for name, path in sorted(modules.items()): - package_dir = Path(*name.split(".")[:-1]) - for file in sorted(path.parent.iterdir()): - if file.suffix not in SCRIPT_EXTENSIONS or file in copied: + root = path.resolve().parents[name.count(".")] + for file in module_files(path): + if not file.is_relative_to(root): + msg = ( + f"The JavaScript module {name} imports {file}, which is " + f"outside {root}, the directory of its package." + ) + raise ReflexRuntimeError(msg) + if file in copied: continue copied.add(file) + target = modules_dir / file.relative_to(root) + if file.suffix not in SCRIPT_SOURCE_EXTENSIONS: + outputs.append((str(target), file.read_text(encoding="utf-8"))) + continue source = file.read_text(encoding="utf-8") - outputs.append(( - str( - web_dir - / constants.Dirs.SCRIPT_MODULES_PATH - / package_dir - / file.name - ), - source, - )) + outputs.append((str(target), source)) for specifier in find_package_imports(source): package_imports[specifier] = [ImportVar(tag=None, render=False)] - target = f"$/{constants.Dirs.SCRIPT_MODULES_PATH}/{package_dir.as_posix()}/{path.name}" - index.append(f'export * as {module_key(name)} from "{target}";') + specifier = (modules_dir / path.resolve().relative_to(root)).relative_to( + web_dir + ) + index.append(f'export * as {module_key(name)} from "$/{specifier.as_posix()}";') exports = find_exports(path.read_text(encoding="utf-8")) outputs.append((str(path.with_suffix(".pyi")), render_stub(path, exports))) outputs.append(( diff --git a/tests/units/compiler/test_compiler_utils.py b/tests/units/compiler/test_compiler_utils.py index 44f3da2d05c..1f130eade9e 100644 --- a/tests/units/compiler/test_compiler_utils.py +++ b/tests/units/compiler/test_compiler_utils.py @@ -12,6 +12,7 @@ from reflex_base import constants from reflex_base.components.memo import create_passthrough_component_memo from reflex_base.registry import RegistrationContext +from reflex_base.utils.exceptions import ReflexRuntimeError from reflex_components_core.base.fragment import Fragment from reflex_components_core.base.script import Script from reflex_components_core.el.elements.metadata import Link @@ -815,7 +816,7 @@ def test_compile_script_modules( monkeypatch: pytest.MonkeyPatch, clean_registration_context: RegistrationContext, ): - """Imported JavaScript modules are copied with their siblings and indexed. + """Imported JavaScript modules are copied with the local files they import. Args: tmp_path: pytest tmp_path fixture. @@ -825,16 +826,25 @@ def test_compile_script_modules( web_dir = tmp_path / ".web" monkeypatch.setattr(utils, "get_web_dir", lambda: web_dir) scripts = tmp_path / "app" / "scripts" - scripts.mkdir(parents=True) + (scripts / "formatters").mkdir(parents=True) party = scripts / "party.js" party.write_text( - 'import confetti from "canvas-confetti";\n' - 'import { describe } from "./helpers.js";\n' + 'import confetti from\n "canvas-confetti";\n' + 'import { describe } from "./helpers";\n' + 'import "./party.css";\n' "export function celebrate(count) {}\n" ) helpers = scripts / "helpers.js" - helpers.write_text('import dayjs from "dayjs";\nexport const describe = 1;\n') - (scripts / "notes.txt").write_text("not a script") + helpers.write_text( + 'import dayjs from "dayjs";\n' + 'import { count } from "./formatters/count.js";\n' + "export const describe = 1;\n" + ) + count = scripts / "formatters" / "count.ts" + count.write_text("export const count = (n: number) => `${n}`;\n") + css = scripts / "party.css" + css.write_text("canvas { pointer-events: none; }\n") + (scripts / "unused.js").write_text('import "left-pad";\n') typed = tmp_path / "app" / "typed.ts" typed.write_text("export function typed(n: number) {}\n") clean_registration_context.script_modules.update({ @@ -844,10 +854,12 @@ def test_compile_script_modules( outputs, package_imports = utils.compile_script_modules() - modules_dir = web_dir / "utils" / "script_modules" + modules_dir = web_dir / "utils" / "script_modules" / "app" assert outputs == [ - (str(modules_dir / "app" / "scripts" / "helpers.js"), helpers.read_text()), - (str(modules_dir / "app" / "scripts" / "party.js"), party.read_text()), + (str(modules_dir / "scripts" / "party.js"), party.read_text()), + (str(modules_dir / "scripts" / "helpers.js"), helpers.read_text()), + (str(modules_dir / "scripts" / "party.css"), css.read_text()), + (str(modules_dir / "scripts" / "formatters" / "count.ts"), count.read_text()), ( str(scripts / "party.pyi"), ( @@ -859,7 +871,7 @@ def test_compile_script_modules( "def celebrate(*args: Any) -> EventSpec: ...\n" ), ), - (str(modules_dir / "app" / "typed.ts"), typed.read_text()), + (str(modules_dir / "typed.ts"), typed.read_text()), ( str(tmp_path / "app" / "typed.pyi"), ( @@ -885,6 +897,35 @@ def test_compile_script_modules( } +def test_compile_script_modules_rejects_files_outside_the_package( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + clean_registration_context: RegistrationContext, +): + """A module may not import a file outside its top-level package's directory. + + Args: + tmp_path: pytest tmp_path fixture. + monkeypatch: pytest monkeypatch fixture. + clean_registration_context: a fresh registry of imported modules. + """ + monkeypatch.setattr(utils, "get_web_dir", lambda: tmp_path / ".web") + root = tmp_path / "project" + (root / "app").mkdir(parents=True) + outside = tmp_path / "shared.js" + outside.write_text("export const x = 1;\n") + party = root / "app" / "party.js" + party.write_text('import { x } from "../../shared.js";\n') + clean_registration_context.script_modules["app.party"] = party + + with pytest.raises(ReflexRuntimeError) as error: + utils.compile_script_modules() + assert str(error.value) == ( + f"The JavaScript module app.party imports {outside.resolve()}, which is " + f"outside {root.resolve()}, the directory of its package." + ) + + def test_compile_script_modules_without_modules( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/units/reflex_base/js/test_modules.py b/tests/units/reflex_base/js/test_modules.py index 51ad9362c16..d349e65dd88 100644 --- a/tests/units/reflex_base/js/test_modules.py +++ b/tests/units/reflex_base/js/test_modules.py @@ -1,6 +1,7 @@ """Tests for importing JavaScript modules from Python.""" import importlib +import subprocess import sys from collections.abc import Generator from pathlib import Path @@ -10,11 +11,11 @@ from reflex_base.js.finder import ScriptFinder from reflex_base.js.modules import ( JsFunction, - find_exports, find_package_imports, module_key, render_stub, ) +from reflex_base.js.source import find_exports from reflex_base.registry import RegistrationContext from reflex_base.vars.base import LiteralVar, Var @@ -39,19 +40,8 @@ """ -def test_find_exports(): - assert find_exports(SOURCE) == [ - "celebrate", - "load", - "steps", - "arrow", - "hidden", - "describeIt", - "reexported", - ] - - def test_find_package_imports(): + """Only specifiers naming an npm package are installed.""" assert find_package_imports(SOURCE) == [ "canvas-confetti", "@scope/pkg/style.css", @@ -61,11 +51,13 @@ def test_find_package_imports(): def test_module_key_cannot_collide(): + """Module keys are JavaScript identifiers no two module names share.""" assert module_key("app.scripts.party") == "app$scripts$party" assert module_key("app.scripts__party") != module_key("app__scripts.party") def test_calling_a_function_builds_the_call_js_event(): + """Calling an export builds the event that names it, with JSON arguments.""" spec = JsFunction("app.scripts.party", "celebrate")(200, "results") assert isinstance(spec, EventSpec) assert str(LiteralVar.create(spec)) == ( @@ -75,6 +67,8 @@ def test_calling_a_function_builds_the_call_js_event(): def test_an_event_handler_argument_is_sent_as_its_name(): + """An event handler argument arrives in JavaScript as its full name.""" + class JsHandlerState(rx.State): @rx.event def done(self, value: str): @@ -93,6 +87,7 @@ def done(self, value: str): def test_var_arguments_are_compiled_into_the_event(): + """A Var argument is compiled into the event payload.""" count = Var(_js_expr="count").to(int) spec = JsFunction("app.scripts.party", "celebrate")(count) literal = LiteralVar.create(spec) @@ -103,6 +98,7 @@ def test_var_arguments_are_compiled_into_the_event(): def test_render_stub_skips_names_python_cannot_import(): + """The stub leaves out exports that are not Python identifiers.""" assert render_stub(Path("party.js"), ["celebrate", "$private", "class"]) == ( "# Generated by Reflex from party.js; rewritten when it changes.\n" "from typing import Any\n" @@ -141,6 +137,12 @@ def script_package( def test_importing_a_module_exposes_its_exports( script_package: Path, clean_registration_context: RegistrationContext ): + """Importing a JavaScript module exposes its exports and registers it. + + Args: + script_package: a package with JavaScript modules on sys.path. + clean_registration_context: the registry the import fills. + """ assert any(isinstance(finder, ScriptFinder) for finder in sys.meta_path) party = importlib.import_module("js_pkg.party") assert party.__all__ == find_exports(SOURCE) @@ -156,17 +158,32 @@ def test_importing_a_module_exposes_its_exports( def test_a_missing_module_still_fails_to_import(script_package: Path): + """A name with neither a Python nor a JavaScript module still fails. + + Args: + script_package: a package with JavaScript modules on sys.path. + """ with pytest.raises(ModuleNotFoundError): importlib.import_module("js_pkg.missing") def test_a_python_module_wins_over_a_script_of_the_same_name(script_package: Path): + """The finder runs last, so a Python module shadows a script of its name. + + Args: + script_package: a package with JavaScript modules on sys.path. + """ (script_package / "both.py").write_text("VALUE = 'python'\n") (script_package / "both.js").write_text("export function VALUE() {}\n") assert importlib.import_module("js_pkg.both").VALUE == "python" def test_a_function_is_an_event_trigger(script_package: Path): + """An export works in an event trigger, called or uncalled. + + Args: + script_package: a package with JavaScript modules on sys.path. + """ from js_pkg.party import celebrate # pyright: ignore[reportMissingImports] called = rx.button("go", on_click=celebrate(1, "out")) @@ -182,3 +199,22 @@ def test_a_function_is_an_event_trigger(script_package: Path): '"js_pkg$party", ["function"] : "celebrate", ' '["args"] : [_e?.["target"]?.["value"]] }), ({ })))], [_e], ({ }))))' ) + + +def test_the_finder_imports_nothing_heavy(): + """``import reflex`` installs the finder, so loading it stays cheap.""" + loaded = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys, reflex_base.js.finder; " + "print(sorted({'pathlib', 'typing', 'importlib.abc', 'reflex_base.event'}" + " & set(sys.modules)))" + ), + ], + capture_output=True, + text=True, + check=True, + ).stdout + assert loaded == "[]\n" diff --git a/tests/units/reflex_base/js/test_source.py b/tests/units/reflex_base/js/test_source.py new file mode 100644 index 00000000000..73ced8b8125 --- /dev/null +++ b/tests/units/reflex_base/js/test_source.py @@ -0,0 +1,111 @@ +"""Tests for reading the exports and imports of JavaScript modules.""" + +from pathlib import Path + +from reflex_base.js.source import find_exports, find_imports, module_files + +SOURCE = r""" +import confetti from + "canvas-confetti"; +import { + describe, +} from "./helpers.js"; +import "@scope/pkg/style.css"; +const route = "/api/*"; +const pattern = /["'`]\/*/g, ratio = 4 / 2 / 1; +const text = `export function fake() {} ${`nested ${"}"}`} import "nope"`; +// export function commentedOut() {} +/* export const alsoCommentedOut = 1; */ +export function celebrate(count) {} +export async function load() {} +export function* steps() {} +export const first = () => 1, second = (a, b) => [a, b]; +export let typed: Record = {}, other = 2 +export const { pickA, pickB: renamed, ...rest } = obj; +export const [x0, , x2 = 3] = arr; +export const multiline = (a) => + a + 1; +export class NotAFunction {} +function hidden() {} +export { hidden, describe as describeIt, "string name" as quoted }; +export { other2 as reexported } from "lodash/fp"; +export * from "everything"; +export * as namespace from "namespaced"; +export default function () {} +const later = await import("dayjs"); +thing.export = 1; +thing.import("not-an-import"); +""" + + +def test_find_exports(): + """Every exported function name is found, and nothing inside text or comments.""" + assert find_exports(SOURCE) == [ + "celebrate", + "load", + "steps", + "first", + "second", + "typed", + "other", + "pickA", + "renamed", + "rest", + "x0", + "x2", + "multiline", + "hidden", + "describeIt", + "quoted", + "reexported", + ] + + +def test_find_imports(): + """Static, side-effect, dynamic and re-export specifiers, across line breaks.""" + assert find_imports(SOURCE) == [ + "canvas-confetti", + "./helpers.js", + "@scope/pkg/style.css", + "lodash/fp", + "everything", + "namespaced", + "dayjs", + ] + + +def test_a_string_with_a_comment_opener_hides_nothing(): + """A comment opener inside a string does not swallow the code after it.""" + source = 'const route = "/api/*";\nexport function after() {}\n/* note */\n' + assert find_exports(source) == ["after"] + + +def test_module_files_follows_relative_imports(tmp_path: Path): + """The module's local imports are found transitively, as the bundler resolves them. + + Args: + tmp_path: pytest tmp_path fixture. + """ + (tmp_path / "lib").mkdir() + (tmp_path / "widgets").mkdir() + module = tmp_path / "party.js" + module.write_text( + 'import { a } from "./lib/a";\n' + 'import { b } from "./b.js";\n' + 'import "./party.css";\n' + 'import { w } from "./widgets";\n' + 'import { missing } from "./missing";\n' + 'import confetti from "canvas-confetti";\n' + ) + (tmp_path / "lib" / "a.ts").write_text('import { b } from "../b.js";\n') + (tmp_path / "b.ts").write_text('import { a } from "./lib/a.js";\n') + (tmp_path / "party.css").write_text('@import "./not-followed.css";\n') + (tmp_path / "widgets" / "index.js").write_text("export const w = 1;\n") + (tmp_path / "unused.js").write_text("export const u = 1;\n") + assert module_files(module) == [ + module.resolve(), + (tmp_path / "lib" / "a.ts").resolve(), + (tmp_path / "b.ts").resolve(), + (tmp_path / "party.css").resolve(), + (tmp_path / "widgets" / "index.js").resolve(), + ] diff --git a/tests/units/reflex_base/test_registry.py b/tests/units/reflex_base/test_registry.py index 82490effd6c..f097e402fe5 100644 --- a/tests/units/reflex_base/test_registry.py +++ b/tests/units/reflex_base/test_registry.py @@ -1,6 +1,7 @@ """Tests for RegistrationContext.""" import sys +from pathlib import Path from textwrap import dedent from typing import Any, cast @@ -436,3 +437,18 @@ def test_reset_compile_caches_empties_per_compile_maps( assert not clean_registration_context._memo_body_analyses assert not clean_registration_context._memoized_event_triggers assert not clean_registration_context._bound_event_chains + + +def test_fork_carries_script_modules(tmp_path: Path): + """JavaScript modules imported before a fork still compile into the forked app. + + Args: + tmp_path: pytest tmp_path fixture. + """ + module = tmp_path / "party.js" + with RegistrationContext.get().fork() as original: + original.script_modules["app.party"] = module + with original.fork() as forked: + assert forked.script_modules == {"app.party": module} + forked.script_modules["app.other"] = module + assert original.script_modules == {"app.party": module} From 9108fef3ea2baed3247250003ba2df5410de0186 Mon Sep 17 00:00:00 2001 From: Farhan Date: Fri, 20 Nov 2026 21:43:58 +0000 Subject: [PATCH 03/16] Copy JS module assets byte for byte and follow stylesheet references Review fixes: - Files a module imports that are not scripts (stylesheets, images, fonts) were read as UTF-8 into the compile output, so `import logo from "./logo.png"` stopped the compile with UnicodeDecodeError. compile_script_modules now returns them as (source, target) copies, which compile_app copies byte for byte after the dry-run return, skipping targets that are up to date. - A copied stylesheet's `@import` and `url(...)` references are followed too, so `./theme.css` and `./bg.png` used by an imported `party.css` are bundled. Absolute paths, URLs, data URIs and fragments are left alone. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- docs/api-reference/browser_javascript.md | 5 +- .../reflex-base/src/reflex_base/js/source.py | 55 +++++++++++++++++-- reflex/compiler/compiler.py | 6 +- reflex/compiler/utils.py | 26 +++++++-- tests/units/compiler/test_compiler_utils.py | 42 +++++++++++++- tests/units/reflex_base/js/test_source.py | 34 +++++++++++- 6 files changed, 150 insertions(+), 18 deletions(-) diff --git a/docs/api-reference/browser_javascript.md b/docs/api-reference/browser_javascript.md index d82e8d499a7..787ae2236b1 100644 --- a/docs/api-reference/browser_javascript.md +++ b/docs/api-reference/browser_javascript.md @@ -229,8 +229,9 @@ rx.button("Report", on_click=report(PartyState.done, "clicked")) through `env.addEvents`. - Packages the module imports, such as `canvas-confetti`, are installed with the app. Pin a version with `frontend_packages` in `rxconfig.py`. -- Local files the module imports with relative paths, such as `./helpers.js` - or `./party.css`, are bundled with it, and so are the files they import. They +- Local files the module imports with relative paths, such as `./helpers.js`, + `./party.css` or `./logo.png`, are bundled with it, and so are the files they + import, including a stylesheet's `@import` and `url(...)` references. They must be inside the directory of your app's package. - Reflex writes a type stub (`party.pyi`) next to the module when it compiles the app, so type checkers resolve the import. diff --git a/packages/reflex-base/src/reflex_base/js/source.py b/packages/reflex-base/src/reflex_base/js/source.py index 64a13e6dc79..6efea923a17 100644 --- a/packages/reflex-base/src/reflex_base/js/source.py +++ b/packages/reflex-base/src/reflex_base/js/source.py @@ -14,9 +14,15 @@ # Files whose imports are followed; other local files (CSS, JSON) are copied. SCRIPT_SOURCE_EXTENSIONS = (".js", ".ts", ".mjs", ".jsx", ".tsx") +# Stylesheets, whose `@import` and `url(...)` references are followed. +STYLESHEET_EXTENSION = ".css" # Tried in order for an import without an extension, as the bundler does. _RESOLVE_EXTENSIONS = (".ts", ".tsx", ".js", ".jsx", ".mjs") +_CSS_COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL) +_CSS_REFERENCE_RE = re.compile( + r"""@import\s+(?:url\(\s*)?["']?([^"')\s;]+)|url\(\s*["']?([^"')\s]+)["']?\s*\)""" +) _IDENTIFIER_RE = re.compile(r"[A-Za-z_$\u0080-￿][\w$\u0080-￿]*") # A `/` after one of these starts a regular expression, not a division. _PUNCT_BEFORE_REGEX = frozenset("(,=:[!&|?{};+-*%<>~^") @@ -475,6 +481,31 @@ def find_imports(source: str) -> list[str]: return list(dict.fromkeys(specifiers)) +def find_stylesheet_references(source: str) -> list[str]: + """Find the local files a stylesheet references. + + Covers ``@import`` and ``url(...)``. Absolute paths, URLs, data URIs and + fragment references are not local files and are left out. + + Args: + source: The stylesheet source. + + Returns: + The relative references, without query or fragment, without duplicates. + """ + references = ( + import_ or url + for import_, url in _CSS_REFERENCE_RE.findall(_CSS_COMMENT_RE.sub("", source)) + ) + return list( + dict.fromkeys( + reference.split("?")[0].split("#")[0] + for reference in references + if not reference.startswith(("/", "#", "data:")) and "://" not in reference + ) + ) + + def _resolve_local(directory: Path, specifier: str) -> Path | None: """Find the file a relative import names, as the bundler would. @@ -500,6 +531,9 @@ def _resolve_local(directory: Path, specifier: str) -> Path | None: def module_files(path: Path) -> list[Path]: """Find a module and every local file it imports, directly or not. + Scripts' relative imports are followed, and so are the ``@import`` and + ``url(...)`` references of the stylesheets among them. + Args: path: The module. @@ -510,12 +544,23 @@ def module_files(path: Path) -> list[Path]: pending = [path.resolve()] while pending: file = pending.pop() - if file.suffix not in SCRIPT_SOURCE_EXTENSIONS: + if file.suffix in SCRIPT_SOURCE_EXTENSIONS: + targets = [ + _resolve_local(file.parent, specifier) + for specifier in find_imports(file.read_text(encoding="utf-8")) + if specifier.startswith(".") + ] + elif file.suffix == STYLESHEET_EXTENSION: + targets = [ + target if (target := file.parent / reference).is_file() else None + for reference in find_stylesheet_references( + file.read_text(encoding="utf-8") + ) + ] + else: + # Images, fonts, data: nothing further to follow. continue - for specifier in find_imports(file.read_text(encoding="utf-8")): - if not specifier.startswith("."): - continue - target = _resolve_local(file.parent, specifier) + for target in targets: if target is not None and (target := target.resolve()) not in files: files[target] = None pending.append(target) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index bbe645ed8e5..cdf2f4e052c 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1590,7 +1590,9 @@ def add_save_task( ), utils._compile_bundled_libraries(), ]) - script_module_outputs, script_module_imports = utils.compile_script_modules() + script_module_outputs, script_module_imports, script_module_copies = ( + utils.compile_script_modules() + ) compile_results.extend(script_module_outputs) progress.advance(task) @@ -1602,6 +1604,8 @@ def add_save_task( if dry_run: return True + utils.copy_files(script_module_copies) + # Delete memo files this compile no longer emits. Done here (not before the # dry-run return) so ``--dry`` never mutates ``.web`` or the manifest. utils.prune_stale_memo_files(path for path, _ in memo_component_files) diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index 5520a67e4da..052a5b1600f 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -303,7 +303,7 @@ def _compile_bundled_libraries() -> tuple[str, str]: def compile_script_modules() -> tuple[ - list[tuple[str, str]], dict[str, list[ImportVar]] + list[tuple[str, str]], dict[str, list[ImportVar]], list[tuple[Path, Path]] ]: """Compile the JavaScript modules the app imports from Python. @@ -314,8 +314,9 @@ def compile_script_modules() -> tuple[ module key, and a type stub is written next to it, for type checkers. Returns: - The (path, code) outputs, and the npm packages the modules import, to - install. + The (path, code) outputs of the scripts, the npm packages they import, + to install, and the (source, target) copies of the other files they + reference, such as stylesheets and images, which may be binary. Raises: ReflexRuntimeError: If a module imports a file outside its top-level @@ -324,6 +325,7 @@ def compile_script_modules() -> tuple[ web_dir = get_web_dir() modules_dir = web_dir / constants.Dirs.SCRIPT_MODULES_PATH outputs: list[tuple[str, str]] = [] + copies: list[tuple[Path, Path]] = [] package_imports: dict[str, list[ImportVar]] = {} index: list[str] = [] copied: set[Path] = set() @@ -342,7 +344,7 @@ def compile_script_modules() -> tuple[ copied.add(file) target = modules_dir / file.relative_to(root) if file.suffix not in SCRIPT_SOURCE_EXTENSIONS: - outputs.append((str(target), file.read_text(encoding="utf-8"))) + copies.append((file, target)) continue source = file.read_text(encoding="utf-8") outputs.append((str(target), source)) @@ -358,7 +360,21 @@ def compile_script_modules() -> tuple[ str(web_dir / (constants.Dirs.SCRIPTS_PATH + constants.Ext.JS)), "\n".join(index) + "\n" if index else "export {};\n", )) - return outputs, package_imports + return outputs, package_imports, copies + + +def copy_files(copies: Iterable[tuple[Path, Path]]) -> None: + """Copy files byte for byte, skipping those whose target is up to date. + + Args: + copies: The (source, target) pairs. + """ + for source, target in copies: + data = source.read_bytes() + if target.is_file() and target.read_bytes() == data: + continue + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) def _restore_bundled_libraries() -> bool: diff --git a/tests/units/compiler/test_compiler_utils.py b/tests/units/compiler/test_compiler_utils.py index 1f130eade9e..fb045808a01 100644 --- a/tests/units/compiler/test_compiler_utils.py +++ b/tests/units/compiler/test_compiler_utils.py @@ -832,6 +832,7 @@ def test_compile_script_modules( 'import confetti from\n "canvas-confetti";\n' 'import { describe } from "./helpers";\n' 'import "./party.css";\n' + 'import logo from "./logo.png";\n' "export function celebrate(count) {}\n" ) helpers = scripts / "helpers.js" @@ -843,7 +844,13 @@ def test_compile_script_modules( count = scripts / "formatters" / "count.ts" count.write_text("export const count = (n: number) => `${n}`;\n") css = scripts / "party.css" - css.write_text("canvas { pointer-events: none; }\n") + css.write_text('@import "./theme.css";\ncanvas { background: url(./bg.png); }\n') + theme = scripts / "theme.css" + theme.write_text("canvas { pointer-events: none; }\n") + background = scripts / "bg.png" + background.write_bytes(b"\x89PNG\r\n\x1a\n\xff\xfe") + logo = scripts / "logo.png" + logo.write_bytes(b"\x89PNG\r\n\x1a\n\x00\x80") (scripts / "unused.js").write_text('import "left-pad";\n') typed = tmp_path / "app" / "typed.ts" typed.write_text("export function typed(n: number) {}\n") @@ -852,13 +859,12 @@ def test_compile_script_modules( "app.scripts.party": party, }) - outputs, package_imports = utils.compile_script_modules() + outputs, package_imports, copies = utils.compile_script_modules() modules_dir = web_dir / "utils" / "script_modules" / "app" assert outputs == [ (str(modules_dir / "scripts" / "party.js"), party.read_text()), (str(modules_dir / "scripts" / "helpers.js"), helpers.read_text()), - (str(modules_dir / "scripts" / "party.css"), css.read_text()), (str(modules_dir / "scripts" / "formatters" / "count.ts"), count.read_text()), ( str(scripts / "party.pyi"), @@ -895,6 +901,16 @@ def test_compile_script_modules( "canvas-confetti": [rx.ImportVar(tag=None, render=False)], "dayjs": [rx.ImportVar(tag=None, render=False)], } + # Stylesheets and binary files are copied byte for byte, with what they reference. + assert copies == [ + (css.resolve(), modules_dir / "scripts" / "party.css"), + (logo.resolve(), modules_dir / "scripts" / "logo.png"), + (theme.resolve(), modules_dir / "scripts" / "theme.css"), + (background.resolve(), modules_dir / "scripts" / "bg.png"), + ] + utils.copy_files(copies) + assert (modules_dir / "scripts" / "logo.png").read_bytes() == logo.read_bytes() + assert (modules_dir / "scripts" / "bg.png").read_bytes() == background.read_bytes() def test_compile_script_modules_rejects_files_outside_the_package( @@ -942,4 +958,24 @@ def test_compile_script_modules_without_modules( assert utils.compile_script_modules() == ( [(str(tmp_path / "utils" / "scripts.js"), "export {};\n")], {}, + [], ) + + +def test_copy_files_skips_up_to_date_targets(tmp_path: Path): + """A target with the same bytes is left alone; others are written. + + Args: + tmp_path: pytest tmp_path fixture. + """ + source = tmp_path / "logo.png" + source.write_bytes(b"\x89PNG\xff") + target = tmp_path / "out" / "nested" / "logo.png" + utils.copy_files([(source, target)]) + assert target.read_bytes() == b"\x89PNG\xff" + modified = target.stat().st_mtime_ns + utils.copy_files([(source, target)]) + assert target.stat().st_mtime_ns == modified + source.write_bytes(b"\x89PNG\x00") + utils.copy_files([(source, target)]) + assert target.read_bytes() == b"\x89PNG\x00" diff --git a/tests/units/reflex_base/js/test_source.py b/tests/units/reflex_base/js/test_source.py index 73ced8b8125..b2af3b5f2ef 100644 --- a/tests/units/reflex_base/js/test_source.py +++ b/tests/units/reflex_base/js/test_source.py @@ -2,7 +2,12 @@ from pathlib import Path -from reflex_base.js.source import find_exports, find_imports, module_files +from reflex_base.js.source import ( + find_exports, + find_imports, + find_stylesheet_references, + module_files, +) SOURCE = r""" import confetti from @@ -99,7 +104,12 @@ def test_module_files_follows_relative_imports(tmp_path: Path): ) (tmp_path / "lib" / "a.ts").write_text('import { b } from "../b.js";\n') (tmp_path / "b.ts").write_text('import { a } from "./lib/a.js";\n') - (tmp_path / "party.css").write_text('@import "./not-followed.css";\n') + (tmp_path / "party.css").write_text( + '@import "./theme.css";\nbody { background: url(img/bg.png?v=1); }\n' + ) + (tmp_path / "theme.css").write_text("@import url('./missing.css');\n") + (tmp_path / "img").mkdir() + (tmp_path / "img" / "bg.png").write_bytes(b"\x89PNG") (tmp_path / "widgets" / "index.js").write_text("export const w = 1;\n") (tmp_path / "unused.js").write_text("export const u = 1;\n") assert module_files(module) == [ @@ -108,4 +118,24 @@ def test_module_files_follows_relative_imports(tmp_path: Path): (tmp_path / "b.ts").resolve(), (tmp_path / "party.css").resolve(), (tmp_path / "widgets" / "index.js").resolve(), + (tmp_path / "theme.css").resolve(), + (tmp_path / "img" / "bg.png").resolve(), + ] + + +def test_find_stylesheet_references(): + """Local @import and url() references, without absolute paths, URLs or data.""" + source = """ + /* @import "./commented.css"; */ + @import "./theme.css"; + @import url('fonts.css') screen; + @import "https://fonts.example.com/inter.css"; + .a { background: url(./bg.png?v=2#x); } + .b { background: url("/static/abs.png"), url(data:image/png;base64,AAA); } + .c { mask: url(#clip); } + """ + assert find_stylesheet_references(source) == [ + "./theme.css", + "fonts.css", + "./bg.png", ] From 9f708ac52ac2a0dc198594efd6d005c9715cb4b9 Mon Sep 17 00:00:00 2001 From: Farhan Date: Fri, 20 Nov 2026 21:50:31 +0000 Subject: [PATCH 04/16] Register the JS module finder without loading a module for it CodSpeed still showed test_import_reflex about 15% slower on c0ebd9c75. The benchmark clears the reflex modules and imports `reflex` again in the same process, so it measures module execution: importing the lazy reflex_base.js package and its finder module on every `import reflex` cost about 200us of about 1300us. It also showed a bug: install() matched the finder with isinstance, and a re-imported finder module is a new class, so every fresh import appended another finder to sys.meta_path (205 after 200 rounds). reflex_base/__init__.py, which every Reflex import loads first, now registers a small finder class that loads reflex_base.js.finder only on the first import no other finder found, matched by name so a fresh import never adds a second. `import reflex` loads no module for it (about 12us of execution), and reflex/__init__.py is back to its previous form, so the generated reflex/__init__.pyi stays free of runtime code. Tests pin one finder after a re-import and that `import reflex` does not load reflex_base.js. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- .../reflex-base/src/reflex_base/__init__.py | 52 +++++++++++++++++++ .../reflex-base/src/reflex_base/js/finder.py | 14 ++--- pyi_hashes.json | 2 +- reflex/__init__.py | 5 -- tests/units/reflex_base/js/test_modules.py | 31 +++++++++-- 5 files changed, 85 insertions(+), 19 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/__init__.py b/packages/reflex-base/src/reflex_base/__init__.py index 4311653df9d..b4a24c41558 100644 --- a/packages/reflex-base/src/reflex_base/__init__.py +++ b/packages/reflex-base/src/reflex_base/__init__.py @@ -1 +1,53 @@ """Reflex core types.""" + +from __future__ import annotations + +import sys + +# Not imported from typing, which `import reflex` does not load otherwise. +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Sequence + from importlib.machinery import ModuleSpec + from types import ModuleType + + +class _ScriptFinder: + """Let app code import the JavaScript modules beside it. + + Registered here, by the package every Reflex import loads first, so that + no module is loaded for it: the real finder, ``reflex_base.js.finder``, + loads on the first import no other finder found. + """ + + @staticmethod + def find_spec( + fullname: str, + path: Sequence[str] | None, + target: ModuleType | None = None, + ) -> ModuleSpec | None: + """Find the JavaScript file for a submodule. + + Args: + fullname: The dotted module name. + path: The search path of the parent package; None for a top-level + module, which is never a JavaScript module. + target: Unused. + + Returns: + The spec of the JavaScript module, or None. + """ + if path is None: + return None + from reflex_base.js.finder import ScriptFinder + + return ScriptFinder.find_spec(fullname, path, target) + + +# By name, not identity: importing this module afresh, as a reload or a +# benchmark that clears sys.modules does, must not register a second one. +if not any( + isinstance(finder, type) and finder.__qualname__ == "_ScriptFinder" + for finder in sys.meta_path +): + sys.meta_path.append(_ScriptFinder) diff --git a/packages/reflex-base/src/reflex_base/js/finder.py b/packages/reflex-base/src/reflex_base/js/finder.py index 7cee3ff91a2..dbe5b3dae49 100644 --- a/packages/reflex-base/src/reflex_base/js/finder.py +++ b/packages/reflex-base/src/reflex_base/js/finder.py @@ -3,8 +3,9 @@ ``from my_app.scripts.party import celebrate`` finds ``my_app/scripts/party.js`` (or ``.ts``) when no Python module of that name exists. -``import reflex`` installs the finder, so this module stays as cheap to import -as it can: ``os.path`` instead of ``pathlib``, and a finder and loader that +``import reflex`` registers a small finder that loads this module on the first +import no other finder found, which can happen while ``reflex`` itself loads, +so it stays cheap: ``os.path`` instead of ``pathlib``, a finder and loader that implement the import protocol without the ``importlib.abc`` base classes, and no ``typing``. The event system loads with the first JavaScript module imported. @@ -13,7 +14,6 @@ from __future__ import annotations import os.path -import sys from importlib.machinery import ModuleSpec # Not imported from typing, which `import reflex` does not load otherwise. @@ -65,8 +65,8 @@ def exec_module(self, module: ModuleType) -> None: class ScriptFinder: """Find a JavaScript file where no Python module of the name exists.""" + @staticmethod def find_spec( - self, fullname: str, path: Sequence[str] | None, target: ModuleType | None = None, @@ -92,9 +92,3 @@ def find_spec( if os.path.isfile(file): # noqa: PTH113 return ModuleSpec(fullname, _ScriptLoader(file), origin=file) return None - - -def install() -> None: - """Let ``import`` find JavaScript modules, after every other finder.""" - if not any(isinstance(finder, ScriptFinder) for finder in sys.meta_path): - sys.meta_path.append(ScriptFinder()) diff --git a/pyi_hashes.json b/pyi_hashes.json index 6543e41e1fd..3538f32191a 100644 --- a/pyi_hashes.json +++ b/pyi_hashes.json @@ -118,7 +118,7 @@ "packages/reflex-components-recharts/src/reflex_components_recharts/polar.pyi": "adc4db1a3ca7cd292b099ec4d97a4346", "packages/reflex-components-recharts/src/reflex_components_recharts/recharts.pyi": "f2a9cf6db58d169289fbc76de6813300", "packages/reflex-components-sonner/src/reflex_components_sonner/toast.pyi": "f170ac685b6ba5892370166c80684db3", - "reflex/__init__.pyi": "060fa790e6c7b1acc4ecae5804ebfe73", + "reflex/__init__.pyi": "9e15759df6266f047709e2a1325c2cb4", "reflex/components/__init__.pyi": "9facd05a776d0641432696bbf8e34388", "reflex/experimental/memo.pyi": "0fc8eae24cd0def5ea9d8d66ce87206e" } diff --git a/reflex/__init__.py b/reflex/__init__.py index 127a3cbd34a..035dda21e53 100644 --- a/reflex/__init__.py +++ b/reflex/__init__.py @@ -84,7 +84,6 @@ from __future__ import annotations -from reflex_base.js import finder as _js_finder from reflex_base.utils import lazy_loader from reflex_components_radix.mappings import RADIX_MAPPING @@ -258,9 +257,5 @@ ) -# Let app code import the JavaScript modules beside it. -_js_finder.install() - - def __getattr__(name: str): return getattr(name) diff --git a/tests/units/reflex_base/js/test_modules.py b/tests/units/reflex_base/js/test_modules.py index d349e65dd88..aee6ffc7cf6 100644 --- a/tests/units/reflex_base/js/test_modules.py +++ b/tests/units/reflex_base/js/test_modules.py @@ -8,7 +8,6 @@ import pytest from reflex_base.event import EventSpec -from reflex_base.js.finder import ScriptFinder from reflex_base.js.modules import ( JsFunction, find_package_imports, @@ -143,7 +142,6 @@ def test_importing_a_module_exposes_its_exports( script_package: a package with JavaScript modules on sys.path. clean_registration_context: the registry the import fills. """ - assert any(isinstance(finder, ScriptFinder) for finder in sys.meta_path) party = importlib.import_module("js_pkg.party") assert party.__all__ == find_exports(SOURCE) assert party.celebrate == JsFunction("js_pkg.party", "celebrate") @@ -201,8 +199,35 @@ def test_a_function_is_an_event_trigger(script_package: Path): ) +def test_import_reflex_registers_one_finder_and_loads_nothing(): + """``import reflex`` registers the finder without loading a module for it. + + Importing it again, as a reload or a benchmark that clears ``sys.modules`` + does, must not register a second one. + """ + output = subprocess.run( + [ + sys.executable, + "-c", + ( + "import importlib, sys\n" + "import reflex\n" + "for name in [n for n in sys.modules if n.startswith('reflex')]:\n" + " del sys.modules[name]\n" + "importlib.import_module('reflex')\n" + "finders = [f for f in sys.meta_path if getattr(f, '__qualname__', '') == '_ScriptFinder']\n" + "print(len(finders), 'reflex_base.js' in sys.modules)" + ), + ], + capture_output=True, + text=True, + check=True, + ).stdout + assert output == "1 False\n" + + def test_the_finder_imports_nothing_heavy(): - """``import reflex`` installs the finder, so loading it stays cheap.""" + """The real finder loads on the first import no other finder found, so it stays cheap.""" loaded = subprocess.run( [ sys.executable, From ec37f31ede09ca0f52671226e9418ae1a97e6135 Mon Sep 17 00:00:00 2001 From: Farhan Date: Fri, 20 Nov 2026 22:04:49 +0000 Subject: [PATCH 05/16] Pass every trigger argument to *args callables; resolve query imports Review fixes: - call_event_fn counted `*args` as one required parameter and passed only as many arguments as the callable declared, so an uncalled JS module function failed to compile on a trigger with no arguments (`on_mount=initialize`) and got only the first argument of one with several. The same broke any `lambda *args: ...`. Variadic parameters are no longer required, and a `*args` callable gets every argument. - A relative import with a query or fragment (Vite's `./message.txt?raw`, `./logo.png?url`) resolves to its file again, so the file is copied with the module, which keeps the import as written. Package specifiers drop the query before install. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- .../news/+varargs-event-callable.bugfix.md | 1 + .../src/reflex_base/event/__init__.py | 26 +++++++++++++++---- .../reflex-base/src/reflex_base/js/modules.py | 16 +++++++----- .../reflex-base/src/reflex_base/js/source.py | 19 ++++++++++++-- tests/units/reflex_base/js/test_modules.py | 14 ++++++++-- tests/units/reflex_base/js/test_source.py | 3 +++ tests/units/test_event.py | 21 +++++++++++++++ 7 files changed, 84 insertions(+), 16 deletions(-) create mode 100644 packages/reflex-base/news/+varargs-event-callable.bugfix.md diff --git a/packages/reflex-base/news/+varargs-event-callable.bugfix.md b/packages/reflex-base/news/+varargs-event-callable.bugfix.md new file mode 100644 index 00000000000..7b332c8f355 --- /dev/null +++ b/packages/reflex-base/news/+varargs-event-callable.bugfix.md @@ -0,0 +1 @@ +A callable taking `*args` passed to an event trigger, such as `on_mount=lambda *args: ...`, now receives all of the trigger's arguments, and works with triggers that pass none, instead of failing to compile or getting only the first argument. diff --git a/packages/reflex-base/src/reflex_base/event/__init__.py b/packages/reflex-base/src/reflex_base/event/__init__.py index 63844816912..90ce6b70d3d 100644 --- a/packages/reflex-base/src/reflex_base/event/__init__.py +++ b/packages/reflex-base/src/reflex_base/event/__init__.py @@ -2624,6 +2624,12 @@ def args_specs_from_fields( } +_VARIADIC_PARAMETER_KINDS = ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, +) + + def check_fn_match_arg_spec( user_func: Callable, user_func_parameters: Mapping[str, inspect.Parameter], @@ -2646,7 +2652,12 @@ def check_fn_match_arg_spec( Raises: EventFnArgMismatchError: Raised if the number of mandatory arguments do not match """ - user_args = list(user_func_parameters) + # `*args` and `**kwargs` take any number of arguments, none included. + user_args = [ + name + for name, parameter in user_func_parameters.items() + if parameter.kind not in _VARIADIC_PARAMETER_KINDS + ] # Drop the first argument if it's a bound method if inspect.ismethod(user_func) and user_func.__self__ is not None: user_args = user_args[1:] @@ -2748,10 +2759,15 @@ def call_event_fn( # Check that fn signature matches arg_spec check_fn_match_arg_spec(fn, parameters, parsed_args, key=key) - number_of_fn_args = len(parameters) - - # Call the function with the parsed args. - out = fn(*[*parsed_args][:number_of_fn_args]) + # Call the function with the parsed args: as many as it takes, or all of + # them when it takes `*args`. + if any( + parameter.kind is inspect.Parameter.VAR_POSITIONAL + for parameter in parameters.values() + ): + out = fn(*parsed_args) + else: + out = fn(*[*parsed_args][: len(parameters)]) def _dispatch_mixed_event_var(event_like_var: Var) -> FunctionVar: """Wrap a mixed event-like Var into a callable frontend dispatcher. diff --git a/packages/reflex-base/src/reflex_base/js/modules.py b/packages/reflex-base/src/reflex_base/js/modules.py index 377b9f35ded..d71ce25f10c 100644 --- a/packages/reflex-base/src/reflex_base/js/modules.py +++ b/packages/reflex-base/src/reflex_base/js/modules.py @@ -17,7 +17,7 @@ from typing import Any from reflex_base.event import EventHandler, EventSpec, server_side -from reflex_base.js.source import find_exports, find_imports +from reflex_base.js.source import find_exports, find_imports, strip_query from reflex_base.registry import RegistrationContext from reflex_base.utils.format import format_event_handler @@ -36,13 +36,15 @@ def find_package_imports(source: str) -> list[str]: Returns: The import specifiers that name a package, such as ``canvas-confetti`` - or ``@scope/pkg/sub``, without duplicates. + or ``@scope/pkg/sub``, without query or fragment, without duplicates. """ - return [ - specifier - for specifier in find_imports(source) - if not specifier.startswith(_LOCAL_PREFIXES) - ] + return list( + dict.fromkeys( + strip_query(specifier) + for specifier in find_imports(source) + if not specifier.startswith(_LOCAL_PREFIXES) + ) + ) def module_key(name: str) -> str: diff --git a/packages/reflex-base/src/reflex_base/js/source.py b/packages/reflex-base/src/reflex_base/js/source.py index 6efea923a17..3b38d5e7be6 100644 --- a/packages/reflex-base/src/reflex_base/js/source.py +++ b/packages/reflex-base/src/reflex_base/js/source.py @@ -499,16 +499,31 @@ def find_stylesheet_references(source: str) -> list[str]: ) return list( dict.fromkeys( - reference.split("?")[0].split("#")[0] + strip_query(reference) for reference in references if not reference.startswith(("/", "#", "data:")) and "://" not in reference ) ) +def strip_query(specifier: str) -> str: + """Remove the query and fragment of an import specifier. + + Args: + specifier: The specifier, such as ``./logo.png?url``. + + Returns: + The path part, such as ``./logo.png``. + """ + return specifier.split("?", 1)[0].split("#", 1)[0] + + def _resolve_local(directory: Path, specifier: str) -> Path | None: """Find the file a relative import names, as the bundler would. + A query or fragment, as in Vite's ``./logo.png?url``, is not part of the + file name; the copied module keeps it. + Args: directory: The importing file's directory. specifier: The relative specifier, such as ``./helpers`` or ``./x.js``. @@ -516,7 +531,7 @@ def _resolve_local(directory: Path, specifier: str) -> Path | None: Returns: The file, or None if there is none. """ - target = directory / specifier + target = directory / strip_query(specifier) candidates = [ target, *(target.with_name(target.name + ext) for ext in _RESOLVE_EXTENSIONS), diff --git a/tests/units/reflex_base/js/test_modules.py b/tests/units/reflex_base/js/test_modules.py index aee6ffc7cf6..215acac8cc5 100644 --- a/tests/units/reflex_base/js/test_modules.py +++ b/tests/units/reflex_base/js/test_modules.py @@ -40,13 +40,16 @@ def test_find_package_imports(): - """Only specifiers naming an npm package are installed.""" + """Only specifiers naming an npm package are installed, without a query.""" assert find_package_imports(SOURCE) == [ "canvas-confetti", "@scope/pkg/style.css", "lodash/fp", "dayjs", ] + assert find_package_imports( + 'import a from "pkg/a.css?inline";\nimport b from "pkg/a.css";\n' + ) == ["pkg/a.css"] def test_module_key_cannot_collide(): @@ -190,7 +193,14 @@ def test_a_function_is_an_event_trigger(script_package: Path): '"js_pkg$party", ["function"] : "celebrate", ["args"] : [1, "out"] }), ' "({ })))], [_e], ({ }))))" ) - # Like an event handler, an uncalled function takes the trigger's arguments. + # Like an event handler, an uncalled function takes the trigger's arguments: + # none, one, or several. + on_mount = rx.box(on_mount=celebrate) + assert str(LiteralVar.create(on_mount.event_triggers["on_mount"])) == ( + '((...args) => (addEvents([(ReflexEvent("_call_js", ({ ["module"] : ' + '"js_pkg$party", ["function"] : "celebrate", ["args"] : [] }), ' + "({ })))], args, ({ }))))" + ) uncalled = rx.input(on_change=celebrate) assert str(LiteralVar.create(uncalled.event_triggers["on_change"])) == ( '((_e) => (addEvents([(ReflexEvent("_call_js", ({ ["module"] : ' diff --git a/tests/units/reflex_base/js/test_source.py b/tests/units/reflex_base/js/test_source.py index b2af3b5f2ef..08c7fa2c403 100644 --- a/tests/units/reflex_base/js/test_source.py +++ b/tests/units/reflex_base/js/test_source.py @@ -100,6 +100,7 @@ def test_module_files_follows_relative_imports(tmp_path: Path): 'import "./party.css";\n' 'import { w } from "./widgets";\n' 'import { missing } from "./missing";\n' + 'import text from "./message.txt?raw";\n' 'import confetti from "canvas-confetti";\n' ) (tmp_path / "lib" / "a.ts").write_text('import { b } from "../b.js";\n') @@ -112,12 +113,14 @@ def test_module_files_follows_relative_imports(tmp_path: Path): (tmp_path / "img" / "bg.png").write_bytes(b"\x89PNG") (tmp_path / "widgets" / "index.js").write_text("export const w = 1;\n") (tmp_path / "unused.js").write_text("export const u = 1;\n") + (tmp_path / "message.txt").write_text("hello\n") assert module_files(module) == [ module.resolve(), (tmp_path / "lib" / "a.ts").resolve(), (tmp_path / "b.ts").resolve(), (tmp_path / "party.css").resolve(), (tmp_path / "widgets" / "index.js").resolve(), + (tmp_path / "message.txt").resolve(), (tmp_path / "theme.css").resolve(), (tmp_path / "img" / "bg.png").resolve(), ] diff --git a/tests/units/test_event.py b/tests/units/test_event.py index 538daadae01..741f5770e98 100644 --- a/tests/units/test_event.py +++ b/tests/units/test_event.py @@ -2040,3 +2040,24 @@ def handle(s: S): assert twice.fn is once.fn assert twice.event_actions == {"stopPropagation": True} assert S.event_handlers[twice.fn.__name__] is twice + + +def test_call_event_fn_passes_every_trigger_argument_to_varargs(): + """A callable taking ``*args`` gets all of a trigger's arguments, however many. + + ``*args`` is not a required argument, so a trigger with none is valid too. + """ + from reflex_base.event import call_event_fn + + received: list[tuple[str, ...]] = [] + + def record(*args: Var) -> EventSpec: + received.append(tuple(str(arg) for arg in args)) + return rx.noop() + + def no_args() -> list[Var]: + return [] + + call_event_fn(record, no_args, key="on_mount") + call_event_fn(record, lambda a, b: [a, b], key="on_resize") + assert received == [(), ("_a", "_b")] From c65c654fb9d5dc177e1d91130808b5ee9422cbac Mon Sep 17 00:00:00 2001 From: Farhan Date: Fri, 20 Nov 2026 22:05:38 +0000 Subject: [PATCH 06/16] Add docstrings to the tokenizer helpers Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- .../reflex-base/src/reflex_base/js/source.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/packages/reflex-base/src/reflex_base/js/source.py b/packages/reflex-base/src/reflex_base/js/source.py index 3b38d5e7be6..d2ee6a39420 100644 --- a/packages/reflex-base/src/reflex_base/js/source.py +++ b/packages/reflex-base/src/reflex_base/js/source.py @@ -174,11 +174,29 @@ def _skip_regex(source: str, start: int) -> int: def _line_end(source: str, index: int) -> int: + """Find the end of the line a ``//`` comment starts on. + + Args: + source: The source. + index: The index of the comment. + + Returns: + The index of the line break, or the end of the source. + """ end = source.find("\n", index) return len(source) if end < 0 else end def _comment_end(source: str, index: int) -> int: + """Find the end of a ``/* */`` comment. + + Args: + source: The source. + index: The index of the comment. + + Returns: + The index after the closing ``*/``, or the end of the source. + """ end = source.find("*/", index + 2) return len(source) if end < 0 else end + 2 @@ -254,6 +272,16 @@ def tokenize(source: str) -> list[Token]: def _is(token: Token | None, kind: str, value: str | None = None) -> bool: + """Whether a token is of a kind, and has a value if one is given. + + Args: + token: The token, or None past the end. + kind: The kind. + value: The value, or None for any. + + Returns: + Whether it matches. + """ return ( token is not None and token.kind == kind @@ -262,6 +290,15 @@ def _is(token: Token | None, kind: str, value: str | None = None) -> bool: def _at(tokens: list[Token], index: int) -> Token | None: + """Get the token at an index. + + Args: + tokens: The tokens. + index: The index, which may be out of range. + + Returns: + The token, or None if there is none. + """ return tokens[index] if 0 <= index < len(tokens) else None From 780ebe36786d97056302b5abc6363149c0a4c0fd Mon Sep 17 00:00:00 2001 From: Farhan Date: Sat, 21 Nov 2026 21:35:49 +0000 Subject: [PATCH 07/16] Pass a trigger's arguments only to positional parameters A callable taking `**kwargs` or a keyword-only parameter was called with one argument per parameter, so `lambda key, **kw: ...` on on_key_down got two positional arguments and raised TypeError. Count only positional parameters, both to call the function and to check that the trigger provides enough arguments; `*args` still takes all of them. The varargs test moves next to the new one in the event module's test file. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- .../src/reflex_base/event/__init__.py | 66 +++++++++++++------ tests/units/reflex_base/event/test_init.py | 38 +++++++++++ tests/units/test_event.py | 21 ------ 3 files changed, 83 insertions(+), 42 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/event/__init__.py b/packages/reflex-base/src/reflex_base/event/__init__.py index 90ce6b70d3d..21bd6a42c7f 100644 --- a/packages/reflex-base/src/reflex_base/event/__init__.py +++ b/packages/reflex-base/src/reflex_base/event/__init__.py @@ -2624,12 +2624,50 @@ def args_specs_from_fields( } -_VARIADIC_PARAMETER_KINDS = ( - inspect.Parameter.VAR_POSITIONAL, - inspect.Parameter.VAR_KEYWORD, +# The parameters a trigger's arguments are passed to, in order. `*args` takes +# any number of them, none included; keyword-only parameters and `**kwargs` +# take none. +_POSITIONAL_PARAMETER_KINDS = ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, ) +def _positional_parameters( + parameters: Mapping[str, inspect.Parameter], +) -> list[inspect.Parameter]: + """Get the parameters a trigger's arguments are passed to. + + Args: + parameters: The parameters of the function. + + Returns: + Its positional parameters, in order. + """ + return [ + parameter + for parameter in parameters.values() + if parameter.kind in _POSITIONAL_PARAMETER_KINDS + ] + + +def _max_positional_args(parameters: Mapping[str, inspect.Parameter]) -> int | None: + """Get how many of a trigger's arguments a function takes. + + Args: + parameters: The parameters of the function. + + Returns: + The number of its positional parameters, or None when it takes ``*args``. + """ + if any( + parameter.kind is inspect.Parameter.VAR_POSITIONAL + for parameter in parameters.values() + ): + return None + return len(_positional_parameters(parameters)) + + def check_fn_match_arg_spec( user_func: Callable, user_func_parameters: Mapping[str, inspect.Parameter], @@ -2652,20 +2690,13 @@ def check_fn_match_arg_spec( Raises: EventFnArgMismatchError: Raised if the number of mandatory arguments do not match """ - # `*args` and `**kwargs` take any number of arguments, none included. - user_args = [ - name - for name, parameter in user_func_parameters.items() - if parameter.kind not in _VARIADIC_PARAMETER_KINDS - ] + user_args = _positional_parameters(user_func_parameters) # Drop the first argument if it's a bound method if inspect.ismethod(user_func) and user_func.__self__ is not None: user_args = user_args[1:] user_default_args = [ - p.default - for p in user_func_parameters.values() - if p.default is not inspect.Parameter.empty + p.default for p in user_args if p.default is not inspect.Parameter.empty ] number_of_user_args = len(user_args) - number_of_bound_args number_of_user_default_args = len(user_default_args) if user_default_args else 0 @@ -2759,15 +2790,8 @@ def call_event_fn( # Check that fn signature matches arg_spec check_fn_match_arg_spec(fn, parameters, parsed_args, key=key) - # Call the function with the parsed args: as many as it takes, or all of - # them when it takes `*args`. - if any( - parameter.kind is inspect.Parameter.VAR_POSITIONAL - for parameter in parameters.values() - ): - out = fn(*parsed_args) - else: - out = fn(*[*parsed_args][: len(parameters)]) + # Call the function with as many of the parsed args as it takes. + out = fn(*parsed_args[: _max_positional_args(parameters)]) def _dispatch_mixed_event_var(event_like_var: Var) -> FunctionVar: """Wrap a mixed event-like Var into a callable frontend dispatcher. diff --git a/tests/units/reflex_base/event/test_init.py b/tests/units/reflex_base/event/test_init.py index fc4869d245b..d70f7a6cea3 100644 --- a/tests/units/reflex_base/event/test_init.py +++ b/tests/units/reflex_base/event/test_init.py @@ -6,7 +6,9 @@ from reflex_base.event import ( EventArgVar, EventChain, + EventSpec, LambdaEventCallback, + call_event_fn, event_arg_placeholder, event_args_signature, no_args_event_spec, @@ -98,3 +100,39 @@ def test_cond_between_function_and_none_inside_lambda_is_a_no_op_when_false() -> str(chain) == '((_ev_0) => (((_ev_0?.valueOf?.() === "go"?.valueOf?.()) ? ((_ev_0) => (flagRxClientStateSet(_ev_0))) : null)?.(_ev_0)))' ) + + +def test_call_event_fn_passes_every_trigger_argument_to_varargs() -> None: + """A callable taking ``*args`` gets all of a trigger's arguments, however many. + + ``*args`` is not a required argument, so a trigger with none is valid too. + """ + received: list[tuple[str, ...]] = [] + + def record(*args: Var) -> EventSpec: + received.append(tuple(str(arg) for arg in args)) + return rx.noop() + + def no_args() -> list[Var]: + return [] + + call_event_fn(record, no_args, key="on_mount") + call_event_fn(record, lambda a, b: [a, b], key="on_resize") + assert received == [(), ("_a", "_b")] + + +def test_call_event_fn_passes_only_positional_arguments() -> None: + """Keyword-only and ``**kwargs`` parameters take none of the trigger's arguments.""" + received: list[tuple[str, ...]] = [] + + def record_kwargs(key: Var, **kwargs: Any) -> EventSpec: + received.append((str(key), *kwargs)) + return rx.noop() + + def record_keyword_only(key: Var, *, flag: int = 1) -> EventSpec: + received.append((str(key), str(flag))) + return rx.noop() + + call_event_fn(record_kwargs, lambda a, b: [a, b], key="on_key_down") + call_event_fn(record_keyword_only, lambda a, b: [a, b], key="on_key_down") + assert received == [("_a",), ("_a", "1")] diff --git a/tests/units/test_event.py b/tests/units/test_event.py index 741f5770e98..538daadae01 100644 --- a/tests/units/test_event.py +++ b/tests/units/test_event.py @@ -2040,24 +2040,3 @@ def handle(s: S): assert twice.fn is once.fn assert twice.event_actions == {"stopPropagation": True} assert S.event_handlers[twice.fn.__name__] is twice - - -def test_call_event_fn_passes_every_trigger_argument_to_varargs(): - """A callable taking ``*args`` gets all of a trigger's arguments, however many. - - ``*args`` is not a required argument, so a trigger with none is valid too. - """ - from reflex_base.event import call_event_fn - - received: list[tuple[str, ...]] = [] - - def record(*args: Var) -> EventSpec: - received.append(tuple(str(arg) for arg in args)) - return rx.noop() - - def no_args() -> list[Var]: - return [] - - call_event_fn(record, no_args, key="on_mount") - call_event_fn(record, lambda a, b: [a, b], key="on_resize") - assert received == [(), ("_a", "_b")] From b1ee1e4c84654c55bc571671cd02cf92522eba53 Mon Sep 17 00:00:00 2001 From: Farhan Date: Sat, 21 Nov 2026 21:54:50 +0000 Subject: [PATCH 08/16] Rework JavaScript module imports after review Scope: the finder only claims modules of the app's package, the package rxconfig's app_name names, so a third-party package's .js file is never imported or registered. The config loads only once a file is found, and the finder is now a function the stub in reflex_base calls. Parsing: one scanner serves tokenize, template expressions and JSX, and each file is read and tokenized once per compile (module_files returns each script's text, exports and imports). `=>`, `++` and `--` are single tokens, so annotations like `() => Map` and `i++ / 2` read right; `.jsx`/`.tsx` files skip JSX text; type items of export lists, generic arrow functions, `import type` and `export type` are handled. A `const` bound to a literal is a value, not a function. Every new function stays within a complexity of 8. Compile: files must be inside the app package's directory, not merely the project's. utils/scripts.js now maps each module's Python name, as a JSON string, to a loader, so any module name or path is safe in the generated code. Specifiers with a scheme, `#` or `@/` are not installed, and Node.js built-ins fail the compile with what to do instead. Errors: importing a missing function names the file and its functions; importing a value says only functions can be called; the browser error for a missing function gives both causes and both fixes. Frontend: state.js loads the app's modules with the page and caches them, so a `_call_js` runs its function before awaiting anything and the next event, even one processEvent starts concurrently, sees its effect. An app without modules loads nothing. Errors of event code are reported through one helper. Public API: the module helpers are private (_CALL_JS, _JsFunction, _load_module, _render_stub, _package_imports), module_key is gone, and RegistrationContext.script_modules is _script_modules. Tests follow the module layout: test_finder.py, test_modules.py, test_source.py, and reflex_base/test_init.py. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- .../reflex-base/src/reflex_base/__init__.py | 6 +- .../reflex-base/src/reflex_base/js/finder.py | 74 +- .../reflex-base/src/reflex_base/js/modules.py | 191 ++-- .../reflex-base/src/reflex_base/js/source.py | 813 ++++++++++++------ .../reflex-base/src/reflex_base/registry.py | 4 +- packages/reflex-frontend/README.md | 12 +- packages/reflex-frontend/src/utils/scripts.js | 7 +- packages/reflex-frontend/src/utils/state.js | 91 +- .../reflex-frontend/tests/call_js.test.js | 153 +++- .../reflex-frontend/tests/public_api.test.js | 2 +- reflex/compiler/utils.py | 108 ++- tests/units/compiler/state_js.test.mjs | 1 + tests/units/compiler/test_compiler_utils.py | 75 +- .../client_event_queue_runtime.mjs | 1 + tests/units/reflex_base/js/conftest.py | 55 ++ tests/units/reflex_base/js/test_finder.py | 94 ++ tests/units/reflex_base/js/test_modules.py | 282 +++--- tests/units/reflex_base/js/test_source.py | 148 +++- tests/units/reflex_base/test_init.py | 45 + tests/units/reflex_base/test_registry.py | 8 +- 20 files changed, 1450 insertions(+), 720 deletions(-) create mode 100644 tests/units/reflex_base/js/conftest.py create mode 100644 tests/units/reflex_base/js/test_finder.py create mode 100644 tests/units/reflex_base/test_init.py diff --git a/packages/reflex-base/src/reflex_base/__init__.py b/packages/reflex-base/src/reflex_base/__init__.py index b4a24c41558..92b648cd533 100644 --- a/packages/reflex-base/src/reflex_base/__init__.py +++ b/packages/reflex-base/src/reflex_base/__init__.py @@ -17,7 +17,7 @@ class _ScriptFinder: Registered here, by the package every Reflex import loads first, so that no module is loaded for it: the real finder, ``reflex_base.js.finder``, - loads on the first import no other finder found. + loads on the first import of a submodule no other finder found. """ @staticmethod @@ -39,9 +39,9 @@ def find_spec( """ if path is None: return None - from reflex_base.js.finder import ScriptFinder + from reflex_base.js.finder import find_spec - return ScriptFinder.find_spec(fullname, path, target) + return find_spec(fullname, path) # By name, not identity: importing this module afresh, as a reload or a diff --git a/packages/reflex-base/src/reflex_base/js/finder.py b/packages/reflex-base/src/reflex_base/js/finder.py index dbe5b3dae49..e7e03465d3e 100644 --- a/packages/reflex-base/src/reflex_base/js/finder.py +++ b/packages/reflex-base/src/reflex_base/js/finder.py @@ -1,14 +1,15 @@ """Import JavaScript modules from Python. ``from my_app.scripts.party import celebrate`` finds ``my_app/scripts/party.js`` -(or ``.ts``) when no Python module of that name exists. +(or ``.ts``) when no Python module of that name exists and ``my_app`` is the +app's package. Other packages' JavaScript files are never imported. ``import reflex`` registers a small finder that loads this module on the first -import no other finder found, which can happen while ``reflex`` itself loads, -so it stays cheap: ``os.path`` instead of ``pathlib``, a finder and loader that -implement the import protocol without the ``importlib.abc`` base classes, and -no ``typing``. The event system loads with the first JavaScript module -imported. +import of a submodule no other finder found, which can happen while ``reflex`` +itself loads, so it stays cheap: ``os.path`` instead of ``pathlib``, a loader +that implements the import protocol without the ``importlib.abc`` base class, +and no ``typing``. The config loads once a JavaScript file is found, and the +event system with the first JavaScript module imported. """ from __future__ import annotations @@ -57,38 +58,41 @@ def exec_module(self, module: ModuleType) -> None: # pay for the event system until a JavaScript module is imported. from pathlib import Path - from reflex_base.js.modules import load_module + from reflex_base.js.modules import _load_module - load_module(module, Path(self.path)) + _load_module(module, Path(self.path)) -class ScriptFinder: - """Find a JavaScript file where no Python module of the name exists.""" +def _app_package() -> str: + """Get the name of the app's top-level package. - @staticmethod - def find_spec( - fullname: str, - path: Sequence[str] | None, - target: ModuleType | None = None, - ) -> ModuleSpec | None: - """Find the JavaScript file for a submodule. + Returns: + The package ``rxconfig.py`` names, or an empty string without one. + """ + # Imported here: the config loads only once a JavaScript file is found. + from reflex_base.config import get_config - Args: - fullname: The dotted module name. - path: The search path of the parent package; None for a top-level - module, which is never a JavaScript module. - target: Unused. + return get_config().module.partition(".")[0] - Returns: - The spec of the JavaScript module, or None. - """ - if path is None: - return None - name = fullname.rpartition(".")[2] - for entry in path: - for extension in SCRIPT_EXTENSIONS: - # Not pathlib: see the module docstring. - file = os.path.join(entry, name + extension) # noqa: PTH118 - if os.path.isfile(file): # noqa: PTH113 - return ModuleSpec(fullname, _ScriptLoader(file), origin=file) - return None + +def find_spec(fullname: str, path: Sequence[str]) -> ModuleSpec | None: + """Find the JavaScript file for a submodule of the app's package. + + Args: + fullname: The dotted module name. + path: The search path of the parent package. + + Returns: + The spec of the JavaScript module, or None if there is no such file or + it is outside the app's package. + """ + name = fullname.rpartition(".")[2] + for entry in path: + for extension in SCRIPT_EXTENSIONS: + # Not pathlib: see the module docstring. + file = os.path.join(entry, name + extension) # noqa: PTH118 + if os.path.isfile(file): # noqa: PTH113 + if fullname.partition(".")[0] != _app_package(): + return None + return ModuleSpec(fullname, _ScriptLoader(file), origin=file) + return None diff --git a/packages/reflex-base/src/reflex_base/js/modules.py b/packages/reflex-base/src/reflex_base/js/modules.py index e7441be7bb7..dae03020d72 100644 --- a/packages/reflex-base/src/reflex_base/js/modules.py +++ b/packages/reflex-base/src/reflex_base/js/modules.py @@ -1,71 +1,110 @@ """JavaScript modules imported from Python, and calls to their exports. -Each exported function of an imported module becomes a ``JsFunction``. Calling -it returns an event that runs the function in the browser by name: the compiler -copies the module into ``.web`` and re-exports it from ``utils/scripts.js``, so -only the module, the function name and the JSON arguments are sent, and nothing -is evaluated. +Each exported function of an imported module becomes a function that returns +an event. The event runs the function in the browser by name: the compiler +copies the module into ``.web`` and ``utils/scripts.js`` loads it by its Python +module name, so only the module name, the function name and the JSON arguments +are sent, and nothing is evaluated. + +Internal: what an app relies on is the import itself and the events the +imported functions return, which ``docs/api-reference/browser_javascript.md`` +documents. """ from __future__ import annotations import dataclasses +import functools import inspect import keyword from pathlib import Path from types import ModuleType -from typing import Any +from typing import Any, NoReturn from reflex_base.event import EventHandler, EventSpec, server_side -from reflex_base.js.source import find_exports, find_imports, strip_query +from reflex_base.js.source import ScriptSource, read_script, strip_query from reflex_base.registry import RegistrationContext +from reflex_base.utils.exceptions import ReflexRuntimeError from reflex_base.utils.format import format_event_handler # The name of the frontend event that calls a function of a module. -CALL_JS = "_call_js" - -# Specifiers that do not name an npm package. -_LOCAL_PREFIXES = (".", "/", "$/", "node:", "http://", "https://") - - -def find_package_imports(source: str) -> list[str]: - """Find the npm packages a JavaScript module imports. +_CALL_JS = "_call_js" + +# Specifiers that do not name an npm package: relative and absolute paths, the +# app's own modules (`$/`), subpath imports (`#`) and a common path alias +# (`@/`). A specifier with a scheme, such as `virtual:` or `https:`, does not +# either. +_NOT_PACKAGE_PREFIXES = (".", "/", "$/", "#", "@/") +# Node.js built-in modules that no browser provides and no npm package of the +# same name stands in for. +_NODE_BUILTINS = frozenset({ + "async_hooks", + "child_process", + "cluster", + "crypto", + "dgram", + "diagnostics_channel", + "dns", + "fs", + "http", + "http2", + "https", + "inspector", + "module", + "net", + "os", + "perf_hooks", + "readline", + "repl", + "tls", + "trace_events", + "tty", + "v8", + "vm", + "wasi", + "worker_threads", + "zlib", +}) + + +def _package_imports(file: Path, imports: list[str]) -> list[str]: + """Find the npm packages a JavaScript file imports, to install them. Args: - source: The module source. + file: The file. + imports: The specifiers it imports. Returns: - The import specifiers that name a package, such as ``canvas-confetti`` - or ``@scope/pkg/sub``, without query or fragment, without duplicates. - """ - return list( - dict.fromkeys( - strip_query(specifier) - for specifier in find_imports(source) - if not specifier.startswith(_LOCAL_PREFIXES) - ) - ) + The specifiers that name a package, such as ``canvas-confetti`` or + ``@scope/pkg/sub``, without query or fragment, without duplicates. - -def module_key(name: str) -> str: - """Get the name ``utils/scripts.js`` exports a module under. - - ``$`` cannot occur in a Python module name, so two modules never share one. - - Args: - name: The dotted Python module name. - - Returns: - A JavaScript identifier. + Raises: + ReflexRuntimeError: If it imports a Node.js built-in module, which + cannot run in the browser. """ - return name.replace(".", "$") + packages: dict[str, None] = {} + for specifier in imports: + if ( + specifier.startswith("node:") + or specifier.partition("/")[0] in _NODE_BUILTINS + ): + msg = ( + f"The JavaScript file {file} imports {specifier!r}, a Node.js " + "built-in module, which does not exist in the browser. Remove " + "the import, and do the work that needs it in a backend event " + "handler instead." + ) + raise ReflexRuntimeError(msg) + if not specifier.startswith(_NOT_PACKAGE_PREFIXES) and ":" not in specifier: + packages[strip_query(specifier)] = None + return list(packages) def _call_js(module: str, function: str, args: list) -> None: """The signature of the event that calls a function of a module. Args: - module: The module key. + module: The Python name of the module. function: The exported name. args: The arguments. """ @@ -75,7 +114,7 @@ def _call_js(module: str, function: str, args: list) -> None: @dataclasses.dataclass(frozen=True, slots=True) -class JsFunction: +class _JsFunction: """An exported function of a JavaScript module imported from Python.""" # The dotted name of the module. @@ -86,8 +125,8 @@ class JsFunction: def __call__(self, *args: Any) -> EventSpec: """Build the event that calls the function in the browser. - An event handler argument arrives as its full name, which the - function can dispatch with ``reflex.addEvents([reflex.event(name, payload)])``. + An event handler argument arrives as its full name, which the function + can dispatch with ``reflex.addEvents([reflex.event(name, payload)])``. Args: *args: The arguments: values, Vars or event handlers. Values are @@ -97,9 +136,9 @@ def __call__(self, *args: Any) -> EventSpec: The event. """ return server_side( - CALL_JS, + _CALL_JS, _CALL_JS_SIGNATURE, - module=module_key(self.module), + module=self.module, function=self.name, args=[ format_event_handler(arg) if isinstance(arg, EventHandler) else arg @@ -108,21 +147,65 @@ def __call__(self, *args: Any) -> EventSpec: ) -def load_module(module: ModuleType, path: Path) -> None: - """Expose the exports of a JavaScript file on a Python module. +def _missing_export( + module: str, path: Path, script: ScriptSource, name: str +) -> NoReturn: + """Fail to get a name that a JavaScript module does not export as a function. + + Installed as the module's ``__getattr__``. It raises ImportError, not + AttributeError, because ``from module import name`` replaces an + AttributeError with a message of its own, which does not say why. + + Args: + module: The Python name of the module. + path: The JavaScript file. + script: Its source. + name: The name looked up. + + Raises: + AttributeError: For a private or special name, which Python and tools + look up to inspect a module. + ImportError: For any other name. + """ + if name.startswith("_"): + msg = f"module {module!r} has no attribute {name!r}" + raise AttributeError(msg) + if name in script.values: + msg = ( + f"Cannot import {name} from {module}: {path} exports {name} as a " + "value, not a function, and Python can only call functions. Export " + f"a function instead, as in `export function {name}() {{ ... }}`." + ) + else: + exports = ", ".join(script.functions) or "none" + msg = ( + f"Cannot import {name} from {module}: {path} exports no function " + f"named {name} (its functions: {exports}). Export it from " + f"{path.name}, as in `export function {name}() {{ ... }}`." + ) + raise ImportError(msg, name=module, path=str(path)) + + +def _load_module(module: ModuleType, path: Path) -> None: + """Expose the exported functions of a JavaScript file on a Python module. Args: module: The module being imported. path: The JavaScript file. """ - exports = find_exports(path.read_text(encoding="utf-8")) - RegistrationContext.ensure_context().script_modules[module.__name__] = path - for name in exports: - setattr(module, name, JsFunction(module.__name__, name)) - setattr(module, "__all__", exports) # noqa: B010 + script = read_script(path) + RegistrationContext.ensure_context()._script_modules[module.__name__] = path + for name in script.functions: + setattr(module, name, _JsFunction(module.__name__, name)) + setattr(module, "__all__", script.functions) # noqa: B010 + setattr( # noqa: B010 + module, + "__getattr__", + functools.partial(_missing_export, module.__name__, path, script), + ) -def render_stub(path: Path, exports: list[str]) -> str: +def _render_stub(path: Path, functions: list[str]) -> str: """Render a type stub for a JavaScript module, so type checkers see it. Exports that are not Python identifiers cannot be imported and are left @@ -130,7 +213,7 @@ def render_stub(path: Path, exports: list[str]) -> str: Args: path: The JavaScript file. - exports: Its export names. + functions: Its exported functions. Returns: The ``.pyi`` source. @@ -144,7 +227,7 @@ def render_stub(path: Path, exports: list[str]) -> str: ] lines.extend( f"def {name}(*args: Any) -> EventSpec: ..." - for name in exports + for name in functions if name.isidentifier() and not keyword.iskeyword(name) ) return "\n".join(lines) + "\n" diff --git a/packages/reflex-base/src/reflex_base/js/source.py b/packages/reflex-base/src/reflex_base/js/source.py index d2ee6a39420..3fe148bda92 100644 --- a/packages/reflex-base/src/reflex_base/js/source.py +++ b/packages/reflex-base/src/reflex_base/js/source.py @@ -1,9 +1,10 @@ """Read the exports and imports of a JavaScript or TypeScript module. A small tokenizer, not a parser: it knows enough of the language (strings, -template literals, regular expressions and comments) that text inside them is -never read as code, which is what a regular expression over the source gets -wrong. Exports and imports are then read from the tokens. +template literals, regular expressions, comments and, in ``.jsx`` and ``.tsx`` +files, JSX elements) that text inside them is never read as code, which is what +a regular expression over the source gets wrong. Exports and imports are then +read from the tokens. """ from __future__ import annotations @@ -14,6 +15,8 @@ # Files whose imports are followed; other local files (CSS, JSON) are copied. SCRIPT_SOURCE_EXTENSIONS = (".js", ".ts", ".mjs", ".jsx", ".tsx") +# Files whose `<` can start a JSX element. +_JSX_EXTENSIONS = (".jsx", ".tsx") # Stylesheets, whose `@import` and `url(...)` references are followed. STYLESHEET_EXTENSION = ".css" # Tried in order for an import without an extension, as the bundler does. @@ -23,10 +26,24 @@ _CSS_REFERENCE_RE = re.compile( r"""@import\s+(?:url\(\s*)?["']?([^"')\s;]+)|url\(\s*["']?([^"')\s]+)["']?\s*\)""" ) -_IDENTIFIER_RE = re.compile(r"[A-Za-z_$\u0080-￿][\w$\u0080-￿]*") -# A `/` after one of these starts a regular expression, not a division. -_PUNCT_BEFORE_REGEX = frozenset("(,=:[!&|?{};+-*%<>~^") -_KEYWORDS_BEFORE_REGEX = frozenset({ +# Whitespace and comments; an unterminated block comment runs to the end. +_SPACE_RE = re.compile(r"(?:\s+|//[^\n]*|/\*.*?(?:\*/|\Z))*", re.DOTALL) +# Identifiers, keywords and numbers. +_NAME_RE = re.compile(r"[\w$\u0080-\U0010ffff]+") +# A `<` that starts a JSX element or fragment, and not the type parameters of a +# generic arrow function, which a `.tsx` file writes `` or ``. +_JSX_START_RE = re.compile(r"<(?!\s*[\w$]+\s*(?:,|extends\b))(?=>|[A-Za-z_$])") +# What ends the text of a template literal, of a JSX element and of a JSX tag. +_TEMPLATE_STOP_RE = re.compile(r"[\\`]|\$\{") +_JSX_STOP_RE = re.compile(r"[{<]") +_JSX_TAG_STOP_RE = re.compile(r"""["'{>]""") +# Punctuation read as one token: `=>`, so that `>` only closes brackets, and +# `++` and `--`, so that a slash after them is a division. +_LONG_PUNCTUATION = ("=>", "++", "--") +# After these, a value is expected: a slash starts a regular expression, and a +# `<` a JSX element or type parameters. +_PUNCT_BEFORE_OPERAND = frozenset([*"(,=:[!&|?{};+-*%<>~^}", "=>"]) +_KEYWORDS_BEFORE_OPERAND = frozenset({ "return", "typeof", "instanceof", @@ -44,15 +61,19 @@ }) # A line break before a token ends a declaration unless the line before ends # in, or the next one starts with, one of these. -_CONTINUES_EXPRESSION = frozenset("=,+-*/%&|^!~?:<>.([{") +_CONTINUES_EXPRESSION = frozenset([*"=,+-*/%&|^!~?:<>.([{", "=>"]) _STARTS_CONTINUATION = frozenset(".,?:+-*/%&|^=<>)]}") +# Brackets, by their opening character. +_CLOSERS = {"(": ")", "[": "]", "{": "}", "<": ">"} +# Names a lone initializer can be that are never functions. +_LITERAL_NAMES = frozenset({"true", "false", "null", "undefined"}) @dataclasses.dataclass(frozen=True, slots=True) class Token: """A token: a name (identifiers, keywords, numbers), a string, or punctuation.""" - # "name", "string", "punct", or "literal" for templates and regexes. + # "name", "string", "punct", or "literal" for templates, regexes and JSX. kind: str # The text; the contents for a string. value: str @@ -60,6 +81,19 @@ class Token: newline: bool +@dataclasses.dataclass(frozen=True, slots=True) +class ScriptSource: + """A module's source, with the names it exports and the modules it loads.""" + + text: str + # Exported names that may be functions, in the order they appear. + functions: list[str] + # Exported names known not to be functions, such as `export const n = 5`. + values: list[str] + # The specifiers it imports from, without TypeScript's type-only imports. + imports: list[str] + + def _read_string(source: str, start: int) -> tuple[str, int]: """Read a quoted string. @@ -83,59 +117,86 @@ def _read_string(source: str, start: int) -> tuple[str, int]: return source[start + 1 :], len(source) -def _skip_template(source: str, start: int) -> int: +def _skip_expression(source: str, index: int, jsx: bool) -> int: + """Skip the expression in a template's ``${...}`` or JSX's ``{...}``. + + Args: + source: The source. + index: The index after the opening brace. + jsx: Whether the source can contain JSX. + + Returns: + The index after the closing brace. + """ + return _scan(source, index, jsx, nested=True)[1] + + +def _skip_template(source: str, start: int, jsx: bool) -> int: """Skip a template literal, including the expressions in it. Args: source: The source. start: The index of the opening backtick. + jsx: Whether the source can contain JSX. Returns: The index after the closing backtick. """ index = start + 1 - while index < len(source): - char = source[index] - if char == "\\": - index += 2 - elif char == "`": - return index + 1 - elif source.startswith("${", index): - index = _skip_braces(source, index + 2) - else: - index += 1 + while match := _TEMPLATE_STOP_RE.search(source, index): + stop = match.group() + if stop == "`": + return match.end() + index = ( + match.end() + 1 + if stop == "\\" + else _skip_expression(source, match.end(), jsx) + ) return len(source) -def _skip_braces(source: str, index: int) -> int: - """Skip to the brace that closes one already open. +def _skip_jsx_tag(source: str, index: int) -> tuple[int, bool]: + """Skip a JSX tag, with its attributes. Args: source: The source. - index: The index after the opening brace. + index: The index after the tag's ``<``. Returns: - The index after the closing brace. + The index after the tag's ``>``, and whether the tag closes itself. """ - depth = 1 - while index < len(source): - char = source[index] - if char in "'\"": - index = _read_string(source, index)[1] - elif char == "`": - index = _skip_template(source, index) - elif source.startswith("//", index): - index = _line_end(source, index) - elif source.startswith("/*", index): - index = _comment_end(source, index) - else: - if char == "{": - depth += 1 - elif char == "}": - depth -= 1 - if depth == 0: - return index + 1 - index += 1 + while match := _JSX_TAG_STOP_RE.search(source, index): + stop = match.group() + if stop == ">": + return match.end(), source[match.start() - 1] == "/" + index = ( + _skip_expression(source, match.end(), jsx=True) + if stop == "{" + else _read_string(source, match.start())[1] + ) + return len(source), True + + +def _skip_jsx(source: str, index: int) -> int: + """Skip a JSX element or fragment, with its children. + + Args: + source: The source. + index: The index of its opening ``<``. + + Returns: + The index after its closing tag. + """ + depth = 0 + while match := _JSX_STOP_RE.search(source, index): + if match.group() == "{": + index = _skip_expression(source, match.end(), jsx=True) + continue + closing = source.startswith(" int: elif char == "]": in_class = False elif char == "/" and not in_class: - index += 1 - while index < len(source) and ( - source[index].isalnum() or source[index] == "_" - ): - index += 1 - return index + flags = _NAME_RE.match(source, index + 1) + return flags.end() if flags else index + 1 index += 1 return len(source) -def _line_end(source: str, index: int) -> int: - """Find the end of the line a ``//`` comment starts on. +def _operand_expected(previous: Token | None) -> bool: + """Whether a value, rather than an operator, follows a token. + + Args: + previous: The token, or None at the start. + + Returns: + Whether a slash after it starts a regular expression rather than a + division, and a ``<`` starts a JSX element or type parameters rather + than a comparison. + """ + if previous is None: + return True + if previous.kind == "punct": + return previous.value in _PUNCT_BEFORE_OPERAND + return previous.kind == "name" and previous.value in _KEYWORDS_BEFORE_OPERAND + + +def _skip_operand(source: str, index: int, jsx: bool) -> int | None: + """Skip a regular expression or JSX element, where a value is expected. Args: source: The source. - index: The index of the comment. + index: The index of a character that could start one. + jsx: Whether the source can contain JSX. Returns: - The index of the line break, or the end of the source. + The index after it, or None if none starts here. """ - end = source.find("\n", index) - return len(source) if end < 0 else end + if source[index] == "/": + return _skip_regex(source, index) + if jsx and _JSX_START_RE.match(source, index): + return _skip_jsx(source, index) + return None -def _comment_end(source: str, index: int) -> int: - """Find the end of a ``/* */`` comment. +def _read_token( + source: str, index: int, previous: Token | None, jsx: bool +) -> tuple[str, str, int]: + """Read the token at an index. Args: source: The source. - index: The index of the comment. + index: The index, where no whitespace or comment starts. + previous: The token before, or None at the start. + jsx: Whether the source can contain JSX. Returns: - The index after the closing ``*/``, or the end of the source. + The token's kind and value, and the index after it. """ - end = source.find("*/", index + 2) - return len(source) if end < 0 else end + 2 + char = source[index] + if char in "'\"": + value, end = _read_string(source, index) + return "string", value, end + if char == "`": + return "literal", "", _skip_template(source, index, jsx) + if _operand_expected(previous) and ( + (end := _skip_operand(source, index, jsx)) is not None + ): + return "literal", "", end + if name := _NAME_RE.match(source, index): + return "name", name.group(), name.end() + if source.startswith(_LONG_PUNCTUATION, index): + return "punct", source[index : index + 2], index + 2 + return "punct", char, index + 1 -def _regex_allowed(previous: Token | None) -> bool: - """Whether a slash after a token starts a regular expression. +def _scan(source: str, index: int, jsx: bool, nested: bool) -> tuple[list[Token], int]: + """Split a source into tokens, leaving out whitespace and comments. Args: - previous: The token before the slash. + source: The source. + index: The index to start at. + jsx: Whether the source can contain JSX. + nested: Whether to stop after the ``}`` that closes a brace opened + before the index, as for the expression in ``${...}``. Returns: - Whether it starts a regular expression rather than a division. + The tokens, and the index after the last. """ - if previous is None: - return True - if previous.kind == "punct": - return previous.value in _PUNCT_BEFORE_REGEX or previous.value == "}" - return previous.kind == "name" and previous.value in _KEYWORDS_BEFORE_REGEX + tokens: list[Token] = [] + depth = 0 + while True: + # The pattern matches the empty string, so it always matches. + start = _SPACE_RE.match(source, index).end() # pyright: ignore[reportOptionalMemberAccess] + if start >= len(source): + return tokens, start + newline = "\n" in source[index:start] + kind, value, index = _read_token( + source, start, tokens[-1] if tokens else None, jsx + ) + if kind == "punct" and value in "{}": + depth += 1 if value == "{" else -1 + if nested and depth < 0: + return tokens, index + tokens.append(Token(kind, value, newline)) -def tokenize(source: str) -> list[Token]: +def tokenize(source: str, jsx: bool = False) -> list[Token]: """Split a JavaScript or TypeScript source into tokens, without comments. Args: source: The source. + jsx: Whether the source can contain JSX, as a ``.jsx`` or ``.tsx`` file. Returns: The tokens. """ - tokens: list[Token] = [] - newline = False - index = 0 - length = len(source) - while index < length: - char = source[index] - if char == "\n": - newline = True - index += 1 - continue - if char.isspace(): - index += 1 - continue - if source.startswith("//", index): - index = _line_end(source, index) - continue - if source.startswith("/*", index): - end = _comment_end(source, index) - newline = newline or "\n" in source[index:end] - index = end - continue - if char in "'\"": - value, index = _read_string(source, index) - tokens.append(Token("string", value, newline)) - elif char == "`": - index = _skip_template(source, index) - tokens.append(Token("literal", "", newline)) - elif char == "/" and _regex_allowed(tokens[-1] if tokens else None): - index = _skip_regex(source, index) - tokens.append(Token("literal", "", newline)) - elif char.isalnum() or char in "_$" or ord(char) > 127: - end = index + 1 - while end < length and ( - source[end].isalnum() or source[end] in "_$" or ord(source[end]) > 127 - ): - end += 1 - tokens.append(Token("name", source[index:end], newline)) - index = end - else: - tokens.append(Token("punct", char, newline)) - index += 1 - newline = False - return tokens + return _scan(source, 0, jsx, nested=False)[0] def _is(token: Token | None, kind: str, value: str | None = None) -> bool: @@ -302,132 +372,261 @@ def _at(tokens: list[Token], index: int) -> Token | None: return tokens[index] if 0 <= index < len(tokens) else None -def _pattern_names(tokens: list[Token], index: int) -> tuple[list[str], int]: - """Read the names a destructuring pattern declares. +def _is_keyword(tokens: list[Token], index: int, keyword: str) -> bool: + """Whether a token is a keyword, and not a property of that name. Args: tokens: The tokens. - index: The index of the pattern's opening brace or bracket. + index: The index, which may be out of range. + keyword: The keyword. Returns: - The names, and the index after the pattern. + Whether the token is the keyword and does not follow a ``.``. """ - names: list[str] = [] + return _is(_at(tokens, index), "name", keyword) and not _is( + _at(tokens, index - 1), "punct", "." + ) + + +def _skip_group(tokens: list[Token], start: int) -> int: + """Skip a bracketed group, with the groups nested in it. + + Args: + tokens: The tokens. + start: The index of the opening bracket: ``(``, ``[``, ``{`` or ``<``. + + Returns: + The index after the closing bracket. + """ + opener = tokens[start].value + brackets = (opener, _CLOSERS[opener]) depth = 0 - skip_depth = 0 - while index < len(tokens): + for index in range(start, len(tokens)): token = tokens[index] - if token.kind == "punct" and token.value in "{[(": - depth += 1 - elif token.kind == "punct" and token.value in "}])": - if skip_depth == depth: - skip_depth = 0 - depth -= 1 + if token.kind == "punct" and token.value in brackets: + depth += 1 if token.value == opener else -1 if depth == 0: - return names, index + 1 - elif _is(token, "punct", ",") and skip_depth == depth: - skip_depth = 0 - elif _is(token, "punct", "="): - # A default value, up to the next element of this pattern. - skip_depth = skip_depth or depth - elif ( - token.kind == "name" - and not skip_depth - and _IDENTIFIER_RE.fullmatch(token.value) - and not _is(_at(tokens, index + 1), "punct", ":") - ): - names.append(token.value) - index += 1 - return names, index + return index + 1 + return len(tokens) -def _declared_names(tokens: list[Token], index: int) -> list[str]: - """Read the names of an exported ``const``, ``let`` or ``var`` declaration. +def _ends_statement(tokens: list[Token], index: int) -> bool: + """Whether automatic semicolon insertion ends a statement before a token. Args: tokens: The tokens. - index: The index after the keyword. + index: The index of the token, after the statement's first. Returns: - Every declared name, ``export const a = 1, b = 2`` declaring two. + Whether a line break precedes the token and neither the line before + nor the token continues the expression. + """ + token = tokens[index] + previous = tokens[index - 1] + return ( + token.newline + and not (previous.kind == "punct" and previous.value in _CONTINUES_EXPRESSION) + and not (token.kind == "punct" and token.value in _STARTS_CONTINUATION) + ) + + +def _skip_to(tokens: list[Token], index: int, stops: str, annotation: bool) -> int: + """Skip a type annotation or an expression, with the groups in it. + + Args: + tokens: The tokens. + index: The index after the ``:`` or ``=`` it follows. + stops: The punctuation that ends it outside any group. + annotation: Whether it is a type annotation, where every ``<`` opens + type arguments. In an expression, only a ``<`` where a value is + expected does, as type parameters or a type assertion; others + compare. + + Returns: + The index of the token that ends it, or the number of tokens. """ - names: list[str] = [] while index < len(tokens): token = tokens[index] - if token.kind == "name" and _IDENTIFIER_RE.fullmatch(token.value): - names.append(token.value) + if token.kind == "punct" and token.value in stops: + return index + if _ends_statement(tokens, index): + return index + opens = token.kind == "punct" and token.value in _CLOSERS + if opens and ( + annotation or token.value != "<" or _operand_expected(tokens[index - 1]) + ): + index = _skip_group(tokens, index) + else: index += 1 - elif token.kind == "punct" and token.value in "{[": - pattern, index = _pattern_names(tokens, index) - names.extend(pattern) + return index + + +def _skip_annotation(tokens: list[Token], index: int) -> int: + """Skip a declarator's type annotation. + + Args: + tokens: The tokens. + index: The index after its ``:``. + + Returns: + The index of the initializer's ``=``, or of what ends the declarator. + """ + return _skip_to(tokens, index, "=,;)]}", annotation=True) + + +def _skip_initializer(tokens: list[Token], index: int) -> int: + """Skip a declarator's initializer, or a default value in a pattern. + + Args: + tokens: The tokens. + index: The index after its ``=``. + + Returns: + The index of the ``,`` before the next declarator, or of what ends the + declaration. + """ + return _skip_to(tokens, index, ",;)]}", annotation=False) + + +def _is_plain_value(tokens: list[Token], start: int, end: int) -> bool: + """Whether an initializer is a literal that is never a function. + + Args: + tokens: The tokens. + start: The index of its first token. + end: The index after its last token. + + Returns: + Whether it is a lone string, number, template, regular expression, JSX + element, boolean, null or undefined, or an array or object literal. + """ + first = tokens[start] + if first.kind == "punct": + return first.value in ("[", "{") and _skip_group(tokens, start) == end + return end == start + 1 and ( + first.kind in ("string", "literal") + or first.value[0].isdigit() + or first.value in _LITERAL_NAMES + ) + + +def _pattern_names(tokens: list[Token], start: int) -> tuple[list[str], int]: + """Read the names a destructuring pattern declares. + + Args: + tokens: The tokens. + start: The index of the pattern's opening brace or bracket. + + Returns: + The names, and the index after the pattern. + """ + end = _skip_group(tokens, start) + names: list[str] = [] + index = start + 1 + while index < end - 1: + token = tokens[index] + if token.kind == "punct" and token.value in ("{", "["): + nested, index = _pattern_names(tokens, index) + names.extend(nested) + elif _is(token, "punct", "="): + index = _skip_initializer(tokens, index + 1) else: - return names - # Skip a type annotation and the initializer, up to the next declarator. - depth = 0 - angle = 0 - annotation = _is(_at(tokens, index), "punct", ":") - while index < len(tokens): - token = tokens[index] - previous = tokens[index - 1] - if token.kind == "punct": - if token.value in "([{": - depth += 1 - elif token.value in ")]}": - if depth == 0: - return names - depth -= 1 - elif annotation and depth == 0 and token.value in "<>": - angle += 1 if token.value == "<" else -1 - elif depth == 0 and angle == 0 and token.value == "=": - annotation = False - elif depth == 0 and angle == 0 and token.value == ",": - index += 1 - break - elif depth == 0 and token.value == ";": - return names - if ( - depth == 0 - and token.newline - and not ( - previous.kind == "punct" and previous.value in _CONTINUES_EXPRESSION - ) - and not (token.kind == "punct" and token.value in _STARTS_CONTINUATION) - ): - return names + # `{ key: name }` declares name, not key. + if token.kind == "name" and not _is(tokens[index + 1], "punct", ":"): + names.append(token.value) index += 1 + return names, end + + +def _declared_names( + tokens: list[Token], index: int, constant: bool +) -> list[tuple[str, bool]]: + """Read the names of an exported ``const``, ``let`` or ``var`` declaration. + + Args: + tokens: The tokens. + index: The index after the keyword. + constant: Whether it is ``const``, whose names keep their initial value. + + Returns: + Every declared name, ``export const a = 1, b = 2`` declaring two, and + whether it may be a function. + """ + names: list[tuple[str, bool]] = [] + while (token := _at(tokens, index)) is not None: + if token.kind == "name": + declared = [token.value] + index += 1 + elif token.kind == "punct" and token.value in ("{", "["): + declared, index = _pattern_names(tokens, index) else: - return names + break + if _is(_at(tokens, index), "punct", ":"): + index = _skip_annotation(tokens, index + 1) + callable_ = True + if _is(_at(tokens, index), "punct", "="): + start = index + 1 + index = _skip_initializer(tokens, start) + # Only a name bound by `const` to a literal is known not to be one. + callable_ = not ( + constant + and token.kind == "name" + and _is_plain_value(tokens, start, index) + ) + names.extend((name, callable_) for name in declared) + if not _is(_at(tokens, index), "punct", ","): + break + index += 1 return names -def _list_names(tokens: list[Token], index: int) -> list[str]: +def _list_names(tokens: list[Token], start: int) -> list[str]: """Read the names an export list exports: ``{ a, b as c, "d" as e }``. Args: tokens: The tokens. - index: The index after the opening brace. + start: The index of the opening brace. Returns: - The exported names. + The exported names, without TypeScript's ``type`` items. """ + end = _skip_group(tokens, start) names: list[str] = [] - current: str | None = None - while index < len(tokens): - token = tokens[index] - if _is(token, "punct", "}") or _is(token, "punct", ","): - if current is not None: - names.append(current) - current = None - if token.value == "}": - break - elif token.kind in ("name", "string") and token.value != "as": + item_start = start + 1 + for index in range(item_start, end): + if index < end - 1 and not _is(tokens[index], "punct", ","): + continue + item = tokens[item_start:index] + item_start = index + 1 + # `type Foo` exports a type, but `type` alone or `type as t` a value. + is_type = ( + len(item) > 1 + and _is(item[0], "name", "type") + and not _is(item[1], "name", "as") + ) + if item and not is_type: # The last name of an item is its exported name: `a as b` exports b. - current = token.value - index += 1 + names.append(item[-1].value) return names -def _export_names(tokens: list[Token], index: int) -> list[str]: +def _function_name(tokens: list[Token], index: int) -> list[tuple[str, bool]]: + """Read the name of an exported function declaration. + + Args: + tokens: The tokens. + index: The index after ``function``. + + Returns: + The name, or none for an anonymous function. + """ + if _is(_at(tokens, index), "punct", "*"): + index += 1 + name = _at(tokens, index) + return [(name.value, True)] if name is not None and name.kind == "name" else [] + + +def _export_names(tokens: list[Token], index: int) -> list[tuple[str, bool]]: """Read the names one ``export`` statement exports. Args: @@ -435,87 +634,123 @@ def _export_names(tokens: list[Token], index: int) -> list[str]: index: The index after ``export``. Returns: - The names; none for a default export, a class, a namespace re-export or - a type. + The names, and whether each may be a function; none for a default + export, a class, a namespace re-export or a type. """ - token = _at(tokens, index) - if _is(token, "name", "async"): + if _is(_at(tokens, index), "name", "async"): index += 1 - token = _at(tokens, index) + token = _at(tokens, index) + if token is None: + return [] if _is(token, "name", "function"): - index += 1 - if _is(_at(tokens, index), "punct", "*"): - index += 1 - name = _at(tokens, index) - return [name.value] if name is not None and name.kind == "name" else [] - if ( - token is not None - and token.kind == "name" - and token.value in ("const", "let", "var") - ): - return _declared_names(tokens, index + 1) + return _function_name(tokens, index + 1) + if token.kind == "name" and token.value in ("const", "let", "var"): + return _declared_names(tokens, index + 1, constant=token.value == "const") if _is(token, "punct", "{"): - return _list_names(tokens, index + 1) + return [(name, True) for name in _list_names(tokens, index)] return [] -def find_exports(source: str) -> list[str]: - """Find the names of the functions a module may export. +def _statement_start(tokens: list[Token], index: int) -> int | None: + """Find the ``import`` or ``export`` keyword an import clause follows. + + Args: + tokens: The tokens. + index: The index of the clause's last token, before ``from``. + + Returns: + The index of the keyword, or None if the tokens are not an import + clause. + """ + while index >= 0: + token = tokens[index] + if token.kind == "name" and token.value in ("import", "export"): + return index + if token.kind == "punct" and token.value not in ("{", "}", ",", "*"): + return None + index -= 1 + return None + + +def _is_runtime_import(tokens: list[Token], index: int) -> bool: + """Whether a string is the specifier of a module the source loads. + + Covers ``import ... from "x"``, ``import "x"``, ``import("x")`` and + ``export ... from "x"``, but not TypeScript's ``import type`` and + ``export type``, which load nothing. - Covers declared functions and variables (``export function f``, + Args: + tokens: The tokens. + index: The index of the string. + + Returns: + Whether it is such a specifier. + """ + if _is(_at(tokens, index - 1), "punct", "("): + return _is_keyword(tokens, index - 2, "import") + if _is_keyword(tokens, index - 1, "import"): + return True + if not _is(_at(tokens, index - 1), "name", "from"): + return False + start = _statement_start(tokens, index - 2) + if start is None: + return False + # `import type X from` loads nothing, but `import type from` and + # `import type, { x } from` import a default export named type. + after = _at(tokens, start + 2) + return not ( + _is(tokens[start + 1], "name", "type") + and not _is(after, "name", "from") + and not _is(after, "punct", ",") + ) + + +def parse_script(text: str, jsx: bool = False) -> ScriptSource: + """Read the exports and imports of a module. + + Exports cover declared functions and variables (``export function f``, ``export const f = ...``, also several or destructured) and export lists (``export { a, b as c }``, also re-exported ``from`` another module). A default export, a class, ``export *`` and TypeScript types have no function name to import in Python and are left out. Args: - source: The module source. + text: The module source. + jsx: Whether it can contain JSX, as a ``.jsx`` or ``.tsx`` file. Returns: - The names, in the order they appear, without duplicates. + The source with its exports and imports, each without duplicates. """ - tokens = tokenize(source) - names: list[str] = [] + tokens = tokenize(text, jsx) + exports: dict[str, bool] = {} + imports: dict[str, None] = {} for index, token in enumerate(tokens): - if _is(token, "name", "export") and not _is( - _at(tokens, index - 1), "punct", "." - ): - names.extend(_export_names(tokens, index + 1)) - return [name for name in dict.fromkeys(names) if name != "default"] - + if token.kind == "string" and _is_runtime_import(tokens, index): + imports[token.value] = None + elif _is_keyword(tokens, index, "export"): + for name, callable_ in _export_names(tokens, index + 1): + exports.setdefault(name, callable_) + exports.pop("default", None) + return ScriptSource( + text=text, + functions=[name for name, callable_ in exports.items() if callable_], + values=[name for name, callable_ in exports.items() if not callable_], + imports=list(imports), + ) -def find_imports(source: str) -> list[str]: - """Find the specifiers a module imports or re-exports from. - Covers ``import ... from "x"``, ``import "x"``, ``import("x")`` and - ``export ... from "x"``, wherever line breaks fall. +def read_script(path: Path) -> ScriptSource: + """Read a module file's exports and imports. Args: - source: The module source. + path: The file. Returns: - The specifiers, without duplicates. + Its source with its exports and imports. """ - tokens = tokenize(source) - specifiers: list[str] = [] - for index, token in enumerate(tokens): - if token.kind != "string": - continue - keyword_index = index - 1 - if _is(_at(tokens, keyword_index), "punct", "("): - # A dynamic import: `import("x")`. - keyword_index -= 1 - if not _is(_at(tokens, keyword_index), "name", "import"): - continue - keyword = _at(tokens, keyword_index) - if ( - keyword is not None - and keyword.kind == "name" - and keyword.value in ("from", "import") - and not _is(_at(tokens, keyword_index - 1), "punct", ".") - ): - specifiers.append(token.value) - return list(dict.fromkeys(specifiers)) + return parse_script( + path.read_text(encoding="utf-8"), jsx=path.suffix in _JSX_EXTENSIONS + ) def find_stylesheet_references(source: str) -> list[str]: @@ -580,40 +815,58 @@ def _resolve_local(directory: Path, specifier: str) -> Path | None: return next((candidate for candidate in candidates if candidate.is_file()), None) -def module_files(path: Path) -> list[Path]: +def _read_references(file: Path) -> tuple[ScriptSource | None, list[Path]]: + """Read a file and find the local files it references. + + Args: + file: The file. + + Returns: + The script, or None for another kind of file, and the resolved files + its relative imports, or a stylesheet's ``@import`` and ``url(...)`` + references, name. + """ + if file.suffix in SCRIPT_SOURCE_EXTENSIONS: + script = read_script(file) + return script, [ + target.resolve() + for specifier in script.imports + if specifier.startswith(".") + and (target := _resolve_local(file.parent, specifier)) is not None + ] + if file.suffix == STYLESHEET_EXTENSION: + return None, [ + target.resolve() + for reference in find_stylesheet_references( + file.read_text(encoding="utf-8") + ) + if (target := file.parent / reference).is_file() + ] + # Images, fonts, data: nothing further to follow. + return None, [] + + +def module_files(path: Path) -> dict[Path, ScriptSource | None]: """Find a module and every local file it imports, directly or not. Scripts' relative imports are followed, and so are the ``@import`` and - ``url(...)`` references of the stylesheets among them. + ``url(...)`` references of the stylesheets among them. Each file is read + once. Args: path: The module. Returns: - The resolved files, the module first. + The resolved files, the module first, each with its script, or None if + it is not a script. """ - files = {path.resolve(): None} + files: dict[Path, ScriptSource | None] = {path.resolve(): None} pending = [path.resolve()] while pending: file = pending.pop() - if file.suffix in SCRIPT_SOURCE_EXTENSIONS: - targets = [ - _resolve_local(file.parent, specifier) - for specifier in find_imports(file.read_text(encoding="utf-8")) - if specifier.startswith(".") - ] - elif file.suffix == STYLESHEET_EXTENSION: - targets = [ - target if (target := file.parent / reference).is_file() else None - for reference in find_stylesheet_references( - file.read_text(encoding="utf-8") - ) - ] - else: - # Images, fonts, data: nothing further to follow. - continue + files[file], targets = _read_references(file) for target in targets: - if target is not None and (target := target.resolve()) not in files: + if target not in files: files[target] = None pending.append(target) - return list(files) + return files diff --git a/packages/reflex-base/src/reflex_base/registry.py b/packages/reflex-base/src/reflex_base/registry.py index 0315be0a8a3..5251c6633cd 100644 --- a/packages/reflex-base/src/reflex_base/registry.py +++ b/packages/reflex-base/src/reflex_base/registry.py @@ -75,7 +75,7 @@ class RegistrationContext(BaseContext): default_factory=dict, repr=False ) # JavaScript modules imported from Python, by module name. - script_modules: dict[str, Path] = dataclasses.field( + _script_modules: dict[str, Path] = dataclasses.field( default_factory=dict, repr=False ) _app: App | None = dataclasses.field(default=None, repr=False) @@ -169,7 +169,7 @@ def fork(self) -> Self: decorated_pages=list(self.decorated_pages), bundled_libraries=list(self.bundled_libraries), _explicit_bundled_libraries=dict(self._explicit_bundled_libraries), - script_modules=dict(self.script_modules), + _script_modules=dict(self._script_modules), ) def _set_config(self, config: Config) -> None: diff --git a/packages/reflex-frontend/README.md b/packages/reflex-frontend/README.md index 336929639ba..97593dc457e 100644 --- a/packages/reflex-frontend/README.md +++ b/packages/reflex-frontend/README.md @@ -47,15 +47,17 @@ without it appearing in its arguments. What there is, and where it is pinned: - **Module-level state in `utils/state.js`**: the event queue, the session token, the socket warmed up before React mounts, the record of sent - localStorage values. Every mounted event loop and every drain shares them. - A test that needs a clean copy reloads the module (`vi.resetModules()`, as + localStorage values, the app's JavaScript modules loaded so far. Every + mounted event loop and every drain shares them. A test that needs a clean + copy reloads the module (`vi.resetModules()`, as `tests/state_connection.test.js` does). Code that awaits must not assume the queue or the socket is as it left them. - **Work done on import**: `utils/state.js` creates the cookie reader, always; publishes the client state store on `refs`, when there is a `document`; and - starts a socket transport, when there is a `window`. A production build - evaluates every module in node to prerender pages, so anything added here has - to load without either. `tests/public_api.test.js` imports each module in + starts a socket transport and loading the JavaScript modules the app imports + from Python (`utils/scripts.js`), when there is a `window`. A production + build evaluates every module in node to prerender pages, so anything added + here has to load without either. `tests/public_api.test.js` imports each module in node to keep it that way. - **Script scope**: event code sent by the backend (`rx.call_script`, event callbacks) runs through `runScript` in `utils/state.js`, which compiles it diff --git a/packages/reflex-frontend/src/utils/scripts.js b/packages/reflex-frontend/src/utils/scripts.js index 844e9c509fc..7b411d14686 100644 --- a/packages/reflex-frontend/src/utils/scripts.js +++ b/packages/reflex-frontend/src/utils/scripts.js @@ -1,3 +1,4 @@ -// The JavaScript modules an app imports from Python, by module key. The -// compiler rewrites this file; an app that imports none keeps it empty. -export {}; +// The JavaScript modules an app imports from Python: each module's Python name +// maps to a function that loads it. The compiler rewrites this file; an app +// that imports none keeps it empty. +export default {}; diff --git a/packages/reflex-frontend/src/utils/state.js b/packages/reflex-frontend/src/utils/state.js index 099a40aecf8..5e500a6483b 100644 --- a/packages/reflex-frontend/src/utils/state.js +++ b/packages/reflex-frontend/src/utils/state.js @@ -14,6 +14,7 @@ import { app, eventLoop } from "$/utils/context-registry"; // This module's own exports, passed to server-provided scripts by runScript. import * as stateExports from "$/utils/state"; import { reflex } from "$/utils/reflex"; +import scriptModules from "$/utils/scripts"; import { CLIENT_STATE_REF, getClientState, @@ -564,25 +565,79 @@ const runScript = (code, addEvents, socket, navigate, params) => { ); }; +/** + * The namespace of each JavaScript module the app imports from Python that has + * loaded, by the module's Python name. + * @type {Record>} + */ +const loadedScriptModules = {}; +/** + * The load of each of those modules that has started, by the same name. + * @type {Record>} + */ +const scriptModuleLoads = {}; + +/** + * Load a JavaScript module the app imports from Python, once. + * @param {string} module The module's Python name. + * @returns {Promise} Settles once the module has loaded or failed to. + */ +const loadScriptModule = (module) => + (scriptModuleLoads[module] ??= scriptModules[module]().then((namespace) => { + loadedScriptModules[module] = namespace; + })); + +// Load the app's modules with the page, so that a `_call_js` finds its module +// loaded and calls the function without awaiting: `processEvent` may start the +// next event while one awaits, and that event must see what the function did. +// An app that imports no module loads nothing. A module that fails to load is +// reported when one of its functions is called. +if (typeof window !== "undefined") { + for (const module of Object.keys(scriptModules)) { + loadScriptModule(module).catch(() => {}); + } +} + /** * Call an exported function of a JavaScript module the app imports from * Python. The modules are bundled, so nothing is evaluated: the event names the * module and the function, and carries the arguments as JSON. * - * Loaded on first use, so an app that never calls one does not fetch the chunk. + * A function of a loaded module runs before this returns, so its effects are + * visible to the next event even if it returns a promise. * @param {{module: string, function: string, args: any[]}} payload The call. - * @returns {Promise} The function's result. + * @returns {any} The function's result, or a promise of it if the module had + * not loaded yet. */ -const callModuleFunction = async ({ module, function: name, args }) => { - /** @type {Record>} */ - const modules = await import("$/utils/scripts"); - const fn = Object.hasOwn(modules, module) ? modules[module][name] : undefined; +const callModuleFunction = (payload) => { + const { module, function: name, args } = payload; + if ( + !Object.hasOwn(loadedScriptModules, module) && + Object.hasOwn(scriptModules, module) + ) { + return loadScriptModule(module).then(() => callModuleFunction(payload)); + } + const fn = Object.hasOwn(loadedScriptModules, module) + ? loadedScriptModules[module][name] + : undefined; if (typeof fn !== "function") { throw new TypeError( - `${module.replaceAll("$", ".")}.${name} is not a function exported by a JavaScript module this app imports.`, + `Cannot call ${module}.${name}: the frontend has no function ${name} in the JavaScript module ${module}. Either the module does not export a function of that name, or the frontend was built before the backend imported it. Export a function named ${name} from the module, or restart \`reflex run\` to rebuild the frontend.`, ); } - return await fn(...args); + return fn(...args); +}; + +/** + * Report an error thrown by code an event ran, as an uncaught error would be. + * @param {string} name The event's name, logged with the error. + * @param {any} error The error. + */ +const reportEventError = (name, error) => { + console.log(name, error); + if (window?.onerror) { + window.onerror(error.message, null, null, null, error); + } }; /** @@ -823,10 +878,7 @@ export const applyEvent = async (event, socket, navigate, params) => { params, ); } catch (e) { - console.log("_client_state_get", e); - if (window?.onerror) { - window.onerror(e.message, null, null, null, e); - } + reportEventError("_client_state_get", e); } return; } @@ -844,10 +896,7 @@ export const applyEvent = async (event, socket, navigate, params) => { params, ); } catch (e) { - console.log("_call_function", e); - if (window?.onerror) { - window.onerror(e.message, null, null, null, e); - } + reportEventError("_call_function", e); } return; } @@ -856,10 +905,7 @@ export const applyEvent = async (event, socket, navigate, params) => { try { await callModuleFunction(event.payload); } catch (e) { - console.log("_call_js", e); - if (window?.onerror) { - window.onerror(e.message, null, null, null, e); - } + reportEventError("_call_js", e); } return; } @@ -885,10 +931,7 @@ export const applyEvent = async (event, socket, navigate, params) => { await applyResultCallback(event, eval_result, socket, navigate, params); } catch (e) { - console.log("_call_script", e); - if (window?.onerror) { - window.onerror(e.message, null, null, null, e); - } + reportEventError("_call_script", e); } return; } diff --git a/packages/reflex-frontend/tests/call_js.test.js b/packages/reflex-frontend/tests/call_js.test.js index c2ccf6a5919..6038b9bf731 100644 --- a/packages/reflex-frontend/tests/call_js.test.js +++ b/packages/reflex-frontend/tests/call_js.test.js @@ -1,75 +1,142 @@ /** * `_call_js` calls an exported function of a JavaScript module an app imports - * from Python. The modules are bundled and re-exported by utils/scripts.js, so - * the event carries only names and JSON arguments, and nothing is evaluated. + * from Python. utils/scripts.js maps each module's Python name to a function + * that loads it, so the event carries only names and JSON arguments, and + * nothing is evaluated. */ -import { afterEach, expect, test, vi } from "vitest"; +import { afterEach, beforeAll, expect, test, vi } from "vitest"; -import { applyEvent } from "$/utils/state"; +import { applyEvent, queueEvents } from "$/utils/state"; -const seen = []; +const modules = vi.hoisted(() => { + let releaseSlow = () => {}; + const slowLoaded = new Promise((resolve) => { + releaseSlow = resolve; + }); + return { seen: [], releaseSlow, slowLoaded }; +}); vi.mock("$/utils/scripts", () => ({ - app$scripts$party: { - celebrate: (count, id) => seen.push(["celebrate", count, id]), - later: async (value) => { - await Promise.resolve(); - seen.push(["later", value]); - }, - notAFunction: 3, + default: { + "app.scripts.party": async () => ({ + celebrate: (count, id) => modules.seen.push(["celebrate", count, id]), + later: async (value) => { + await Promise.resolve(); + modules.seen.push(["later", value]); + }, + write: (value) => { + globalThis.partyValue = value; + }, + fails: () => { + throw new RangeError("The party is over."); + }, + notAFunction: 3, + }), + "app.scripts.slow": () => + modules.slowLoaded.then(() => ({ + ping: (value) => modules.seen.push(["ping", value]), + })), + "app.scripts.broken": () => + Promise.reject(new Error("Failed to fetch the module.")), }, })); +beforeAll(async () => { + // Let the loads state.js starts on import settle. + await new Promise((resolve) => setTimeout(resolve)); +}); + afterEach(() => { - seen.length = 0; + modules.seen.length = 0; window.onerror = null; }); +const callJsEvent = (module, fn, args) => ({ + name: "_call_js", + payload: { module, function: fn, args }, +}); + const callJs = (module, fn, args) => - applyEvent( - { name: "_call_js", payload: { module, function: fn, args } }, + applyEvent(callJsEvent(module, fn, args), null, null, null); + +test("calls the named function with the arguments", async () => { + await callJs("app.scripts.party", "celebrate", [200, "results"]); + expect(modules.seen).toEqual([["celebrate", 200, "results"]]); +}); + +test("waits for an async function before the next event", async () => { + await callJs("app.scripts.party", "later", ["done"]); + expect(modules.seen).toEqual([["later", "done"]]); +}); + +test("the next event sees what a function did, though events overlap", async () => { + // processEvent starts an event that arrives while another awaits, so the + // function must run before _call_js awaits anything. + globalThis.reads = []; + const socket = { connected: true, emit: vi.fn() }; + const first = queueEvents( + [callJsEvent("app.scripts.party", "write", ["new"])], + socket, + false, + null, null, + ); + const second = queueEvents( + [ + { + name: "_call_script", + payload: { javascript_code: "reads.push(partyValue)" }, + }, + ], + socket, + false, null, null, ); - -test("calls the named function with the arguments", async () => { - await callJs("app$scripts$party", "celebrate", [200, "results"]); - expect(seen).toEqual([["celebrate", 200, "results"]]); + await Promise.all([first, second]); + expect(globalThis.reads).toEqual(["new"]); }); -test("waits for an async function before the next event", async () => { - await callJs("app$scripts$party", "later", ["done"]); - expect(seen).toEqual([["later", "done"]]); +test("a function of a module still loading runs once it has loaded", async () => { + const call = callJs("app.scripts.slow", "ping", ["late"]); + await Promise.resolve(); + expect(modules.seen).toEqual([]); + modules.releaseSlow(); + await call; + expect(modules.seen).toEqual([["ping", "late"]]); }); test("reports a missing module, function or non-function export", async () => { const onerror = vi.fn(); window.onerror = onerror; - await callJs("app$scripts$party", "missing", []); - await callJs("app$scripts$party", "notAFunction", []); - await callJs("app$other", "celebrate", []); - expect(onerror.mock.calls.map((call) => [call[0], call[4].name])).toEqual([ - [ - "app.scripts.party.missing is not a function exported by a JavaScript module this app imports.", - "TypeError", - ], - [ - "app.scripts.party.notAFunction is not a function exported by a JavaScript module this app imports.", - "TypeError", - ], - [ - "app.other.celebrate is not a function exported by a JavaScript module this app imports.", - "TypeError", - ], + await callJs("app.scripts.party", "missing", []); + await callJs("app.scripts.party", "notAFunction", []); + await callJs("app.other", "celebrate", []); + const reported = (module, name) => { + const message = `Cannot call ${module}.${name}: the frontend has no function ${name} in the JavaScript module ${module}. Either the module does not export a function of that name, or the frontend was built before the backend imported it. Export a function named ${name} from the module, or restart \`reflex run\` to rebuild the frontend.`; + return [message, null, null, null, new TypeError(message)]; + }; + expect(onerror.mock.calls).toEqual([ + reported("app.scripts.party", "missing"), + reported("app.scripts.party", "notAFunction"), + reported("app.other", "celebrate"), ]); - expect(seen).toEqual([]); + expect(modules.seen).toEqual([]); }); -test("a function that throws is reported, not raised", async () => { +test("a function that throws, or a module that fails to load, is reported", async () => { const onerror = vi.fn(); window.onerror = onerror; - await callJs("app$scripts$party", "celebrate", null); - expect(onerror).toHaveBeenCalledTimes(1); - expect(onerror.mock.calls[0][4]).toBeInstanceOf(TypeError); + await callJs("app.scripts.party", "fails", []); + await callJs("app.scripts.broken", "anything", []); + expect(onerror.mock.calls).toEqual([ + ["The party is over.", null, null, null, new RangeError("The party is over.")], + [ + "Failed to fetch the module.", + null, + null, + null, + new Error("Failed to fetch the module."), + ], + ]); }); diff --git a/packages/reflex-frontend/tests/public_api.test.js b/packages/reflex-frontend/tests/public_api.test.js index e1090734e0c..9e975888a4d 100644 --- a/packages/reflex-frontend/tests/public_api.test.js +++ b/packages/reflex-frontend/tests/public_api.test.js @@ -83,7 +83,7 @@ const PUBLIC_API = { ], "$/utils/react-theme": ["ThemeProvider", "useTheme"], "$/utils/reflex": ["reflex"], - "$/utils/scripts": [], + "$/utils/scripts": ["default"], "$/utils/helpers/array": ["default"], "$/utils/helpers/datetime": ["default"], "$/utils/helpers/debounce": ["default"], diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index 052a5b1600f..a942dc9ffd0 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -15,7 +15,7 @@ from dataclasses import MISSING from datetime import datetime from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal, TypedDict, get_args, get_origin +from typing import TYPE_CHECKING, Any, Literal, TypedDict, cast, get_args, get_origin from urllib.parse import urlparse from reflex_base import constants @@ -32,8 +32,8 @@ MemoParamKind, ) from reflex_base.constants.state import FIELD_MARKER -from reflex_base.js.modules import find_package_imports, module_key, render_stub -from reflex_base.js.source import SCRIPT_SOURCE_EXTENSIONS, find_exports, module_files +from reflex_base.js.modules import _package_imports, _render_stub +from reflex_base.js.source import ScriptSource, module_files from reflex_base.registry import RegistrationContext from reflex_base.style import Style from reflex_base.utils import format, imports, memo_paths, serializers, types @@ -302,6 +302,45 @@ def _compile_bundled_libraries() -> tuple[str, str]: return constants.Dirs.BUNDLED_LIBRARIES, format.json_dumps(bundled_libraries) +def _script_module_files( + modules_dir: Path, +) -> tuple[dict[Path, tuple[Path, ScriptSource | None]], dict[str, Path]]: + """Find the files of the JavaScript modules the app imports from Python. + + Args: + modules_dir: The directory the files are copied to. + + Returns: + Each resolved file with its target and its script, or None if it is + not a script, and each module's file, by module name. + + Raises: + ReflexRuntimeError: If a module imports a file outside its package's + directory. + """ + files: dict[Path, tuple[Path, ScriptSource | None]] = {} + modules: dict[str, Path] = {} + for name, path in sorted( + RegistrationContext.ensure_context()._script_modules.items() + ): + package_dir = path.resolve().parents[name.count(".") - 1] + for file, script in module_files(path).items(): + if not file.is_relative_to(package_dir): + msg = ( + f"The JavaScript module {name} imports {file}, which is " + f"outside {package_dir}, the directory of the app's package, " + "and only that directory is copied into the compiled app. " + "Move the file into it, or publish it as an npm package " + "and import it by name." + ) + raise ReflexRuntimeError(msg) + files.setdefault( + file, (modules_dir / file.relative_to(package_dir.parent), script) + ) + modules[name] = path.resolve() + return files, modules + + def compile_script_modules() -> tuple[ list[tuple[str, str]], dict[str, list[ImportVar]], list[tuple[Path, Path]] ]: @@ -309,56 +348,41 @@ def compile_script_modules() -> tuple[ Each module and the local files it imports, directly or not, are copied under ``utils/script_modules`` at their paths from the directory that holds - the module's top-level package, so relative imports resolve as they do in - the source tree. ``utils/scripts.js`` re-exports each module under its - module key, and a type stub is written next to it, for type checkers. + the app's package, so relative imports resolve as they do in the source + tree. ``utils/scripts.js`` maps each module's Python name to a function + that loads it, and a type stub is written next to each module, for type + checkers. Returns: - The (path, code) outputs of the scripts, the npm packages they import, - to install, and the (source, target) copies of the other files they + The (path, code) outputs, the npm packages the scripts import, to + install, and the (source, target) copies of the other files they reference, such as stylesheets and images, which may be binary. - - Raises: - ReflexRuntimeError: If a module imports a file outside its top-level - package's directory. """ web_dir = get_web_dir() - modules_dir = web_dir / constants.Dirs.SCRIPT_MODULES_PATH + files, modules = _script_module_files(web_dir / constants.Dirs.SCRIPT_MODULES_PATH) outputs: list[tuple[str, str]] = [] copies: list[tuple[Path, Path]] = [] package_imports: dict[str, list[ImportVar]] = {} - index: list[str] = [] - copied: set[Path] = set() - modules = RegistrationContext.ensure_context().script_modules - for name, path in sorted(modules.items()): - root = path.resolve().parents[name.count(".")] - for file in module_files(path): - if not file.is_relative_to(root): - msg = ( - f"The JavaScript module {name} imports {file}, which is " - f"outside {root}, the directory of its package." - ) - raise ReflexRuntimeError(msg) - if file in copied: - continue - copied.add(file) - target = modules_dir / file.relative_to(root) - if file.suffix not in SCRIPT_SOURCE_EXTENSIONS: - copies.append((file, target)) - continue - source = file.read_text(encoding="utf-8") - outputs.append((str(target), source)) - for specifier in find_package_imports(source): - package_imports[specifier] = [ImportVar(tag=None, render=False)] - specifier = (modules_dir / path.resolve().relative_to(root)).relative_to( - web_dir + for file, (target, script) in files.items(): + if script is None: + copies.append((file, target)) + continue + outputs.append((str(target), script.text)) + package_imports.update( + (package, [ImportVar(tag=None, render=False)]) + for package in _package_imports(file, script.imports) ) - index.append(f'export * as {module_key(name)} from "$/{specifier.as_posix()}";') - exports = find_exports(path.read_text(encoding="utf-8")) - outputs.append((str(path.with_suffix(".pyi")), render_stub(path, exports))) + loaders = "" + for name, path in modules.items(): + target, script = files[path] + # The module itself is a .js or .ts file, so it is a script. + functions = cast("ScriptSource", script).functions + outputs.append((str(path.with_suffix(".pyi")), _render_stub(path, functions))) + specifier = json.dumps(f"$/{target.relative_to(web_dir).as_posix()}") + loaders += f" {json.dumps(name)}: () => import({specifier}),\n" outputs.append(( str(web_dir / (constants.Dirs.SCRIPTS_PATH + constants.Ext.JS)), - "\n".join(index) + "\n" if index else "export {};\n", + f"export default {{\n{loaders}}};\n" if loaders else "export default {};\n", )) return outputs, package_imports, copies diff --git a/tests/units/compiler/state_js.test.mjs b/tests/units/compiler/state_js.test.mjs index 90c1990d017..2ca636a398a 100644 --- a/tests/units/compiler/state_js.test.mjs +++ b/tests/units/compiler/state_js.test.mjs @@ -228,6 +228,7 @@ async function setup({ "$/utils/helpers/throttle": { default() {} }, "$/utils/helpers/upload": { uploadFiles() {} }, "$/utils/reflex": { reflex: {} }, + "$/utils/scripts": { default: {} }, }; const module = new vm.SourceTextModule(source, { context, diff --git a/tests/units/compiler/test_compiler_utils.py b/tests/units/compiler/test_compiler_utils.py index fb045808a01..7d49d6c5386 100644 --- a/tests/units/compiler/test_compiler_utils.py +++ b/tests/units/compiler/test_compiler_utils.py @@ -854,46 +854,46 @@ def test_compile_script_modules( (scripts / "unused.js").write_text('import "left-pad";\n') typed = tmp_path / "app" / "typed.ts" typed.write_text("export function typed(n: number) {}\n") - clean_registration_context.script_modules.update({ + # Only importlib can import a module whose name is not an identifier. + widget = tmp_path / "app" / 'my-"widget".js' + widget.write_text("export function show() {}\n") + clean_registration_context._script_modules.update({ "app.typed": typed, "app.scripts.party": party, + 'app.my-"widget"': widget, }) outputs, package_imports, copies = utils.compile_script_modules() modules_dir = web_dir / "utils" / "script_modules" / "app" + + def stub(name: str, function: str) -> str: + return ( + f"# Generated by Reflex from {name}; rewritten when it changes.\n" + "from typing import Any\n" + "\n" + "from reflex_base.event import EventSpec\n" + "\n" + f"def {function}(*args: Any) -> EventSpec: ...\n" + ) + assert outputs == [ + (str(modules_dir / 'my-"widget".js'), widget.read_text()), (str(modules_dir / "scripts" / "party.js"), party.read_text()), (str(modules_dir / "scripts" / "helpers.js"), helpers.read_text()), (str(modules_dir / "scripts" / "formatters" / "count.ts"), count.read_text()), - ( - str(scripts / "party.pyi"), - ( - "# Generated by Reflex from party.js; rewritten when it changes.\n" - "from typing import Any\n" - "\n" - "from reflex_base.event import EventSpec\n" - "\n" - "def celebrate(*args: Any) -> EventSpec: ...\n" - ), - ), (str(modules_dir / "typed.ts"), typed.read_text()), - ( - str(tmp_path / "app" / "typed.pyi"), - ( - "# Generated by Reflex from typed.ts; rewritten when it changes.\n" - "from typing import Any\n" - "\n" - "from reflex_base.event import EventSpec\n" - "\n" - "def typed(*args: Any) -> EventSpec: ...\n" - ), - ), + (str(tmp_path / "app" / 'my-"widget".pyi'), stub('my-"widget".js', "show")), + (str(scripts / "party.pyi"), stub("party.js", "celebrate")), + (str(tmp_path / "app" / "typed.pyi"), stub("typed.ts", "typed")), ( str(web_dir / "utils" / "scripts.js"), ( - 'export * as app$scripts$party from "$/utils/script_modules/app/scripts/party.js";\n' - 'export * as app$typed from "$/utils/script_modules/app/typed.ts";\n' + "export default {\n" + ' "app.my-\\"widget\\"": () => import("$/utils/script_modules/app/my-\\"widget\\".js"),\n' + ' "app.scripts.party": () => import("$/utils/script_modules/app/scripts/party.js"),\n' + ' "app.typed": () => import("$/utils/script_modules/app/typed.ts"),\n' + "};\n" ), ), ] @@ -918,7 +918,7 @@ def test_compile_script_modules_rejects_files_outside_the_package( monkeypatch: pytest.MonkeyPatch, clean_registration_context: RegistrationContext, ): - """A module may not import a file outside its top-level package's directory. + """A module may not import a file outside the app package's directory. Args: tmp_path: pytest tmp_path fixture. @@ -926,19 +926,22 @@ def test_compile_script_modules_rejects_files_outside_the_package( clean_registration_context: a fresh registry of imported modules. """ monkeypatch.setattr(utils, "get_web_dir", lambda: tmp_path / ".web") - root = tmp_path / "project" - (root / "app").mkdir(parents=True) - outside = tmp_path / "shared.js" - outside.write_text("export const x = 1;\n") - party = root / "app" / "party.js" - party.write_text('import { x } from "../../shared.js";\n') - clean_registration_context.script_modules["app.party"] = party + package = tmp_path / "project" / "app" + package.mkdir(parents=True) + # In the project, beside the package, but not in it. + outside = tmp_path / "project" / ".env" + outside.write_text("SECRET=1\n") + party = package / "party.js" + party.write_text('import secrets from "../.env?raw";\n') + clean_registration_context._script_modules["app.party"] = party with pytest.raises(ReflexRuntimeError) as error: utils.compile_script_modules() assert str(error.value) == ( f"The JavaScript module app.party imports {outside.resolve()}, which is " - f"outside {root.resolve()}, the directory of its package." + f"outside {package.resolve()}, the directory of the app's package, and " + "only that directory is copied into the compiled app. Move the file " + "into it, or publish it as an npm package and import it by name." ) @@ -947,7 +950,7 @@ def test_compile_script_modules_without_modules( monkeypatch: pytest.MonkeyPatch, clean_registration_context: RegistrationContext, ): - """An app that imports no JavaScript module gets an empty index. + """An app that imports no JavaScript module gets an empty map, which loads nothing. Args: tmp_path: pytest tmp_path fixture. @@ -956,7 +959,7 @@ def test_compile_script_modules_without_modules( """ monkeypatch.setattr(utils, "get_web_dir", lambda: tmp_path) assert utils.compile_script_modules() == ( - [(str(tmp_path / "utils" / "scripts.js"), "export {};\n")], + [(str(tmp_path / "utils" / "scripts.js"), "export default {};\n")], {}, [], ) diff --git a/tests/units/reflex_base/client_event_queue_runtime.mjs b/tests/units/reflex_base/client_event_queue_runtime.mjs index b0080357475..00a60ddfea6 100644 --- a/tests/units/reflex_base/client_event_queue_runtime.mjs +++ b/tests/units/reflex_base/client_event_queue_runtime.mjs @@ -63,6 +63,7 @@ export async function createQueueRuntime( uploadFiles: uploadFiles ?? unused, }, "$/utils/reflex": { reflex: {} }, + "$/utils/scripts": { default: {} }, ...dependencyOverrides, }; // Let Node parse the unchanged module; only expose private state to tests. diff --git a/tests/units/reflex_base/js/conftest.py b/tests/units/reflex_base/js/conftest.py new file mode 100644 index 00000000000..fa08a703bb1 --- /dev/null +++ b/tests/units/reflex_base/js/conftest.py @@ -0,0 +1,55 @@ +"""Fixtures for importing JavaScript modules from Python.""" + +import sys +from collections.abc import Generator +from pathlib import Path + +import pytest +from reflex_base.registry import RegistrationContext + +import reflex as rx + +# The source of js_app/party.js. +PARTY_SOURCE = """ +import confetti from "canvas-confetti"; +export function celebrate(count) {} +export async function load() {} +export const arrow = () => 1; +export const notAFunction = 5; +export default function () {} +""" + + +@pytest.fixture +def app_package( + tmp_path: Path, clean_registration_context: RegistrationContext +) -> Generator[Path, None, None]: + """Put an app package and another package, each with JavaScript modules, on sys.path. + + The app's config names ``js_app``; ``js_vendor`` stands for a third-party + package. + + Args: + tmp_path: pytest tmp_path fixture. + clean_registration_context: a fresh registry for the app's config and + the imported modules. + + Yields: + The app package's directory. + """ + clean_registration_context._set_config(rx.Config(app_name="js_app")) + app = tmp_path / "js_app" + vendor = tmp_path / "js_vendor" + for package in (app, vendor): + package.mkdir() + (package / "__init__.py").write_text("") + (app / "party.js").write_text(PARTY_SOURCE) + (app / "typed.ts").write_text("export function typed(n: number) {}\n") + (vendor / "widget.js").write_text("export function widget() {}\n") + sys.path.insert(0, str(tmp_path)) + yield app + sys.path.remove(str(tmp_path)) + for name in [ + n for n in sys.modules if n.partition(".")[0] in ("js_app", "js_vendor") + ]: + del sys.modules[name] diff --git a/tests/units/reflex_base/js/test_finder.py b/tests/units/reflex_base/js/test_finder.py new file mode 100644 index 00000000000..46b8ea37159 --- /dev/null +++ b/tests/units/reflex_base/js/test_finder.py @@ -0,0 +1,94 @@ +"""Tests for finding the JavaScript modules an app imports from Python.""" + +import importlib +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest +from reflex_base.js.modules import _JsFunction +from reflex_base.registry import RegistrationContext + + +def test_importing_a_module_exposes_its_functions( + app_package: Path, clean_registration_context: RegistrationContext +): + """A JavaScript or TypeScript module of the app imports, and is registered. + + Args: + app_package: the app package, on sys.path. + clean_registration_context: the registry the import fills. + """ + party = importlib.import_module("js_app.party") + assert party.__all__ == ["celebrate", "load", "arrow"] + assert [party.celebrate, party.load, party.arrow] == [ + _JsFunction("js_app.party", "celebrate"), + _JsFunction("js_app.party", "load"), + _JsFunction("js_app.party", "arrow"), + ] + assert party.__spec__ is not None + assert party.__spec__.origin == str(app_package / "party.js") + typed = importlib.import_module("js_app.typed") + assert typed.typed == _JsFunction("js_app.typed", "typed") + assert clean_registration_context._script_modules == { + "js_app.party": app_package / "party.js", + "js_app.typed": app_package / "typed.ts", + } + + +def test_only_the_app_package_is_searched( + app_package: Path, clean_registration_context: RegistrationContext +): + """Another package's JavaScript files are never imported or registered. + + Args: + app_package: the app package, on sys.path. + clean_registration_context: the registry an import would fill. + """ + assert importlib.util.find_spec("js_vendor.widget") is None + with pytest.raises(ModuleNotFoundError) as error: + importlib.import_module("js_vendor.widget") + assert str(error.value) == "No module named 'js_vendor.widget'" + assert clean_registration_context._script_modules == {} + + +def test_a_missing_module_still_fails_to_import(app_package: Path): + """A name with neither a Python nor a JavaScript module still fails. + + Args: + app_package: the app package, on sys.path. + """ + with pytest.raises(ModuleNotFoundError) as error: + importlib.import_module("js_app.missing") + assert str(error.value) == "No module named 'js_app.missing'" + + +def test_a_python_module_wins_over_a_script_of_the_same_name(app_package: Path): + """The finder runs last, so a Python module shadows a script of its name. + + Args: + app_package: the app package, on sys.path. + """ + (app_package / "both.py").write_text("VALUE = 'python'\n") + (app_package / "both.js").write_text("export function VALUE() {}\n") + assert importlib.import_module("js_app.both").VALUE == "python" + + +def test_the_finder_imports_nothing_heavy(): + """The finder loads on the first import no other finder found, so it stays cheap.""" + loaded = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys, reflex_base.js.finder; " + "print(sorted({'pathlib', 'typing', 'importlib.abc', " + "'reflex_base.config', 'reflex_base.event'} & set(sys.modules)))" + ), + ], + capture_output=True, + text=True, + check=True, + ).stdout + assert loaded == "[]\n" diff --git a/tests/units/reflex_base/js/test_modules.py b/tests/units/reflex_base/js/test_modules.py index d0633ae5476..f706a043f99 100644 --- a/tests/units/reflex_base/js/test_modules.py +++ b/tests/units/reflex_base/js/test_modules.py @@ -1,69 +1,62 @@ -"""Tests for importing JavaScript modules from Python.""" +"""Tests for the functions of JavaScript modules imported from Python.""" import importlib -import subprocess -import sys -from collections.abc import Generator from pathlib import Path import pytest -from reflex_base.event import EventSpec -from reflex_base.js.modules import ( - JsFunction, - find_package_imports, - module_key, - render_stub, -) -from reflex_base.js.source import find_exports -from reflex_base.registry import RegistrationContext +from reflex_base.js.modules import _JsFunction, _package_imports, _render_stub +from reflex_base.utils.exceptions import ReflexRuntimeError from reflex_base.vars.base import LiteralVar, Var import reflex as rx -SOURCE = """ -import confetti from "canvas-confetti"; -import { describe } from "./helpers.js"; -import { reflex } from "$/utils/reflex"; -import "@scope/pkg/style.css"; -// export function commentedOut() {} -/* export const alsoCommentedOut = 1; */ -export function celebrate(count) {} -export async function load() {} -export function* steps() {} -export const arrow = () => 1; -function hidden() {} -export { hidden, describe as describeIt }; -export { other as reexported } from "lodash/fp"; -export default function () {} -const later = await import("dayjs"); -""" - - -def test_find_package_imports(): + +def test_package_imports(): """Only specifiers naming an npm package are installed, without a query.""" - assert find_package_imports(SOURCE) == [ - "canvas-confetti", - "@scope/pkg/style.css", - "lodash/fp", - "dayjs", - ] - assert find_package_imports( - 'import a from "pkg/a.css?inline";\nimport b from "pkg/a.css";\n' - ) == ["pkg/a.css"] + assert _package_imports( + Path("party.js"), + [ + "canvas-confetti", + "./helpers.js", + "/abs.js", + "$/utils/reflex", + "#internal", + "@/components/button", + "virtual:pwa-register", + "https://esm.sh/lodash", + "@scope/pkg/style.css", + "pkg/a.css?inline", + "pkg/a.css", + "path", + ], + ) == ["canvas-confetti", "@scope/pkg/style.css", "pkg/a.css", "path"] -def test_module_key_cannot_collide(): - """Module keys are JavaScript identifiers no two module names share.""" - assert module_key("app.scripts.party") == "app$scripts$party" - assert module_key("app.scripts__party") != module_key("app__scripts.party") +@pytest.mark.parametrize("specifier", ["fs", "fs/promises", "node:path"]) +def test_package_imports_rejects_node_builtins(specifier: str): + """A Node.js built-in module cannot run in the browser. + + Args: + specifier: the built-in module's specifier. + """ + with pytest.raises(ReflexRuntimeError) as error: + _package_imports(Path("/app/party.js"), ["canvas-confetti", specifier]) + assert str(error.value) == ( + f"The JavaScript file /app/party.js imports {specifier!r}, a Node.js " + "built-in module, which does not exist in the browser. Remove the " + "import, and do the work that needs it in a backend event handler " + "instead." + ) def test_calling_a_function_builds_the_call_js_event(): - """Calling an export builds the event that names it, with JSON arguments.""" - spec = JsFunction("app.scripts.party", "celebrate")(200, "results") - assert isinstance(spec, EventSpec) + """Calling a function builds the event that names it, with JSON arguments. + + The module is named as Python names it, whatever characters that has. + """ + spec = _JsFunction("app.my-widget", "celebrate")(200, "results") assert str(LiteralVar.create(spec)) == ( - '(ReflexEvent("_call_js", ({ ["module"] : "app$scripts$party", ' + '(ReflexEvent("_call_js", ({ ["module"] : "app.my-widget", ' '["function"] : "celebrate", ["args"] : [200, "results"] }), ({ })))' ) @@ -80,10 +73,10 @@ def done(self, value: str): value: The value. """ - spec = JsFunction("app.scripts.party", "report")(JsHandlerState.done, "x") + spec = _JsFunction("app.scripts.party", "report")(JsHandlerState.done, "x") name = f"{JsHandlerState.get_full_name()}.done" assert str(LiteralVar.create(spec)) == ( - '(ReflexEvent("_call_js", ({ ["module"] : "app$scripts$party", ' + '(ReflexEvent("_call_js", ({ ["module"] : "app.scripts.party", ' f'["function"] : "report", ["args"] : ["{name}", "x"] }}), ({{ }})))' ) @@ -91,17 +84,16 @@ def done(self, value: str): def test_var_arguments_are_compiled_into_the_event(): """A Var argument is compiled into the event payload.""" count = Var(_js_expr="count").to(int) - spec = JsFunction("app.scripts.party", "celebrate")(count) - literal = LiteralVar.create(spec) - assert str(literal) == ( - '(ReflexEvent("_call_js", ({ ["module"] : "app$scripts$party", ' + spec = _JsFunction("app.scripts.party", "celebrate")(count) + assert str(LiteralVar.create(spec)) == ( + '(ReflexEvent("_call_js", ({ ["module"] : "app.scripts.party", ' '["function"] : "celebrate", ["args"] : [count] }), ({ })))' ) def test_render_stub_skips_names_python_cannot_import(): - """The stub leaves out exports that are not Python identifiers.""" - assert render_stub(Path("party.js"), ["celebrate", "$private", "class"]) == ( + """The stub leaves out functions whose names are not Python identifiers.""" + assert _render_stub(Path("party.js"), ["celebrate", "$private", "class"]) == ( "# Generated by Reflex from party.js; rewritten when it changes.\n" "from typing import Any\n" "\n" @@ -111,86 +103,18 @@ def test_render_stub_skips_names_python_cannot_import(): ) -@pytest.fixture -def script_package( - tmp_path: Path, clean_registration_context: RegistrationContext -) -> Generator[Path, None, None]: - """Put a package with a JavaScript module on sys.path. - - Args: - tmp_path: pytest tmp_path fixture. - clean_registration_context: a fresh registry for the imported module. - - Yields: - The package directory. - """ - package = tmp_path / "js_pkg" - package.mkdir() - (package / "__init__.py").write_text("") - (package / "party.js").write_text(SOURCE) - (package / "typed.ts").write_text("export function typed(n: number) {}\n") - sys.path.insert(0, str(tmp_path)) - yield package - sys.path.remove(str(tmp_path)) - for name in [n for n in sys.modules if n == "js_pkg" or n.startswith("js_pkg.")]: - del sys.modules[name] - - -def test_importing_a_module_exposes_its_exports( - script_package: Path, clean_registration_context: RegistrationContext -): - """Importing a JavaScript module exposes its exports and registers it. - - Args: - script_package: a package with JavaScript modules on sys.path. - clean_registration_context: the registry the import fills. - """ - party = importlib.import_module("js_pkg.party") - assert party.__all__ == find_exports(SOURCE) - assert party.celebrate == JsFunction("js_pkg.party", "celebrate") - assert party.__spec__ is not None - assert party.__spec__.origin == str(script_package / "party.js") - typed = importlib.import_module("js_pkg.typed") - assert typed.typed == JsFunction("js_pkg.typed", "typed") - assert clean_registration_context.script_modules == { - "js_pkg.party": script_package / "party.js", - "js_pkg.typed": script_package / "typed.ts", - } - - -def test_a_missing_module_still_fails_to_import(script_package: Path): - """A name with neither a Python nor a JavaScript module still fails. - - Args: - script_package: a package with JavaScript modules on sys.path. - """ - with pytest.raises(ModuleNotFoundError): - importlib.import_module("js_pkg.missing") - - -def test_a_python_module_wins_over_a_script_of_the_same_name(script_package: Path): - """The finder runs last, so a Python module shadows a script of its name. +def test_a_function_is_an_event_trigger(app_package: Path): + """A function works in an event trigger, called or uncalled. Args: - script_package: a package with JavaScript modules on sys.path. + app_package: the app package, on sys.path. """ - (script_package / "both.py").write_text("VALUE = 'python'\n") - (script_package / "both.js").write_text("export function VALUE() {}\n") - assert importlib.import_module("js_pkg.both").VALUE == "python" - - -def test_a_function_is_an_event_trigger(script_package: Path): - """An export works in an event trigger, called or uncalled. - - Args: - script_package: a package with JavaScript modules on sys.path. - """ - from js_pkg.party import celebrate # pyright: ignore[reportMissingImports] + from js_app.party import celebrate # pyright: ignore[reportMissingImports] called = rx.button("go", on_click=celebrate(1, "out")) assert str(LiteralVar.create(called.event_triggers["on_click"])) == ( '((_e) => (addEvents([(ReflexEvent("_call_js", ({ ["module"] : ' - '"js_pkg$party", ["function"] : "celebrate", ["args"] : [1, "out"] }), ' + '"js_app.party", ["function"] : "celebrate", ["args"] : [1, "out"] }), ' "({ })))], [_e], ({ }))))" ) # Like an event handler, an uncalled function takes the trigger's arguments: @@ -198,58 +122,70 @@ def test_a_function_is_an_event_trigger(script_package: Path): on_mount = rx.box(on_mount=celebrate) assert str(LiteralVar.create(on_mount.event_triggers["on_mount"])) == ( '((...args) => (addEvents([(ReflexEvent("_call_js", ({ ["module"] : ' - '"js_pkg$party", ["function"] : "celebrate", ["args"] : [] }), ' + '"js_app.party", ["function"] : "celebrate", ["args"] : [] }), ' "({ })))], args, ({ }))))" ) uncalled = rx.input(on_change=celebrate) assert str(LiteralVar.create(uncalled.event_triggers["on_change"])) == ( '((_e) => (addEvents([(ReflexEvent("_call_js", ({ ["module"] : ' - '"js_pkg$party", ["function"] : "celebrate", ' + '"js_app.party", ["function"] : "celebrate", ' '["args"] : [_e?.["target"]?.["value"]] }), ({ })))], [_e], ({ }))))' ) -def test_import_reflex_registers_one_finder_and_loads_nothing(): - """``import reflex`` registers the finder without loading a module for it. +def test_importing_a_missing_function_says_what_the_file_exports(app_package: Path): + """The error names the file, its functions, and the export to add. - Importing it again, as a reload or a benchmark that clears ``sys.modules`` - does, must not register a second one. + Args: + app_package: the app package, on sys.path. """ - output = subprocess.run( - [ - sys.executable, - "-c", - ( - "import importlib, sys\n" - "import reflex\n" - "for name in [n for n in sys.modules if n.startswith('reflex')]:\n" - " del sys.modules[name]\n" - "importlib.import_module('reflex')\n" - "finders = [f for f in sys.meta_path if getattr(f, '__qualname__', '') == '_ScriptFinder']\n" - "print(len(finders), 'reflex_base.js' in sys.modules)" - ), - ], - capture_output=True, - text=True, - check=True, - ).stdout - assert output == "1 False\n" + with pytest.raises(ImportError) as error: + from js_app.party import ( # pyright: ignore[reportMissingImports] + nope, # noqa: F401 + ) + assert str(error.value) == ( + f"Cannot import nope from js_app.party: {app_package / 'party.js'} " + "exports no function named nope (its functions: celebrate, load, " + "arrow). Export it from party.js, as in `export function nope() { ... }`." + ) + (app_package / "empty.js").write_text("export class Widget {}\n") + with pytest.raises(ImportError) as error: + from js_app.empty import ( # pyright: ignore[reportMissingImports] + Widget, # noqa: F401 + ) + assert str(error.value) == ( + f"Cannot import Widget from js_app.empty: {app_package / 'empty.js'} " + "exports no function named Widget (its functions: none). Export it " + "from empty.js, as in `export function Widget() { ... }`." + ) -def test_the_finder_imports_nothing_heavy(): - """The real finder loads on the first import no other finder found, so it stays cheap.""" - loaded = subprocess.run( - [ - sys.executable, - "-c", - ( - "import sys, reflex_base.js.finder; " - "print(sorted({'pathlib', 'typing', 'importlib.abc', 'reflex_base.event'}" - " & set(sys.modules)))" - ), - ], - capture_output=True, - text=True, - check=True, - ).stdout - assert loaded == "[]\n" +def test_importing_a_value_says_only_functions_can_be_called(app_package: Path): + """A ``const`` bound to a literal is a value, which Python cannot call. + + Args: + app_package: the app package, on sys.path. + """ + with pytest.raises(ImportError) as error: + from js_app.party import ( # pyright: ignore[reportMissingImports] + notAFunction, # noqa: F401 + ) + assert str(error.value) == ( + f"Cannot import notAFunction from js_app.party: {app_package / 'party.js'} " + "exports notAFunction as a value, not a function, and Python can only " + "call functions. Export a function instead, as in " + "`export function notAFunction() { ... }`." + ) + + +def test_a_private_name_is_a_plain_attribute_error(app_package: Path): + """Names tools probe a module for raise the AttributeError they expect. + + Args: + app_package: the app package, on sys.path. + """ + party = importlib.import_module("js_app.party") + assert getattr(party, "__wrapped__", None) is None + with pytest.raises(AttributeError) as error: + _ = party._private + assert str(error.value) == "module 'js_app.party' has no attribute '_private'" diff --git a/tests/units/reflex_base/js/test_source.py b/tests/units/reflex_base/js/test_source.py index 08c7fa2c403..36cffb92207 100644 --- a/tests/units/reflex_base/js/test_source.py +++ b/tests/units/reflex_base/js/test_source.py @@ -2,11 +2,12 @@ from pathlib import Path +import pytest from reflex_base.js.source import ( - find_exports, - find_imports, find_stylesheet_references, module_files, + parse_script, + read_script, ) SOURCE = r""" @@ -26,6 +27,8 @@ export function* steps() {} export const first = () => 1, second = (a, b) => [a, b]; export let typed: Record = {}, other = 2 +export const count = 5, label = "party", flags = [1, 2], options = { a: 1 }, none = null; +export const total = count + 1; export const { pickA, pickB: renamed, ...rest } = obj; export const [x0, , x2 = 3] = arr; export const multiline = (a) => @@ -36,6 +39,10 @@ export { other2 as reexported } from "lodash/fp"; export * from "everything"; export * as namespace from "namespaced"; +import type { Party } from "party-types"; +export type { Guest } from "guest-types"; +export type * from "all-types"; +import type, { other3 } from "type-named"; export default function () {} const later = await import("dayjs"); thing.export = 1; @@ -43,9 +50,15 @@ """ -def test_find_exports(): - """Every exported function name is found, and nothing inside text or comments.""" - assert find_exports(SOURCE) == [ +def test_parse_script(): + """Every export and import is found, and nothing inside text or comments. + + A ``const`` bound to a literal is a value; anything else may be a function. + Type-only imports load nothing. + """ + script = parse_script(SOURCE) + assert script.text == SOURCE + assert script.functions == [ "celebrate", "load", "steps", @@ -53,6 +66,7 @@ def test_find_exports(): "second", "typed", "other", + "total", "pickA", "renamed", "rest", @@ -64,25 +78,123 @@ def test_find_exports(): "quoted", "reexported", ] - - -def test_find_imports(): - """Static, side-effect, dynamic and re-export specifiers, across line breaks.""" - assert find_imports(SOURCE) == [ + assert script.values == ["count", "label", "flags", "options", "none"] + assert script.imports == [ "canvas-confetti", "./helpers.js", "@scope/pkg/style.css", "lodash/fp", "everything", "namespaced", + "type-named", "dayjs", ] -def test_a_string_with_a_comment_opener_hides_nothing(): - """A comment opener inside a string does not swallow the code after it.""" - source = 'const route = "/api/*";\nexport function after() {}\n/* note */\n' - assert find_exports(source) == ["after"] +@pytest.mark.parametrize( + ("source", "functions", "values"), + [ + # A comment opener inside a string hides nothing. + ( + 'const route = "/api/*";\nexport function after() {}\n/* note */\n', + ["after"], + [], + ), + # A slash after `++` or `--` divides; it does not start a regex. + ("let i = 0; const y = i++ / 2; export function after() {}", ["after"], []), + ("let i = 9; const y = i-- / 2; export function after() {}", ["after"], []), + # TypeScript: a type item of an export list exports no value. + ("export { type Foo, bar, type, type as t };", ["bar", "type", "t"], []), + # `=>` and type arguments in an annotation. + ( + "export const f: () => Map> = () => new Map();", + ["f"], + [], + ), + # Type parameters of a generic arrow function; a comparison. + ( + "export const id = (x: T) => x, two = 2, less = 1 < 2, more = 3;", + ["id", "less"], + ["two", "more"], + ), + # A statement ends at a line break that nothing continues. + ("export const a = 1\nexport const b = () =>\n 2\n", ["b"], ["a"]), + ("export let mutable = 1;", ["mutable"], []), + ("export function () {}\nexport function* () {}", [], []), + # Escaped and unterminated strings and regular expressions. + ("const s = 'it\\'s'; export function after() {}", ["after"], []), + ("export function a() {}\nconst s = 'open", ["a"], []), + ("export function a() {}\nconst r = /open", ["a"], []), + # Nested patterns, declarators without an initializer, and no names. + ("export const { a: { b }, c: [d, [e]] } = obj;", ["b", "d", "e"], []), + ("export let later, typedLater: number;", ["later", "typedLater"], []), + ("export let;", [], []), + ("export const a = 1,", [], ["a"]), + ], +) +def test_parse_script_exports(source: str, functions: list[str], values: list[str]): + """Exports are read past regexes, divisions and TypeScript syntax. + + Args: + source: The module source. + functions: The exports that may be functions. + values: The exports known to be values. + """ + script = parse_script(source) + assert (script.functions, script.values) == (functions, values) + + +def test_parse_script_skips_jsx(): + """In JSX, text and attributes are not code, but expressions in braces are.""" + source = ( + "const el =
go(1 > 0)}>it's {`a${'}'}`}" + "
<>/* not a comment
; export function same() {}\n" + "const id = (x: T) => x;\n" + "export const element =

it's

;\n" + ) + script = parse_script(source, jsx=True) + assert (script.functions, script.values) == (["same"], ["element"]) + + +@pytest.mark.parametrize( + ("source", "jsx", "functions"), + [ + ("const t = `${\nexport function a() {}", False, []), + ("const t = `abc\nexport function a() {}", False, []), + ("/* open\nexport function a() {}", False, []), + ("export { a", False, []), + # A regular expression ends at the end of its line. + ("const r = /abc\nexport function a() {}", False, ["a"]), + ("const e =
{\nexport function a() {}", True, []), + ("const e =
text\nexport function a() {}", True, []), + ("const e =
Date: Sat, 21 Nov 2026 21:54:50 +0000 Subject: [PATCH 09/16] Test that a script sees what a module function did first A handler yields a module function and then a script that reads what it wrote; each arrives in a message of its own. With the module loaded on demand, the script ran first and read the old text. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- .../tests_playwright/test_call_script.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/integration/tests_playwright/test_call_script.py b/tests/integration/tests_playwright/test_call_script.py index 3df86508454..c8bf5d3730f 100644 --- a/tests/integration/tests_playwright/test_call_script.py +++ b/tests/integration/tests_playwright/test_call_script.py @@ -225,6 +225,16 @@ def set_mounted_library(self, value: str): def js_module_backend(self): return page_script.setText("js_module_text", "from the backend") + @rx.event + def js_module_then_script(self): + # Each yield is a message of its own, and the script must see what + # the module's function did. + yield page_script.setText("js_module_text", "written by the module") + yield rx.call_script( + "document.getElementById('js_module_text').textContent", + callback=CallScriptState.set_reflex_result, + ) + @rx.event def add_events_from_backend(self): return rx.run_script( @@ -466,6 +476,11 @@ def index(): on_click=CallScriptState.js_module_backend, id="js_module_backend", ), + rx.button( + "js_module_then_script", + on_click=CallScriptState.js_module_then_script, + id="js_module_then_script", + ), rx.button( "js_module_report", on_click=page_script.report( @@ -689,3 +704,8 @@ def test_js_module(call_script_page: Page): call_script_page.locator("#js_module_report").click() expect(call_script_page.locator("#reflex_result")).to_have_value("from a module") + + call_script_page.locator("#js_module_then_script").click() + expect(call_script_page.locator("#reflex_result")).to_have_value( + "written by the module" + ) From f9e1852a6b50d5700e60a826bfe01edae7d35ef4 Mon Sep 17 00:00:00 2001 From: Farhan Date: Sat, 21 Nov 2026 21:54:51 +0000 Subject: [PATCH 10/16] Document JavaScript modules usage-first, with every error The section leads with a runnable app, then the rules (the app package is the scope, files must be inside it, what is installed, arguments as JSON, how to get a value back, JSX only in .jsx/.tsx) and each error with its fix. News fragments say the same for downstream users. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- docs/api-reference/browser_javascript.md | 97 ++++++++++++++----- news/+js-modules.feature.md | 2 +- .../reflex-base/news/+js-modules.feature.md | 2 +- .../news/+varargs-event-callable.bugfix.md | 2 +- 4 files changed, 77 insertions(+), 26 deletions(-) diff --git a/docs/api-reference/browser_javascript.md b/docs/api-reference/browser_javascript.md index 8ee28c53b93..6d5a5d28dce 100644 --- a/docs/api-reference/browser_javascript.md +++ b/docs/api-reference/browser_javascript.md @@ -276,12 +276,9 @@ event handlers that take no arguments, such as `SearchState.clear`. ## JavaScript Modules -For more than a line of Javascript, write it in a `.js` (or `.ts`) file in your -app package and import its exported functions in Python. You get your editor's -Javascript support in the file, and nothing is evaluated: the module is bundled -with the app, and calling one of its functions sends only the function's name -and its arguments as JSON. Apps whose Content Security Policy forbids -`unsafe-eval` can use them. +Write JavaScript in a `.js` or `.ts` file in your app's package and import its +exported functions in Python, like a Python module. Each function returns an +event: use it in an event trigger, or return or yield it from an event handler. ```javascript // my_app/scripts/party.js @@ -293,47 +290,101 @@ export function celebrate(count, elementId) { document.getElementById(elementId).scrollIntoView(); } -export function report(handler, value) { - reflex.addEvents([reflex.event(handler, { value })]); +export function measure(id, handler) { + const { width } = document.getElementById(id).getBoundingClientRect(); + reflex.addEvents([reflex.event(handler, { width })]); } ``` ```python -from my_app.scripts.party import celebrate, report +# my_app/my_app.py +import reflex as rx + +from my_app.scripts.party import celebrate, measure class PartyState(rx.State): count: int = 200 + width: float = 0 @rx.event def finish(self): return celebrate(self.count, "results") @rx.event - def done(self, value: str): ... + def set_width(self, width: float): + self.width = width + +def index(): + return rx.vstack( + rx.button("Celebrate", on_click=celebrate(PartyState.count, "results")), + rx.button("Finish", on_click=PartyState.finish), + rx.button("Measure", on_click=measure("results", PartyState.set_width)), + rx.text(f"Width: {PartyState.width}", id="results"), + ) -rx.button("🎉", on_click=celebrate(PartyState.count, "results")) -rx.button("Report", on_click=report(PartyState.done, "clicked")) + +app = rx.App() +app.add_page(index) ``` -- Each named export becomes a function returning an event. Use it in an event - trigger, or return or yield it from an event handler. Passed to a trigger - uncalled, like `rx.input(on_change=set_title)`, it receives the trigger's - arguments. -- Arguments are sent as JSON. An event handler arrives as its full name, which - the function can dispatch with `reflex.addEvents([reflex.event(name, payload)])`. -- The function's return value is not sent back to the backend; report results - through `reflex.addEvents`. -- Packages the module imports, such as `canvas-confetti`, are installed with - the app. Pin a version with `frontend_packages` in `rxconfig.py`. +Nothing is evaluated: the module is bundled with the app, and calling one of its +functions sends only the module's name, the function's name and the arguments. +Apps whose Content Security Policy forbids `unsafe-eval` can use them. + +### Rules + +- Only modules in your app's package import this way: the package that + `app_name` in `rxconfig.py` names. A `.js` file in another package, such as + a library you installed, is never imported. +- A Python module of the same name wins over a JavaScript one. +- Every named export that may be a function becomes a function. A `const` + initialized with a literal, as in `export const limit = 5`, is a value, and + Python cannot call it. A default export, a class and TypeScript types are + left out. +- Arguments are sent as JSON. A var is sent as its value in the browser. An + event handler arrives as its full name, which the function dispatches with + `reflex.addEvents([reflex.event(name, payload)])`, as `measure` does above. +- The function's return value is not sent back. To get a value back, pass an + event handler and dispatch it with the value, as `measure` does. +- The npm packages the module imports, such as `canvas-confetti`, are installed + with the app. Pin a version with `frontend_packages` in `rxconfig.py`. Path + aliases (`@/...`), subpath imports (`#...`) and specifiers with a scheme + (`virtual:...`, `https://...`) are left to the bundler and not installed. - Local files the module imports with relative paths, such as `./helpers.js`, `./party.css` or `./logo.png`, are bundled with it, and so are the files they import, including a stylesheet's `@import` and `url(...)` references. They - must be inside the directory of your app's package. + must be inside your app's package directory, which is all of your project + that is copied into the compiled app. +- In a `.jsx` or `.tsx` file, JSX is read as JSX. In a `.js` or `.ts` file, + `<` is never JSX, so a quote in what would be JSX text hides the code after + it on that line. - Reflex writes a type stub (`party.pyi`) next to the module when it compiles the app, so type checkers resolve the import. +### Errors + +- `Cannot import nope from my_app.scripts.party: .../party.js exports no + function named nope (its functions: celebrate, measure).` Export the function + from the file, as in `export function nope() { ... }`, or fix the name you + import. +- `Cannot import limit from my_app.scripts.party: .../party.js exports limit as + a value, not a function, and Python can only call functions.` Export a + function that uses the value instead. +- `The JavaScript module my_app.scripts.party imports .../shared.js, which is + outside .../my_app, the directory of the app's package, and only that + directory is copied into the compiled app.` Move the file into your app's + package, or publish it as an npm package and import it by name. +- `The JavaScript file .../party.js imports 'fs', a Node.js built-in module, + which does not exist in the browser.` Remove the import, and do the work that + needs it in a backend event handler instead. +- In the browser, `Cannot call my_app.scripts.party.nope: the frontend has no + function nope in the JavaScript module my_app.scripts.party.` The module does + not export a function of that name, or the frontend was built before the + backend imported it. Export the function, or restart `reflex run` to rebuild + the frontend. + ## Using React Hooks To use React Hooks directly in a Reflex app, you must subclass `rx.Component`, diff --git a/news/+js-modules.feature.md b/news/+js-modules.feature.md index 4552c6ecd9b..230c84849df 100644 --- a/news/+js-modules.feature.md +++ b/news/+js-modules.feature.md @@ -1,4 +1,4 @@ -Import functions from a `.js` or `.ts` file in your app package like a Python module, and call them from event triggers and event handlers. The module is bundled with the app and its npm imports are installed; see [JavaScript Modules](https://reflex.dev/docs/api-reference/browser-javascript/#javascript-modules). +Import functions from a `.js` or `.ts` file in your app's package like a Python module, and call them from event triggers and event handlers. The module is bundled with the app and its npm imports are installed; see [JavaScript Modules](https://reflex.dev/docs/api-reference/browser-javascript/#javascript-modules). ```python from my_app.scripts.party import celebrate diff --git a/packages/reflex-base/news/+js-modules.feature.md b/packages/reflex-base/news/+js-modules.feature.md index cc1bb78f888..c30e9038f17 100644 --- a/packages/reflex-base/news/+js-modules.feature.md +++ b/packages/reflex-base/news/+js-modules.feature.md @@ -1 +1 @@ -Call exported functions of the app's own JavaScript modules from Python without evaluating any code: the `_call_js` event runs a function bundled from `utils/scripts.js` by name, with JSON arguments, so it works under a Content Security Policy without `unsafe-eval`. +The `_call_js` event calls a function of one of the app's JavaScript modules by name, with JSON arguments, so the frontend evaluates no code. `utils/scripts.js` maps each module's Python name to a function that loads it, and the page loads them as it starts. diff --git a/packages/reflex-base/news/+varargs-event-callable.bugfix.md b/packages/reflex-base/news/+varargs-event-callable.bugfix.md index 7b332c8f355..70565a37e42 100644 --- a/packages/reflex-base/news/+varargs-event-callable.bugfix.md +++ b/packages/reflex-base/news/+varargs-event-callable.bugfix.md @@ -1 +1 @@ -A callable taking `*args` passed to an event trigger, such as `on_mount=lambda *args: ...`, now receives all of the trigger's arguments, and works with triggers that pass none, instead of failing to compile or getting only the first argument. +A callable passed to an event trigger gets the trigger's arguments only through its positional parameters: `lambda *args: ...` gets all of them, even none, and `lambda key, **kwargs: ...` or a keyword-only parameter no longer raises `TypeError`. From 19483fb1289950298d843ced44923917b30f1f10 Mon Sep 17 00:00:00 2001 From: Farhan Date: Sat, 21 Nov 2026 21:55:56 +0000 Subject: [PATCH 11/16] Test errors of event code; drop unreachable import-clause checks The helper that reports errors of event code now serves four blocks, so a test pins what each logs and reports, with and without window.onerror. A string after `from` always ends an import or export clause in valid code, so finding the clause's keyword needs no bail-out branches. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- .../reflex-base/src/reflex_base/js/source.py | 18 ++++------- .../reflex-frontend/tests/run_script.test.js | 31 +++++++++++++++++++ tests/units/reflex_base/js/test_source.py | 1 + 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/js/source.py b/packages/reflex-base/src/reflex_base/js/source.py index 3fe148bda92..2ec8deec32d 100644 --- a/packages/reflex-base/src/reflex_base/js/source.py +++ b/packages/reflex-base/src/reflex_base/js/source.py @@ -651,7 +651,7 @@ def _export_names(tokens: list[Token], index: int) -> list[tuple[str, bool]]: return [] -def _statement_start(tokens: list[Token], index: int) -> int | None: +def _statement_start(tokens: list[Token], index: int) -> int: """Find the ``import`` or ``export`` keyword an import clause follows. Args: @@ -659,17 +659,13 @@ def _statement_start(tokens: list[Token], index: int) -> int | None: index: The index of the clause's last token, before ``from``. Returns: - The index of the keyword, or None if the tokens are not an import - clause. + The index of the keyword. """ - while index >= 0: - token = tokens[index] - if token.kind == "name" and token.value in ("import", "export"): - return index - if token.kind == "punct" and token.value not in ("{", "}", ",", "*"): - return None + while index > 0 and not ( + tokens[index].kind == "name" and tokens[index].value in ("import", "export") + ): index -= 1 - return None + return index def _is_runtime_import(tokens: list[Token], index: int) -> bool: @@ -693,8 +689,6 @@ def _is_runtime_import(tokens: list[Token], index: int) -> bool: if not _is(_at(tokens, index - 1), "name", "from"): return False start = _statement_start(tokens, index - 2) - if start is None: - return False # `import type X from` loads nothing, but `import type from` and # `import type, { x } from` import a default export named type. after = _at(tokens, start + 2) diff --git a/packages/reflex-frontend/tests/run_script.test.js b/packages/reflex-frontend/tests/run_script.test.js index cfcc0f29cbf..93626a84341 100644 --- a/packages/reflex-frontend/tests/run_script.test.js +++ b/packages/reflex-frontend/tests/run_script.test.js @@ -133,3 +133,34 @@ describe("_call_script", () => { expect(seen).toEqual([7]); }); }); + +test("event code that throws is logged, and reported to window.onerror", async () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const onerror = vi.fn(); + const error = new Error("The callback failed."); + const fail = () => { + throw error; + }; + const callFunction = { name: "_call_function", payload: { function: fail } }; + window.onerror = onerror; + await applyEvent(callFunction, null, null, null); + await applyEvent( + { name: "_client_state_get", payload: { var_name: "x", callback: fail } }, + null, + null, + null, + ); + window.onerror = null; + await applyEvent(callFunction, null, null, null); + const logged = [...log.mock.calls]; + log.mockRestore(); + expect(logged).toEqual([ + ["_call_function", error], + ["_client_state_get", error], + ["_call_function", error], + ]); + expect(onerror.mock.calls).toEqual([ + ["The callback failed.", null, null, null, error], + ["The callback failed.", null, null, null, error], + ]); +}); diff --git a/tests/units/reflex_base/js/test_source.py b/tests/units/reflex_base/js/test_source.py index 36cffb92207..c4727ebf4dc 100644 --- a/tests/units/reflex_base/js/test_source.py +++ b/tests/units/reflex_base/js/test_source.py @@ -129,6 +129,7 @@ def test_parse_script(): ("export const { a: { b }, c: [d, [e]] } = obj;", ["b", "d", "e"], []), ("export let later, typedLater: number;", ["later", "typedLater"], []), ("export let;", [], []), + ("export", [], []), ("export const a = 1,", [], ["a"]), ], ) From a15fd4062f2e7963b87a186a060d36bbadc32b1c Mon Sep 17 00:00:00 2001 From: Farhan Date: Sat, 21 Nov 2026 21:57:32 +0000 Subject: [PATCH 12/16] Say why the loader imports the event system lazily Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- packages/reflex-base/src/reflex_base/js/finder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/js/finder.py b/packages/reflex-base/src/reflex_base/js/finder.py index e7e03465d3e..d87a9a0e7e6 100644 --- a/packages/reflex-base/src/reflex_base/js/finder.py +++ b/packages/reflex-base/src/reflex_base/js/finder.py @@ -54,8 +54,8 @@ def exec_module(self, module: ModuleType) -> None: Args: module: The module to populate. """ - # Imported here: the finder loads with `import reflex`, which must not - # pay for the event system until a JavaScript module is imported. + # Imported here: loading the finder must not load the event system, + # which only a JavaScript module being imported needs. from pathlib import Path from reflex_base.js.modules import _load_module From 660a450d33c4bd50af59baab4545a04045b76f94 Mon Sep 17 00:00:00 2001 From: Farhan Date: Sat, 21 Nov 2026 21:58:00 +0000 Subject: [PATCH 13/16] Format the _call_js tests Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- packages/reflex-frontend/tests/call_js.test.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/reflex-frontend/tests/call_js.test.js b/packages/reflex-frontend/tests/call_js.test.js index 6038b9bf731..7cbb74ab29b 100644 --- a/packages/reflex-frontend/tests/call_js.test.js +++ b/packages/reflex-frontend/tests/call_js.test.js @@ -130,7 +130,13 @@ test("a function that throws, or a module that fails to load, is reported", asyn await callJs("app.scripts.party", "fails", []); await callJs("app.scripts.broken", "anything", []); expect(onerror.mock.calls).toEqual([ - ["The party is over.", null, null, null, new RangeError("The party is over.")], + [ + "The party is over.", + null, + null, + null, + new RangeError("The party is over."), + ], [ "Failed to fetch the module.", null, From d854e12a592908e6313abc912253b2ff00551444 Mon Sep 17 00:00:00 2001 From: Farhan Date: Sat, 21 Nov 2026 22:09:00 +0000 Subject: [PATCH 14/16] Hold events back while a module call waits for its module Loading the modules at page load does not guarantee they finish before events arrive. While a _call_js waits for its module, processEvent now holds the events after it, in the order they arrived, until the call has run, so they see what its function did. Nothing waits once the module has loaded. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- packages/reflex-frontend/src/utils/state.js | 21 +++++++++- .../reflex-frontend/tests/call_js.test.js | 41 ++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/packages/reflex-frontend/src/utils/state.js b/packages/reflex-frontend/src/utils/state.js index 5e500a6483b..654d23fcc9a 100644 --- a/packages/reflex-frontend/src/utils/state.js +++ b/packages/reflex-frontend/src/utils/state.js @@ -598,6 +598,12 @@ if (typeof window !== "undefined") { } } +// The call of a `_call_js` whose module is still loading, until it has run. +// `processEvent` starts an event that arrives while another awaits, so the +// events after this call wait for it, in the order they arrived, to see what +// its function did. +let pendingModuleCall; + /** * Call an exported function of a JavaScript module the app imports from * Python. The modules are bundled, so nothing is evaluated: the event names the @@ -615,7 +621,15 @@ const callModuleFunction = (payload) => { !Object.hasOwn(loadedScriptModules, module) && Object.hasOwn(scriptModules, module) ) { - return loadScriptModule(module).then(() => callModuleFunction(payload)); + const call = loadScriptModule(module).then(() => + callModuleFunction(payload), + ); + pendingModuleCall = call; + const clear = () => { + pendingModuleCall = undefined; + }; + call.then(clear, clear); + return call; } const fn = Object.hasOwn(loadedScriptModules, module) ? loadedScriptModules[module][name] @@ -1079,6 +1093,11 @@ export const processEvent = async (socket, navigate, params) => { // Apply the next event in the queue. const event = event_queue.shift(); + if (pendingModuleCall !== undefined) { + // The call reports its own failure. + await pendingModuleCall.catch(() => {}); + } + try { // Process events with handlers via REST and all others via websockets. if (event.handler) { diff --git a/packages/reflex-frontend/tests/call_js.test.js b/packages/reflex-frontend/tests/call_js.test.js index 7cbb74ab29b..d19a147fe9a 100644 --- a/packages/reflex-frontend/tests/call_js.test.js +++ b/packages/reflex-frontend/tests/call_js.test.js @@ -13,7 +13,11 @@ const modules = vi.hoisted(() => { const slowLoaded = new Promise((resolve) => { releaseSlow = resolve; }); - return { seen: [], releaseSlow, slowLoaded }; + let releaseGated = () => {}; + const gatedLoaded = new Promise((resolve) => { + releaseGated = resolve; + }); + return { seen: [], releaseSlow, slowLoaded, releaseGated, gatedLoaded }; }); vi.mock("$/utils/scripts", () => ({ @@ -36,6 +40,12 @@ vi.mock("$/utils/scripts", () => ({ modules.slowLoaded.then(() => ({ ping: (value) => modules.seen.push(["ping", value]), })), + "app.scripts.gated": () => + modules.gatedLoaded.then(() => ({ + write: (value) => { + globalThis.gatedValue = value; + }, + })), "app.scripts.broken": () => Promise.reject(new Error("Failed to fetch the module.")), }, @@ -97,6 +107,35 @@ test("the next event sees what a function did, though events overlap", async () expect(globalThis.reads).toEqual(["new"]); }); +test("an event queued while the modules load runs after the function before it", async () => { + // Until the modules the page loads have settled, events wait for them in + // the order they arrived, so the script still sees what the function did. + globalThis.reads = []; + const socket = { connected: true, emit: vi.fn() }; + const first = queueEvents( + [callJsEvent("app.scripts.gated", "write", ["gated"])], + socket, + false, + null, + null, + ); + const second = queueEvents( + [ + { + name: "_call_script", + payload: { javascript_code: "reads.push(globalThis.gatedValue)" }, + }, + ], + socket, + false, + null, + null, + ); + modules.releaseGated(); + await Promise.all([first, second]); + expect(globalThis.reads).toEqual(["gated"]); +}); + test("a function of a module still loading runs once it has loaded", async () => { const call = callJs("app.scripts.slow", "ping", ["late"]); await Promise.resolve(); From 31c16d904841299354660902345798289c58c30d Mon Sep 17 00:00:00 2001 From: Farhan Date: Sat, 21 Nov 2026 22:10:26 +0000 Subject: [PATCH 15/16] Find dynamic imports inside template and JSX expressions The tokenizer skipped the expressions of templates and JSX without keeping their tokens, so an import("./x.js") in ${...} or {...} was never copied. A literal's token now holds the tokens of its expressions, and the import search walks them; the export search still sees one token per literal. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- .../reflex-base/src/reflex_base/js/source.py | 77 ++++++++++++------- tests/units/reflex_base/js/test_source.py | 24 ++++++ 2 files changed, 75 insertions(+), 26 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/js/source.py b/packages/reflex-base/src/reflex_base/js/source.py index 2ec8deec32d..116e5f4894c 100644 --- a/packages/reflex-base/src/reflex_base/js/source.py +++ b/packages/reflex-base/src/reflex_base/js/source.py @@ -11,6 +11,7 @@ import dataclasses import re +from collections.abc import Iterator, Sequence from pathlib import Path # Files whose imports are followed; other local files (CSS, JSON) are copied. @@ -79,6 +80,8 @@ class Token: value: str # Whether a line break precedes the token. newline: bool + # For a template or JSX literal, the tokens of the expressions in it. + inner: tuple[Token, ...] = () @dataclasses.dataclass(frozen=True, slots=True) @@ -117,27 +120,31 @@ def _read_string(source: str, start: int) -> tuple[str, int]: return source[start + 1 :], len(source) -def _skip_expression(source: str, index: int, jsx: bool) -> int: +def _skip_expression(source: str, index: int, jsx: bool, inner: list[Token]) -> int: """Skip the expression in a template's ``${...}`` or JSX's ``{...}``. Args: source: The source. index: The index after the opening brace. jsx: Whether the source can contain JSX. + inner: Receives the expression's tokens. Returns: The index after the closing brace. """ - return _scan(source, index, jsx, nested=True)[1] + tokens, index = _scan(source, index, jsx, nested=True) + inner.extend(tokens) + return index -def _skip_template(source: str, start: int, jsx: bool) -> int: +def _skip_template(source: str, start: int, jsx: bool, inner: list[Token]) -> int: """Skip a template literal, including the expressions in it. Args: source: The source. start: The index of the opening backtick. jsx: Whether the source can contain JSX. + inner: Receives the tokens of its expressions. Returns: The index after the closing backtick. @@ -150,17 +157,18 @@ def _skip_template(source: str, start: int, jsx: bool) -> int: index = ( match.end() + 1 if stop == "\\" - else _skip_expression(source, match.end(), jsx) + else _skip_expression(source, match.end(), jsx, inner) ) return len(source) -def _skip_jsx_tag(source: str, index: int) -> tuple[int, bool]: +def _skip_jsx_tag(source: str, index: int, inner: list[Token]) -> tuple[int, bool]: """Skip a JSX tag, with its attributes. Args: source: The source. index: The index after the tag's ``<``. + inner: Receives the tokens of its attribute expressions. Returns: The index after the tag's ``>``, and whether the tag closes itself. @@ -170,19 +178,20 @@ def _skip_jsx_tag(source: str, index: int) -> tuple[int, bool]: if stop == ">": return match.end(), source[match.start() - 1] == "/" index = ( - _skip_expression(source, match.end(), jsx=True) + _skip_expression(source, match.end(), True, inner) if stop == "{" else _read_string(source, match.start())[1] ) return len(source), True -def _skip_jsx(source: str, index: int) -> int: +def _skip_jsx(source: str, index: int, inner: list[Token]) -> int: """Skip a JSX element or fragment, with its children. Args: source: The source. index: The index of its opening ``<``. + inner: Receives the tokens of the expressions in it. Returns: The index after its closing tag. @@ -190,10 +199,10 @@ def _skip_jsx(source: str, index: int) -> int: depth = 0 while match := _JSX_STOP_RE.search(source, index): if match.group() == "{": - index = _skip_expression(source, match.end(), jsx=True) + index = _skip_expression(source, match.end(), True, inner) continue closing = source.startswith(" bool: return previous.kind == "name" and previous.value in _KEYWORDS_BEFORE_OPERAND -def _skip_operand(source: str, index: int, jsx: bool) -> int | None: +def _skip_operand(source: str, index: int, jsx: bool, inner: list[Token]) -> int | None: """Skip a regular expression or JSX element, where a value is expected. Args: source: The source. index: The index of a character that could start one. jsx: Whether the source can contain JSX. + inner: Receives the tokens of a JSX element's expressions. Returns: The index after it, or None if none starts here. @@ -262,12 +272,12 @@ def _skip_operand(source: str, index: int, jsx: bool) -> int | None: if source[index] == "/": return _skip_regex(source, index) if jsx and _JSX_START_RE.match(source, index): - return _skip_jsx(source, index) + return _skip_jsx(source, index, inner) return None def _read_token( - source: str, index: int, previous: Token | None, jsx: bool + source: str, index: int, previous: Token | None, jsx: bool, inner: list[Token] ) -> tuple[str, str, int]: """Read the token at an index. @@ -276,6 +286,7 @@ def _read_token( index: The index, where no whitespace or comment starts. previous: The token before, or None at the start. jsx: Whether the source can contain JSX. + inner: Receives the tokens of the expressions in a template or JSX. Returns: The token's kind and value, and the index after it. @@ -285,9 +296,9 @@ def _read_token( value, end = _read_string(source, index) return "string", value, end if char == "`": - return "literal", "", _skip_template(source, index, jsx) + return "literal", "", _skip_template(source, index, jsx, inner) if _operand_expected(previous) and ( - (end := _skip_operand(source, index, jsx)) is not None + (end := _skip_operand(source, index, jsx, inner)) is not None ): return "literal", "", end if name := _NAME_RE.match(source, index): @@ -318,14 +329,15 @@ def _scan(source: str, index: int, jsx: bool, nested: bool) -> tuple[list[Token] if start >= len(source): return tokens, start newline = "\n" in source[index:start] + inner: list[Token] = [] kind, value, index = _read_token( - source, start, tokens[-1] if tokens else None, jsx + source, start, tokens[-1] if tokens else None, jsx, inner ) if kind == "punct" and value in "{}": depth += 1 if value == "{" else -1 if nested and depth < 0: return tokens, index - tokens.append(Token(kind, value, newline)) + tokens.append(Token(kind, value, newline, tuple(inner))) def tokenize(source: str, jsx: bool = False) -> list[Token]: @@ -359,7 +371,7 @@ def _is(token: Token | None, kind: str, value: str | None = None) -> bool: ) -def _at(tokens: list[Token], index: int) -> Token | None: +def _at(tokens: Sequence[Token], index: int) -> Token | None: """Get the token at an index. Args: @@ -372,7 +384,7 @@ def _at(tokens: list[Token], index: int) -> Token | None: return tokens[index] if 0 <= index < len(tokens) else None -def _is_keyword(tokens: list[Token], index: int, keyword: str) -> bool: +def _is_keyword(tokens: Sequence[Token], index: int, keyword: str) -> bool: """Whether a token is a keyword, and not a property of that name. Args: @@ -651,7 +663,7 @@ def _export_names(tokens: list[Token], index: int) -> list[tuple[str, bool]]: return [] -def _statement_start(tokens: list[Token], index: int) -> int: +def _statement_start(tokens: Sequence[Token], index: int) -> int: """Find the ``import`` or ``export`` keyword an import clause follows. Args: @@ -668,7 +680,7 @@ def _statement_start(tokens: list[Token], index: int) -> int: return index -def _is_runtime_import(tokens: list[Token], index: int) -> bool: +def _is_runtime_import(tokens: Sequence[Token], index: int) -> bool: """Whether a string is the specifier of a module the source loads. Covers ``import ... from "x"``, ``import "x"``, ``import("x")`` and @@ -699,6 +711,22 @@ def _is_runtime_import(tokens: list[Token], index: int) -> bool: ) +def _runtime_imports(tokens: Sequence[Token]) -> Iterator[str]: + """Yield the specifiers of the modules some tokens load. + + Args: + tokens: The tokens, which may hold templates and JSX with expressions. + + Yields: + Each specifier, including those inside a template's or JSX's expressions. + """ + for index, token in enumerate(tokens): + if token.kind == "string" and _is_runtime_import(tokens, index): + yield token.value + elif token.inner: + yield from _runtime_imports(token.inner) + + def parse_script(text: str, jsx: bool = False) -> ScriptSource: """Read the exports and imports of a module. @@ -717,11 +745,8 @@ def parse_script(text: str, jsx: bool = False) -> ScriptSource: """ tokens = tokenize(text, jsx) exports: dict[str, bool] = {} - imports: dict[str, None] = {} - for index, token in enumerate(tokens): - if token.kind == "string" and _is_runtime_import(tokens, index): - imports[token.value] = None - elif _is_keyword(tokens, index, "export"): + for index in range(len(tokens)): + if _is_keyword(tokens, index, "export"): for name, callable_ in _export_names(tokens, index + 1): exports.setdefault(name, callable_) exports.pop("default", None) @@ -729,7 +754,7 @@ def parse_script(text: str, jsx: bool = False) -> ScriptSource: text=text, functions=[name for name, callable_ in exports.items() if callable_], values=[name for name, callable_ in exports.items() if not callable_], - imports=list(imports), + imports=list(dict.fromkeys(_runtime_imports(tokens))), ) diff --git a/tests/units/reflex_base/js/test_source.py b/tests/units/reflex_base/js/test_source.py index c4727ebf4dc..4574b579061 100644 --- a/tests/units/reflex_base/js/test_source.py +++ b/tests/units/reflex_base/js/test_source.py @@ -157,6 +157,30 @@ def test_parse_script_skips_jsx(): assert (script.functions, script.values) == (["same"], ["element"]) +@pytest.mark.parametrize( + ("source", "jsx", "imports"), + [ + ('const t = `${(await import("./a.js")).label}`;', False, ["./a.js"]), + ('const t = `${`${import("./a.js")}`}`;', False, ["./a.js"]), + ('const e =

x

;', True, ["./a.js"]), + ('const e =

{import("./a.js")}

;', True, ["./a.js"]), + ('const e =

{`${import("./a.js")}`}

;', True, ["./a.js"]), + ('const t = `${"./a.js"}`;', False, []), + ], +) +def test_parse_script_finds_imports_inside_expressions( + source: str, jsx: bool, imports: list[str] +): + """A dynamic import in a template's or JSX's expression is a module the source loads. + + Args: + source: The module source. + jsx: Whether it can contain JSX. + imports: The specifiers it imports. + """ + assert parse_script(source, jsx).imports == imports + + @pytest.mark.parametrize( ("source", "jsx", "functions"), [ From 1d464d9d6494deb7d80c441fa21cada745ffddd4 Mon Sep 17 00:00:00 2001 From: Farhan Date: Sat, 21 Nov 2026 22:20:22 +0000 Subject: [PATCH 16/16] Hold an event back behind every module call before it An event that waited for one pending module call could run ahead of the next one: the event before it may be a module call that only starts waiting for its module once the first has run. processEvent now waits until no call is pending. Claude-Session: https://claude.ai/code/session_01LwJUCVxPh3ihrnaFn6CWnf --- packages/reflex-frontend/src/utils/state.js | 4 +- .../reflex-frontend/tests/call_js.test.js | 57 ++++++++++++++++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/packages/reflex-frontend/src/utils/state.js b/packages/reflex-frontend/src/utils/state.js index 654d23fcc9a..f33f8e35268 100644 --- a/packages/reflex-frontend/src/utils/state.js +++ b/packages/reflex-frontend/src/utils/state.js @@ -1093,7 +1093,9 @@ export const processEvent = async (socket, navigate, params) => { // Apply the next event in the queue. const event = event_queue.shift(); - if (pendingModuleCall !== undefined) { + // An event that waited may find another call pending after it: the event + // before it can be a module call that now waits for its own module. + while (pendingModuleCall !== undefined) { // The call reports its own failure. await pendingModuleCall.catch(() => {}); } diff --git a/packages/reflex-frontend/tests/call_js.test.js b/packages/reflex-frontend/tests/call_js.test.js index d19a147fe9a..4a5cacb145c 100644 --- a/packages/reflex-frontend/tests/call_js.test.js +++ b/packages/reflex-frontend/tests/call_js.test.js @@ -17,7 +17,25 @@ const modules = vi.hoisted(() => { const gatedLoaded = new Promise((resolve) => { releaseGated = resolve; }); - return { seen: [], releaseSlow, slowLoaded, releaseGated, gatedLoaded }; + let releaseFirst = () => {}; + const firstLoaded = new Promise((resolve) => { + releaseFirst = resolve; + }); + let releaseSecond = () => {}; + const secondLoaded = new Promise((resolve) => { + releaseSecond = resolve; + }); + return { + seen: [], + releaseSlow, + slowLoaded, + releaseGated, + gatedLoaded, + releaseFirst, + firstLoaded, + releaseSecond, + secondLoaded, + }; }); vi.mock("$/utils/scripts", () => ({ @@ -46,6 +64,18 @@ vi.mock("$/utils/scripts", () => ({ globalThis.gatedValue = value; }, })), + "app.scripts.first": () => + modules.firstLoaded.then(() => ({ + write: (value) => { + globalThis.firstValue = value; + }, + })), + "app.scripts.second": () => + modules.secondLoaded.then(() => ({ + write: (value) => { + globalThis.secondValue = value; + }, + })), "app.scripts.broken": () => Promise.reject(new Error("Failed to fetch the module.")), }, @@ -136,6 +166,31 @@ test("an event queued while the modules load runs after the function before it", expect(globalThis.reads).toEqual(["gated"]); }); +test("an event waits for each module call before it, not only the first", async () => { + // The second call starts waiting for its module only once the first has run, + // and the script queued after both must wait for it too. + globalThis.reads = []; + const socket = { connected: true, emit: vi.fn() }; + const queued = [ + [callJsEvent("app.scripts.first", "write", ["first"])], + [callJsEvent("app.scripts.second", "write", ["second"])], + [ + { + name: "_call_script", + payload: { + javascript_code: + "reads.push([globalThis.firstValue, globalThis.secondValue])", + }, + }, + ], + ].map((events) => queueEvents(events, socket, false, null, null)); + modules.releaseFirst(); + await new Promise((resolve) => setTimeout(resolve)); + modules.releaseSecond(); + await Promise.all(queued); + expect(globalThis.reads).toEqual([["first", "second"]]); +}); + test("a function of a module still loading runs once it has loaded", async () => { const call = callJs("app.scripts.slow", "ping", ["late"]); await Promise.resolve();