diff --git a/docs/api-reference/browser_javascript.md b/docs/api-reference/browser_javascript.md index 6d5a5d28dce..c6f46122d02 100644 --- a/docs/api-reference/browser_javascript.md +++ b/docs/api-reference/browser_javascript.md @@ -360,8 +360,81 @@ Apps whose Content Security Policy forbids `unsafe-eval` can use them. - In a `.jsx` or `.tsx` file, JSX is read as JSX. In a `.js` or `.ts` file, `<` is never JSX, so a quote in what would be JSX text hides the code after it on that line. -- Reflex writes a type stub (`party.pyi`) next to the module when it compiles - the app, so type checkers resolve the import. + +### Type Stubs + +While you develop, Reflex writes a type stub next to each module the app +imports, `party.pyi` beside `party.js`, so type checkers resolve the import and +check each call. Argument types come from TypeScript annotations or, in a `.js` +file, JSDoc: + +```javascript +// my_app/scripts/party.js +/** + * @param {number} count + * @param {string} elementId + * @param {"left" | "right"} [side] + */ +export function celebrate(count, elementId, side) { /* ... */ } +``` + +gives this stub, shown without its imports and with each private alias, such +as `_float`, as the name it stands for: + +```python +# my_app/scripts/party.pyi +def celebrate( + count: float | Var[float], + elementId: str | Var[str] | EventCallback[*tuple[Any, ...]], + side: Literal["left", "right"] | Var[Literal["left", "right"]] = ..., + /, +) -> EventSpec: ... +``` + +so `celebrate("lots", "results")` is a type error. Arguments are positional, +as JavaScript passes them, and each also takes a var. A function reads its +arguments as JSON, so each type becomes the type of the JSON value Python sends: + +| TypeScript or JSDoc | Python | +| --- | --- | +| `string` | `str`, or an event handler, sent as its name | +| `number` | `float` | +| `boolean` | `bool` | +| `null`, `undefined`, `void` | `None` | +| `"a" \| "b"`, `1 \| 2`, `true` | `Literal["a", "b"]`, `Literal[1, 2]`, `Literal[True]` | +| an enum | `Literal` of its values | +| `T[]`, `[T, U]` | `Sequence[T]`, `Sequence[T \| U]` | +| `Record`, `{ [key: string]: T }` | `Mapping[str, T]` | +| an object type or interface of your own | `Mapping[str, Any]` | +| `A \| B` | the union of their types, `Any` if either is `Any` | +| no type, `any`, `unknown`, `object`, `bigint`, `A & B`, a type parameter | `Any` | +| a function, a class, `Date`, `Map`, `Set`, `Promise`, `HTMLElement` and other types of the browser or an npm package | `Any` | +| `x?: T`, or a default value | `x: T = ...`, when no required parameter follows | +| `...rest: T[]` | `*rest: T` | + +An overloaded function gets one `@overload` per signature. An export +TypeScript knows is only a type, such as `export interface Options`, or is not +a function, is left out of the stub. + +- Commit the stubs, so your editor and type checks in CI see them without + running the app. A stub only changes when the types do: Reflex rewrites it + when the module, a file it imports or reads types from (a `.d.ts` file, + `import type`, a JSDoc `import("./types")`), the app's frontend packages or + Reflex's stub generator changes. Writing a stub does not reload the backend. +- Stubs are written only in development. `reflex export` and `reflex run --env + prod` leave your source tree alone. +- When the app stops importing a module, Reflex removes the stub it wrote for + it. It never removes a `.pyi` file it did not write. +- Reflex reads the types with TypeScript, which it installs with the app's + frontend packages, run by Node.js or, without it, Bun. +- Pyright reports a stub that has no Python module beside it as + `reportMissingModuleSource`. Turn that rule off: + + ```toml + # pyproject.toml + [tool.pyright] + reportMissingModuleSource = "none" + ``` ### Errors @@ -384,6 +457,17 @@ Apps whose Content Security Policy forbids `unsafe-eval` can use them. not export a function of that name, or the frontend was built before the backend imported it. Export the function, or restart `reflex run` to rebuild the frontend. +- `Reflex could not read the argument types of the JavaScript modules + .../party.js, so their type stubs accept any arguments: reading them needs + Node.js or Bun, and neither was found.` The app still runs, but type + checkers accept any arguments. Install Node.js 22.22.0 or newer, then run + the app again. +- `Reflex could not read the argument types of the JavaScript modules + .../party.js, so their type stubs accept any arguments: script-signatures.js + failed: ...` The error after `failed:` says why. Reflex retries on the next + compile; if it keeps failing, delete `.web/node_modules` so the next run + reinstalls TypeScript, or report the error at + https://github.com/reflex-dev/reflex/issues. ## Using React Hooks diff --git a/news/+js-module-stubs.feature.md b/news/+js-module-stubs.feature.md new file mode 100644 index 00000000000..c03e3055d1c --- /dev/null +++ b/news/+js-module-stubs.feature.md @@ -0,0 +1 @@ +While you develop, the type stub Reflex writes next to an imported JavaScript module types each function's arguments from its TypeScript annotations or JSDoc, so type checkers catch a wrong argument in Python, such as `celebrate("lots")` for `@param {number} count`. Production builds leave the source tree alone, and writing a stub does not reload the backend; see [Type Stubs](https://reflex.dev/docs/api-reference/browser-javascript/#type-stubs). diff --git a/packages/reflex-base/news/+js-module-stubs.feature.md b/packages/reflex-base/news/+js-module-stubs.feature.md new file mode 100644 index 00000000000..cab5e8d141f --- /dev/null +++ b/packages/reflex-base/news/+js-module-stubs.feature.md @@ -0,0 +1 @@ +The type stub written next to a JavaScript module imported from Python types each function's arguments from TypeScript annotations or JSDoc, as the JSON value Python sends (`number` is `float`, `"a" | "b"` is `Literal["a", "b"]`, a `Map` or `HTMLElement` is `Any`), and declares only the functions the module exports at runtime. diff --git a/packages/reflex-base/src/reflex_base/constants/base.py b/packages/reflex-base/src/reflex_base/constants/base.py index 539758c3a6a..fa1cc933599 100644 --- a/packages/reflex-base/src/reflex_base/constants/base.py +++ b/packages/reflex-base/src/reflex_base/constants/base.py @@ -46,6 +46,9 @@ class Dirs(SimpleNamespace): SCRIPTS_PATH = UTILS + "/scripts" # The directory those modules are copied to. SCRIPT_MODULES_PATH = UTILS + "/script_modules" + # The type stubs written next to those modules in development, so the next + # compile removes the stub of a module the app no longer imports. + SCRIPT_STUBS = "script_stubs.json" # The name of the output directory. BUILD_DIR = "build" # The name of the static files directory. diff --git a/packages/reflex-base/src/reflex_base/js/modules.py b/packages/reflex-base/src/reflex_base/js/modules.py index dae03020d72..12dbb88d6fa 100644 --- a/packages/reflex-base/src/reflex_base/js/modules.py +++ b/packages/reflex-base/src/reflex_base/js/modules.py @@ -16,7 +16,6 @@ import dataclasses import functools import inspect -import keyword from pathlib import Path from types import ModuleType from typing import Any, NoReturn @@ -203,31 +202,3 @@ def _load_module(module: ModuleType, path: Path) -> None: "__getattr__", functools.partial(_missing_export, module.__name__, path, script), ) - - -def _render_stub(path: Path, functions: list[str]) -> str: - """Render a type stub for a JavaScript module, so type checkers see it. - - Exports that are not Python identifiers cannot be imported and are left - out. The arguments are untyped. - - Args: - path: The JavaScript file. - functions: Its exported functions. - - Returns: - The ``.pyi`` source. - """ - lines = [ - f"# Generated by Reflex from {path.name}; rewritten when it changes.", - "from typing import Any", - "", - "from reflex_base.event import EventSpec", - "", - ] - lines.extend( - f"def {name}(*args: Any) -> EventSpec: ..." - for name in functions - if name.isidentifier() and not keyword.iskeyword(name) - ) - return "\n".join(lines) + "\n" diff --git a/packages/reflex-base/src/reflex_base/js/source.py b/packages/reflex-base/src/reflex_base/js/source.py index 116e5f4894c..db75c686c59 100644 --- a/packages/reflex-base/src/reflex_base/js/source.py +++ b/packages/reflex-base/src/reflex_base/js/source.py @@ -11,7 +11,7 @@ import dataclasses import re -from collections.abc import Iterator, Sequence +from collections.abc import Iterator, Mapping, Sequence from pathlib import Path # Files whose imports are followed; other local files (CSS, JSON) are copied. @@ -22,11 +22,18 @@ STYLESHEET_EXTENSION = ".css" # Tried in order for an import without an extension, as the bundler does. _RESOLVE_EXTENSIONS = (".ts", ".tsx", ".js", ".jsx", ".mjs") +# The extension of the declaration file that types a runtime file. +_DECLARATION_EXTENSION = ".d.ts" _CSS_COMMENT_RE = re.compile(r"/\*.*?\*/", re.DOTALL) _CSS_REFERENCE_RE = re.compile( r"""@import\s+(?:url\(\s*)?["']?([^"')\s;]+)|url\(\s*["']?([^"')\s]+)["']?\s*\)""" ) +# The files a module reads types from in comments, which no tokenizer sees: +# JSDoc `import("./types")` and `/// `. +_TYPE_REFERENCE_RE = re.compile( + r"""\bimport\(\s*["'](\.[^"']+)["']\s*\)|///\s* tuple[str, int]: @@ -680,51 +691,52 @@ def _statement_start(tokens: Sequence[Token], index: int) -> int: return index -def _is_runtime_import(tokens: Sequence[Token], index: int) -> bool: - """Whether a string is the specifier of a module the source loads. - - Covers ``import ... from "x"``, ``import "x"``, ``import("x")`` and - ``export ... from "x"``, but not TypeScript's ``import type`` and - ``export type``, which load nothing. +def _import_kind(tokens: Sequence[Token], index: int) -> str | None: + """Tell whether a string is the specifier of an import, and of which kind. Args: tokens: The tokens. index: The index of the string. Returns: - Whether it is such a specifier. + "runtime" for a module the source loads: ``import ... from "x"``, + ``import "x"``, ``import("x")`` and ``export ... from "x"``; "type" for + TypeScript's ``import type`` and ``export type``, which load nothing + but types; None for any other string. """ if _is(_at(tokens, index - 1), "punct", "("): - return _is_keyword(tokens, index - 2, "import") + return "runtime" if _is_keyword(tokens, index - 2, "import") else None if _is_keyword(tokens, index - 1, "import"): - return True + return "runtime" if not _is(_at(tokens, index - 1), "name", "from"): - return False + return None start = _statement_start(tokens, index - 2) # `import type X from` loads nothing, but `import type from` and # `import type, { x } from` import a default export named type. after = _at(tokens, start + 2) - return not ( + type_only = ( _is(tokens[start + 1], "name", "type") and not _is(after, "name", "from") and not _is(after, "punct", ",") ) + return "type" if type_only else "runtime" -def _runtime_imports(tokens: Sequence[Token]) -> Iterator[str]: - """Yield the specifiers of the modules some tokens load. +def _imports(tokens: Sequence[Token]) -> Iterator[tuple[str, str]]: + """Yield the specifiers some tokens import from, with their kind. Args: tokens: The tokens, which may hold templates and JSX with expressions. Yields: - Each specifier, including those inside a template's or JSX's expressions. + Each specifier's ``_import_kind`` and the specifier, including those + inside a template's or JSX's expressions. """ for index, token in enumerate(tokens): - if token.kind == "string" and _is_runtime_import(tokens, index): - yield token.value + if token.kind == "string" and (kind := _import_kind(tokens, index)): + yield kind, token.value elif token.inner: - yield from _runtime_imports(token.inner) + yield from _imports(token.inner) def parse_script(text: str, jsx: bool = False) -> ScriptSource: @@ -749,12 +761,24 @@ def parse_script(text: str, jsx: bool = False) -> ScriptSource: if _is_keyword(tokens, index, "export"): for name, callable_ in _export_names(tokens, index + 1): exports.setdefault(name, callable_) + imports: dict[str, None] = {} + type_imports: dict[str, None] = {} + for kind, specifier in _imports(tokens): + if kind == "runtime": + imports[specifier] = None + elif specifier.startswith("."): + type_imports[specifier] = None + for jsdoc, reference in _TYPE_REFERENCE_RE.findall(text): + type_imports[jsdoc or reference] = None exports.pop("default", None) return ScriptSource( text=text, functions=[name for name, callable_ in exports.items() if callable_], values=[name for name, callable_ in exports.items() if not callable_], - imports=list(dict.fromkeys(_runtime_imports(tokens))), + imports=list(imports), + type_imports=[ + specifier for specifier in type_imports if specifier not in imports + ], ) @@ -889,3 +913,66 @@ def module_files(path: Path) -> dict[Path, ScriptSource | None]: files[target] = None pending.append(target) return files + + +def _resolve_declaration(directory: Path, specifier: str) -> Path | None: + """Find the declaration file TypeScript types a relative import with. + + That is ``helpers.d.ts`` for ``./helpers`` and ``./helpers.js`` alike, or + ``types/index.d.ts`` for a directory ``./types``. + + Args: + directory: The importing file's directory. + specifier: The relative specifier. + + Returns: + The declaration file, or None if there is none. + """ + target = directory / strip_query(specifier) + stem = target.with_suffix("") if target.suffix in _RESOLVE_EXTENSIONS else target + candidates = ( + stem.with_name(stem.name + _DECLARATION_EXTENSION), + target / f"index{_DECLARATION_EXTENSION}", + ) + return next((candidate for candidate in candidates if candidate.is_file()), None) + + +def type_files(files: Mapping[Path, ScriptSource | None]) -> dict[Path, ScriptSource]: + """Find the files a module reads types from that the bundler never loads. + + These are the declaration files beside the files it imports and the files + its type-only imports, JSDoc ``import("./types")`` and triple-slash + references name, directly or not. They change the module's types, but are + not copied into the compiled app. + + Args: + files: The module's files, from ``module_files``. + + Returns: + The resolved files not among ``files``, each with its script. + """ + found: dict[Path, ScriptSource] = {} + pending = [(file, script) for file, script in files.items() if script is not None] + while pending: + file, script = pending.pop() + specifiers = [ + *(specifier for specifier in script.imports if specifier.startswith(".")), + *script.type_imports, + ] + for specifier in specifiers: + for target in ( + _resolve_local(file.parent, specifier), + _resolve_declaration(file.parent, specifier), + ): + if target is None: + continue + target = target.resolve() + if ( + target in files + or target in found + or target.suffix not in SCRIPT_SOURCE_EXTENSIONS + ): + continue + found[target] = read_script(target) + pending.append((target, found[target])) + return found diff --git a/packages/reflex-base/src/reflex_base/js/stubs.py b/packages/reflex-base/src/reflex_base/js/stubs.py new file mode 100644 index 00000000000..730d17349f1 --- /dev/null +++ b/packages/reflex-base/src/reflex_base/js/stubs.py @@ -0,0 +1,433 @@ +"""Type stubs for JavaScript modules imported from Python. + +A stub sits next to its module (``party.js`` gets ``party.pyi``), so type +checkers resolve ``from my_app.scripts.party import celebrate`` and check the +call. ``script-signatures.js``, run in the app's ``.web``, reads each export's +parameters from TypeScript annotations or JSDoc and classifies their types into +the JSON values Python can send; this module renders them. + +Internal: what an app relies on is the stub itself, which +``docs/api-reference/browser_javascript.md`` documents. +""" + +from __future__ import annotations + +import functools +import hashlib +import json +import keyword +import os +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Literal, TypeAlias, TypedDict + +from reflex_base import constants +from reflex_base.js.source import ScriptSource, type_files + +# Installed with an app that imports a JavaScript module, to read its types. +# Kept equal to the typescript reflex-frontend type-checks itself with. +_TYPESCRIPT_PACKAGE = "typescript@5.9.3" + +# The script in .web that prints the signatures of modules' exports. +_SIGNATURES_SCRIPT = "script-signatures.js" + +_HEADER_PREFIX = "# Generated by Reflex from " +_HEADER = _HEADER_PREFIX + "{name} ({digest}); rewritten when it changes." + +# The modules of the names a stub uses for types, in import order. The stub +# imports them as private aliases (`_str`), so that an export named `str` or +# `Var` cannot replace one. +_TYPE_MODULES = { + "builtins": ("bool", "float", "str", "tuple"), + "collections.abc": ("Mapping", "Sequence"), + "typing": ("Any", "Literal", "overload"), + "reflex_base.event": ("EventCallback", "EventSpec"), + "reflex_base.vars.base": ("Var",), +} +_ALIASES = frozenset(f"_{name}" for names in _TYPE_MODULES.values() for name in names) + +# The JSON types `script-signatures.js` classifies parameters into, whose +# grammar it documents. +_Leaf: TypeAlias = Literal["any", "str", "float", "bool", "None"] +_LEAF_NAMES = {"any": "Any", "str": "str", "float": "float", "bool": "bool"} + + +class _JsonLiteral(TypedDict): + """A literal string, integer or boolean.""" + + kind: Literal["literal"] + value: str | int | bool + + +class _JsonList(TypedDict): + """An array or tuple, of the union of its items' types.""" + + kind: Literal["list"] + item: JsonType + + +class _JsonMapping(TypedDict): + """An object with string keys.""" + + kind: Literal["mapping"] + value: JsonType + + +class _JsonUnion(TypedDict): + """A union of two or more types.""" + + kind: Literal["union"] + members: list[JsonType] + + +JsonType: TypeAlias = "_Leaf | _JsonLiteral | _JsonList | _JsonMapping | _JsonUnion" + + +class Parameter(TypedDict): + """A parameter of a signature, as ``script-signatures.js`` prints it.""" + + name: str + type: JsonType + optional: bool + rest: bool + + +class Signature(TypedDict): + """A call signature of an export.""" + + parameters: list[Parameter] + + +class Export(TypedDict): + """An export and its call signatures; none when it is not a function.""" + + name: str + signatures: list[Signature] + # Whether its type is unknown (`any`), so it may still be a function. + untyped: bool + + +@functools.cache +def _renderer_digest() -> bytes: + """Hash the code that makes a stub, so a change to it rewrites every stub. + + Returns: + The digest of this module and ``script-signatures.js``. + """ + script = constants.Templates.Dirs.WEB_TEMPLATE / _SIGNATURES_SCRIPT + return hashlib.sha256(Path(__file__).read_bytes() + script.read_bytes()).digest() + + +def _module_digest( + path: Path, files: Mapping[Path, ScriptSource | None], dependencies: bytes +) -> str: + """Hash what a module's stub is made from. + + That is the scripts the module loads, the files it reads types from, the + app's package manifest, whose packages' declarations it may use, and the + code that makes stubs. + + Args: + path: The module. + files: Its files, from ``module_files``. + dependencies: The app's ``package.json``. + + Returns: + A short hex digest. + """ + digest = hashlib.sha256(_renderer_digest()) + directory = path.resolve().parent + scripts = [ + *((file, script) for file, script in files.items() if script is not None), + *type_files(files).items(), + ] + for file, script in scripts: + # Relative, so the same sources give the same stub on every machine. + digest.update(os.path.relpath(file, directory).encode()) + digest.update(b"\0") + digest.update(script.text.encode()) + digest.update(b"\0") + digest.update(dependencies) + return digest.hexdigest()[:16] + + +def _stub_header(path: Path, digest: str) -> str: + """Get the first line of a module's stub, which records what it was made from. + + Args: + path: The module. + digest: Its ``_module_digest``. + + Returns: + The header line. + """ + return _HEADER.format(name=path.name, digest=digest) + + +def _read_header(stub: Path) -> str | None: + """Read the first line of a stub. + + Args: + stub: The stub. + + Returns: + The line, or None if there is no such file. + """ + try: + with stub.open(encoding="utf-8") as file: + return file.readline().rstrip("\n") + except FileNotFoundError: + return None + + +def _is_generated(header: str | None) -> bool: + """Whether a stub's header says Reflex wrote it, and so may rewrite it. + + Args: + header: The stub's first line, from ``_read_header``. + + Returns: + Whether it is a header ``_stub_header`` makes. + """ + return header is not None and header.startswith(_HEADER_PREFIX) + + +def _use(name: str, used: set[str]) -> str: + """Get a stub's alias for a type name, and record that the stub imports it. + + Args: + name: The name, such as ``str``. + used: The names the stub uses. + + Returns: + The alias, such as ``_str``. + """ + used.add(name) + return f"_{name}" + + +def _render_literal(value: str | int | bool) -> str: + """Render a literal value as Python source. + + Args: + value: The value. + + Returns: + The source: a double-quoted string, ``True``, ``False`` or a number. + """ + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False) + return repr(value) + + +def _render_type(type_: JsonType, used: set[str]) -> str: + """Render the Python type of a JSON value. + + A union's literals are merged into one ``Literal``, after the other + members and before ``None``. + + Args: + type_: The JSON type. + used: The names the stub uses, which this adds to. + + Returns: + The type. + """ + if isinstance(type_, str): + return "None" if type_ == "None" else _use(_LEAF_NAMES[type_], used) + if type_["kind"] == "literal": + return f"{_use('Literal', used)}[{_render_literal(type_['value'])}]" + if type_["kind"] == "list": + return f"{_use('Sequence', used)}[{_render_type(type_['item'], used)}]" + if type_["kind"] == "mapping": + key = _use("str", used) + return f"{_use('Mapping', used)}[{key}, {_render_type(type_['value'], used)}]" + members: list[str] = [] + literals: list[str] = [] + for member in type_["members"]: + if isinstance(member, str) or member["kind"] != "literal": + if member != "None": + members.append(_render_type(member, used)) + else: + literals.append(_render_literal(member["value"])) + if literals: + members.append(f"{_use('Literal', used)}[{', '.join(literals)}]") + if "None" in type_["members"]: + members.append("None") + return " | ".join(members) + + +def _annotation(type_: JsonType, used: set[str]) -> str: + """Get the annotation of a parameter: its value, or a Var of it. + + A string parameter also takes an event handler, which is sent as its name. + + Args: + type_: The parameter's JSON type. + used: The names the stub uses, which this adds to. + + Returns: + The annotation. + """ + if type_ == "any": + return _use("Any", used) + value = _render_type(type_, used) + annotation = f"{value} | {_use('Var', used)}[{value}]" + if type_ == "str" or ( + not isinstance(type_, str) + and type_["kind"] == "union" + and "str" in type_["members"] + ): + handler = _use("EventCallback", used) + annotation += f" | {handler}[*{_use('tuple', used)}[{_use('Any', used)}, ...]]" + return annotation + + +def _render_signature(name: str, signature: Signature, used: set[str]) -> str: + """Render one signature of an export as a ``def``. + + Arguments are positional-only, since JavaScript passes them by position, + so a parameter name that is not a Python identifier is renamed safely. + + Args: + name: The export name. + signature: The signature. + used: The names the stub uses, which this adds to. + + Returns: + The ``def`` line. + """ + parameters = signature["parameters"] + rest_index = next( + (index for index, parameter in enumerate(parameters) if parameter["rest"]), + len(parameters), + ) + # Python can only omit trailing arguments: a parameter with a default that a + # required one follows is required. + omittable = rest_index + while omittable > 0 and parameters[omittable - 1]["optional"]: + omittable -= 1 + # Rename the parameters that are not Python identifiers to names no other + # parameter has; arguments are positional-only, so callers never see them. + names = { + parameter["name"] + for parameter in parameters + if parameter["name"].isidentifier() and not keyword.iskeyword(parameter["name"]) + } + params: list[str] = [] + rest = "" + for index, parameter in enumerate(parameters[: rest_index + 1]): + param_name = parameter["name"] + if param_name not in names: + param_name = f"arg{index}" + while param_name in names: + param_name += "_" + names.add(param_name) + annotation = _annotation(parameter["type"], used) + if parameter["rest"]: + rest = f"*{param_name}: {annotation}" + break + default = " = ..." if index >= omittable else "" + params.append(f"{param_name}: {annotation}{default}") + rendered = [*params, "/"] if params else [] + if rest: + rendered.append(rest) + return f"def {name}({', '.join(rendered)}) -> {_use('EventSpec', used)}: ..." + + +def _importable(name: str) -> bool: + """Whether an export can be imported in Python. + + Args: + name: The export name. + + Returns: + Whether it is an identifier, not a keyword, and not one of the stub's + own names for the types it uses. + """ + return name.isidentifier() and not keyword.iskeyword(name) and name not in _ALIASES + + +def _render_imports(used: set[str]) -> list[str]: + """Render the imports of the names a stub uses, the standard library first. + + Args: + used: The names. + + Returns: + The import lines. + """ + groups: tuple[list[str], list[str]] = ([], []) + for module, names in _TYPE_MODULES.items(): + if aliases := [f"{name} as _{name}" for name in names if name in used]: + group = groups[module.startswith("reflex_base.")] + group.append(f"from {module} import {', '.join(aliases)}") + return [*groups[0], *([""] if all(groups) else []), *groups[1]] + + +def _render_stub( + path: Path, + digest: str, + functions: Iterable[str], + exports: Mapping[str, Export] | None, +) -> str: + """Render the stub of a module. + + It declares the functions the module exports that Python can import. An + export TypeScript knows is not a value, such as a type, is left out, so is + one it knows is not a function; one of unknown type, or of a module whose + types could not be read, has untyped arguments. An export with several + signatures gets one overload each. + + Args: + path: The module. + digest: Its ``_module_digest``. + functions: The names it exports that may be functions, from + ``parse_script``. + exports: Its exports by name, from ``script-signatures.js``, or None if + they could not be read. + + Returns: + The ``.pyi`` source. + """ + used: set[str] = set() + definitions: list[str] = [] + for name in functions: + if not _importable(name): + continue + if exports is None: + signatures, untyped = [], True + elif name in exports: + signatures, untyped = exports[name]["signatures"], exports[name]["untyped"] + else: + continue + if untyped: + any_ = _use("Any", used) + definitions.append( + f"def {name}(*args: {any_}) -> {_use('EventSpec', used)}: ..." + ) + for signature in signatures: + if len(signatures) > 1: + definitions.append(f"@{_use('overload', used)}") + definitions.append(_render_signature(name, signature, used)) + lines = [_stub_header(path, digest)] + if definitions: + lines.extend([*_render_imports(used), "", *definitions]) + return "\n".join(lines) + "\n" + + +def _exports_by_file( + output: Mapping[str, list[Export]], +) -> dict[Path, dict[str, Export]]: + """Key the output of ``script-signatures.js`` by resolved path and name. + + Args: + output: The parsed JSON. + + Returns: + The exports of each module, by name. + """ + return { + Path(file).resolve(): {export["name"]: export for export in exports} + for file, exports in output.items() + } diff --git a/packages/reflex-frontend/README.md b/packages/reflex-frontend/README.md index 97593dc457e..0e59cd01341 100644 --- a/packages/reflex-frontend/README.md +++ b/packages/reflex-frontend/README.md @@ -17,7 +17,10 @@ In a checkout of the repository there is no embedded copy: reflex-base reads `src/` in place, so the next `reflex init` copies an edit without a rebuild. Everything in `src/` ships to every app, including the build tooling an app -runs (`compress-static.js`, the PostCSS config, the Vite plugins). Tests, +runs (`compress-static.js`, the PostCSS config, the Vite plugins) and +`script-signatures.js`, which reads the types of the JavaScript modules an app +imports from Python for their stubs; its output, the contract with +`reflex_base/js/stubs.py`, is `tests/fixtures/signatures/signatures.json`. Tests, fixtures and this package's own lint and type-check setup stay outside it. A change under `src/` is a change to reflex-base: its news fragment goes in diff --git a/packages/reflex-frontend/src/script-signatures.js b/packages/reflex-frontend/src/script-signatures.js new file mode 100644 index 00000000000..6394b692bf9 --- /dev/null +++ b/packages/reflex-frontend/src/script-signatures.js @@ -0,0 +1,248 @@ +/* Print the call signatures of the functions JavaScript modules export, as + * JSON, for the Python type stubs Reflex writes next to modules an app imports + * from Python. Types come from TypeScript annotations or, in a .js file, JSDoc. + * + * Arguments reach the browser as JSON, so each parameter's type is classified + * here into the JSON value Python can send for it ("any" for a type JSON has + * no form of, such as a function, a `Map` or an `HTMLElement`): + * + * type = "any" | "str" | "float" | "bool" | "None" + * | {kind: "literal", value: string | integer | boolean} + * | {kind: "list", item: type} + * | {kind: "mapping", value: type} (string keys) + * | {kind: "union", members: type[]} (two or more, sorted) + * + * Usage: node script-signatures.js ... + * Run from the app's .web directory, where typescript is installed. */ + +import { createRequire } from "node:module"; +import { join } from "node:path"; + +const require = createRequire(import.meta.url); +const ts = require("typescript"); + +const web = process.cwd(); +const files = process.argv.slice(2); +const program = ts.createProgram(files, { + allowJs: true, + noEmit: true, + skipLibCheck: true, + // Keep `null` and `undefined` in the types, for `None` in Python. + strictNullChecks: true, + target: ts.ScriptTarget.ES2022, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.Bundler, + // Resolve `$/...` and npm packages as the app's bundler does, so imported + // types are known; a module outside .web finds no node_modules of its own. + baseUrl: web, + paths: { + "$/*": [join(web, "*")], + "*": [ + join(web, "node_modules", "@types", "*"), + join(web, "node_modules", "*"), + ], + }, +}); +const checker = program.getTypeChecker(); + +/** @typedef {string | {kind: string, [key: string]: any}} JsonType */ + +const BOOLEAN_LITERALS = [true, false].map((value) => + JSON.stringify({ kind: "literal", value }), +); + +/** + * Combine types into their union, in an order that does not depend on the + * order the checker happened to create them in. + * @param {JsonType[]} types The types. + * @returns {JsonType} The union; "any" if a member is, or if there is none. + */ +const union = (types) => { + if (types.includes("any")) return "any"; + const members = new Map(types.map((type) => [JSON.stringify(type), type])); + if (BOOLEAN_LITERALS.every((key) => members.has(key))) { + for (const key of BOOLEAN_LITERALS) members.delete(key); + members.set(JSON.stringify("bool"), "bool"); + } + const sorted = [...members.keys()].sort().map((key) => members.get(key)); + if (sorted.length < 2) return sorted[0] ?? "any"; + return { kind: "union", members: sorted }; +}; + +/** + * Whether an object type has no JSON form: a function, a class instance, or an + * interface TypeScript's libraries or an npm package declare, such as `Map`, + * `Promise` or `HTMLElement`. + * @param {any} type The type. + * @returns {boolean} Whether it is. + */ +const isOpaque = (type) => { + if ( + type.getCallSignatures().length > 0 || + type.getConstructSignatures().length > 0 + ) { + return true; + } + // Every object type has a symbol: `__type` for a type literal. + const symbol = type.getSymbol(); + if (symbol.flags & ts.SymbolFlags.Class) return true; + return ( + Boolean(symbol.flags & ts.SymbolFlags.Interface) && + symbol.declarations.some((declaration) => { + const file = declaration.getSourceFile(); + return ( + program.isSourceFileDefaultLibrary(file) || + program.isSourceFileFromExternalLibrary(file) + ); + }) + ); +}; + +// The object types being classified, outermost first. A type that contains +// itself, such as `interface Tree { [key: string]: Tree }`, has no finite JSON +// shape: its repeat is classified as any. +const classifying = new Set(); + +/** + * Classify an object type: an array or tuple is a list, a string-keyed record + * a mapping of its values, and another plain object a mapping. + * @param {any} type The type. + * @returns {JsonType} Its JSON type. + */ +const classifyObject = (type) => { + if (classifying.has(type)) return "any"; + classifying.add(type); + try { + return classifyShape(type); + } finally { + classifying.delete(type); + } +}; + +/** + * Classify an object type that is not already being classified. + * @param {any} type The type. + * @returns {JsonType} Its JSON type. + */ +const classifyShape = (type) => { + if (checker.isArrayType(type)) { + return { kind: "list", item: classify(checker.getTypeArguments(type)[0]) }; + } + if (checker.isTupleType(type)) { + const items = checker.getTypeArguments(type).map(classify); + return { kind: "list", item: union(items) }; + } + const index = checker.getIndexInfoOfType(type, ts.IndexKind.String); + if (index) return { kind: "mapping", value: classify(index.type) }; + return isOpaque(type) ? "any" : { kind: "mapping", value: "any" }; +}; + +/** + * Classify a type into the JSON value Python can send for it. An enum is the + * union of its members' values. + * @param {any} type The type. + * @returns {JsonType} Its JSON type. + */ +const classify = (type) => { + const flags = type.flags; + if (type.isUnion()) return union(type.types.map(classify)); + if (type.isStringLiteral()) return { kind: "literal", value: type.value }; + if (type.isNumberLiteral()) { + // Python's `Literal` takes integers, not other numbers. + return Number.isSafeInteger(type.value) + ? { kind: "literal", value: type.value } + : "float"; + } + if (flags & ts.TypeFlags.BooleanLiteral) { + return { kind: "literal", value: type === checker.getTrueType() }; + } + if (flags & ts.TypeFlags.StringLike) return "str"; + // `boolean` needs no case: it is the union `true | false`, which `union` + // makes "bool". + if (flags & ts.TypeFlags.NumberLike) return "float"; + if ( + flags & + (ts.TypeFlags.Null | ts.TypeFlags.Undefined | ts.TypeFlags.Void) + ) { + return "None"; + } + if (flags & ts.TypeFlags.Object) return classifyObject(type); + // `any`, `unknown`, `object`, `bigint`, symbols, intersections, generics. + return "any"; +}; + +/** + * Describe one parameter of a signature. + * @param {any} parameter The parameter symbol. + * @param {any} location The node to resolve its type at. + * @returns {{name: string, type: JsonType, optional: boolean, rest: boolean}} The parameter. + */ +const describeParameter = (parameter, location) => { + const declaration = parameter.valueDeclaration; + const rest = Boolean(declaration?.dotDotDotToken); + const optional = Boolean( + declaration && + (declaration.questionToken || + declaration.initializer || + checker.isOptionalParameter(declaration)), + ); + /** @type {any} */ + const type = checker.getTypeOfSymbolAtLocation(parameter, location); + let described; + if (rest) { + // A rest parameter's element type; a tuple's elements have no single type. + described = checker.isArrayType(type) + ? classify(checker.getTypeArguments(type)[0]) + : "any"; + } else if (optional && type.isUnion()) { + // The checker adds `undefined` to an optional parameter's type; Python + // leaves the argument out instead. + const defined = type.types.filter( + (member) => !(member.flags & ts.TypeFlags.Undefined), + ); + described = union(defined.map(classify)); + } else { + described = classify(type); + } + return { name: parameter.getName(), type: described, optional, rest }; +}; + +/** + * Whether an export is a value, which the module has at runtime, and not only + * a type, such as `export type Foo` or `export interface Bar`. + * @param {any} symbol The exported symbol. + * @returns {boolean} Whether it is a value. + */ +const isValue = (symbol) => { + const target = + symbol.flags & ts.SymbolFlags.Alias + ? checker.getAliasedSymbol(symbol) + : symbol; + return Boolean(target.flags & ts.SymbolFlags.Value); +}; + +/** @type {Record} */ +const result = {}; +for (const file of files) { + const source = program.getSourceFile(file); + const moduleSymbol = checker.getSymbolAtLocation(source); + result[file] = (moduleSymbol ? checker.getExportsOfModule(moduleSymbol) : []) + .filter((symbol) => symbol.getName() !== "default" && isValue(symbol)) + .map((symbol) => { + const type = checker.getTypeOfSymbolAtLocation(symbol, source); + return { + name: symbol.getName(), + signatures: type.getCallSignatures().map((signature) => ({ + parameters: signature + .getParameters() + .map((parameter) => describeParameter(parameter, source)), + })), + // An export of unknown type, such as one from a package without + // declarations, may still be a function. + untyped: Boolean( + type.flags & (ts.TypeFlags.Any | ts.TypeFlags.Unknown), + ), + }; + }); +} +process.stdout.write(JSON.stringify(result)); diff --git a/packages/reflex-frontend/tests/fixtures/signatures/jsdoc.js b/packages/reflex-frontend/tests/fixtures/signatures/jsdoc.js new file mode 100644 index 00000000000..74dd3c6e515 --- /dev/null +++ b/packages/reflex-frontend/tests/fixtures/signatures/jsdoc.js @@ -0,0 +1,22 @@ +/** + * @param {number} count + * @param {string} elementId + * @param {boolean} [loud] + */ +export function celebrate(count, elementId, loud = false) { + return [count, elementId, loud]; +} + +/** @param {...string} names */ +export function many(...names) { + return names; +} + +export const untyped = (a, b) => a + b; + +export const notAFunction = 3; + +export default function () {} + +/** @type {any} */ +export const anything = globalThis.somethingUntyped; diff --git a/packages/reflex-frontend/tests/fixtures/signatures/recursive.ts b/packages/reflex-frontend/tests/fixtures/signatures/recursive.ts new file mode 100644 index 00000000000..77d142ac5de --- /dev/null +++ b/packages/reflex-frontend/tests/fixtures/signatures/recursive.ts @@ -0,0 +1,7 @@ +export interface Tree { + [key: string]: Tree; +} + +export type Nested = Nested[]; + +export function grow(tree: Tree, nested: Nested, depth: number): void {} diff --git a/packages/reflex-frontend/tests/fixtures/signatures/reexported.js b/packages/reflex-frontend/tests/fixtures/signatures/reexported.js new file mode 100644 index 00000000000..289df7dc46a --- /dev/null +++ b/packages/reflex-frontend/tests/fixtures/signatures/reexported.js @@ -0,0 +1 @@ +export function fromReexported() {} diff --git a/packages/reflex-frontend/tests/fixtures/signatures/script.js b/packages/reflex-frontend/tests/fixtures/signatures/script.js new file mode 100644 index 00000000000..17b159bc23a --- /dev/null +++ b/packages/reflex-frontend/tests/fixtures/signatures/script.js @@ -0,0 +1,2 @@ +// A script, not a module: it exports nothing. +globalThis.loaded = true; diff --git a/packages/reflex-frontend/tests/fixtures/signatures/shapes.d.ts b/packages/reflex-frontend/tests/fixtures/signatures/shapes.d.ts new file mode 100644 index 00000000000..298f3e8368a --- /dev/null +++ b/packages/reflex-frontend/tests/fixtures/signatures/shapes.d.ts @@ -0,0 +1 @@ +export type Shape = "circle" | "square"; diff --git a/packages/reflex-frontend/tests/fixtures/signatures/signatures.json b/packages/reflex-frontend/tests/fixtures/signatures/signatures.json new file mode 100644 index 00000000000..a327f3304d0 --- /dev/null +++ b/packages/reflex-frontend/tests/fixtures/signatures/signatures.json @@ -0,0 +1,509 @@ +{ + "jsdoc.js": [ + { + "name": "celebrate", + "signatures": [ + { + "parameters": [ + { + "name": "count", + "type": "float", + "optional": false, + "rest": false + }, + { + "name": "elementId", + "type": "str", + "optional": false, + "rest": false + }, + { + "name": "loud", + "type": "bool", + "optional": true, + "rest": false + } + ] + } + ], + "untyped": false + }, + { + "name": "many", + "signatures": [ + { + "parameters": [ + { + "name": "names", + "type": "str", + "optional": false, + "rest": true + } + ] + } + ], + "untyped": false + }, + { + "name": "untyped", + "signatures": [ + { + "parameters": [ + { + "name": "a", + "type": "any", + "optional": false, + "rest": false + }, + { + "name": "b", + "type": "any", + "optional": false, + "rest": false + } + ] + } + ], + "untyped": false + }, + { + "name": "notAFunction", + "signatures": [], + "untyped": false + }, + { + "name": "anything", + "signatures": [], + "untyped": true + } + ], + "typed.ts": [ + { + "name": "pick", + "signatures": [ + { + "parameters": [ + { + "name": "mode", + "type": { + "kind": "union", + "members": [ + { + "kind": "literal", + "value": "a" + }, + { + "kind": "literal", + "value": "b" + } + ] + }, + "optional": false, + "rest": false + }, + { + "name": "maybe", + "type": { + "kind": "union", + "members": ["None", "str"] + }, + "optional": false, + "rest": false + } + ] + } + ], + "untyped": false + }, + { + "name": "overloaded", + "signatures": [ + { + "parameters": [ + { + "name": "n", + "type": "float", + "optional": false, + "rest": false + } + ] + }, + { + "parameters": [] + } + ], + "untyped": false + }, + { + "name": "tupleRest", + "signatures": [ + { + "parameters": [ + { + "name": "args", + "type": "any", + "optional": false, + "rest": true + } + ] + } + ], + "untyped": false + }, + { + "name": "defaultFirst", + "signatures": [ + { + "parameters": [ + { + "name": "prefix", + "type": "str", + "optional": true, + "rest": false + }, + { + "name": "count", + "type": "float", + "optional": false, + "rest": false + } + ] + } + ], + "untyped": false + } + ], + "types.ts": [ + { + "name": "literals", + "signatures": [ + { + "parameters": [ + { + "name": "color", + "type": { + "kind": "union", + "members": [ + { + "kind": "literal", + "value": "green" + }, + { + "kind": "literal", + "value": "red" + } + ] + }, + "optional": false, + "rest": false + }, + { + "name": "level", + "type": { + "kind": "union", + "members": [ + { + "kind": "literal", + "value": 0 + }, + { + "kind": "literal", + "value": 1 + } + ] + }, + "optional": false, + "rest": false + }, + { + "name": "mode", + "type": { + "kind": "union", + "members": [ + { + "kind": "literal", + "value": "a" + }, + { + "kind": "literal", + "value": "b" + }, + { + "kind": "literal", + "value": 2 + }, + { + "kind": "literal", + "value": true + } + ] + }, + "optional": false, + "rest": false + }, + { + "name": "ratio", + "type": "float", + "optional": false, + "rest": false + }, + { + "name": "flag", + "type": { + "kind": "literal", + "value": true + }, + "optional": false, + "rest": false + }, + { + "name": "shape", + "type": { + "kind": "union", + "members": [ + { + "kind": "literal", + "value": "circle" + }, + { + "kind": "literal", + "value": "square" + } + ] + }, + "optional": false, + "rest": false + } + ] + } + ], + "untyped": false + }, + { + "name": "containers", + "signatures": [ + { + "parameters": [ + { + "name": "names", + "type": { + "kind": "list", + "item": "str" + }, + "optional": false, + "rest": false + }, + { + "name": "pair", + "type": { + "kind": "list", + "item": { + "kind": "union", + "members": ["float", "str"] + } + }, + "optional": false, + "rest": false + }, + { + "name": "empty", + "type": { + "kind": "list", + "item": "any" + }, + "optional": false, + "rest": false + }, + { + "name": "counts", + "type": { + "kind": "mapping", + "value": "float" + }, + "optional": false, + "rest": false + }, + { + "name": "options", + "type": { + "kind": "mapping", + "value": "any" + }, + "optional": false, + "rest": false + }, + { + "name": "inline", + "type": { + "kind": "mapping", + "value": "any" + }, + "optional": false, + "rest": false + }, + { + "name": "partial", + "type": { + "kind": "mapping", + "value": "any" + }, + "optional": false, + "rest": false + }, + { + "name": "keyed", + "type": { + "kind": "mapping", + "value": "any" + }, + "optional": false, + "rest": false + }, + { + "name": "nested", + "type": { + "kind": "list", + "item": { + "kind": "union", + "members": ["None", "str"] + } + }, + "optional": false, + "rest": false + } + ] + } + ], + "untyped": false + }, + { + "name": "noJsonForm", + "signatures": [ + { + "parameters": [ + { + "name": "tags", + "type": "any", + "optional": false, + "rest": false + }, + { + "name": "lookup", + "type": "any", + "optional": false, + "rest": false + }, + { + "name": "big", + "type": "any", + "optional": false, + "rest": false + }, + { + "name": "later", + "type": "any", + "optional": false, + "rest": false + }, + { + "name": "make", + "type": "any", + "optional": false, + "rest": false + }, + { + "name": "element", + "type": "any", + "optional": false, + "rest": false + }, + { + "name": "widget", + "type": "any", + "optional": false, + "rest": false + }, + { + "name": "anything", + "type": "any", + "optional": false, + "rest": false + }, + { + "name": "both", + "type": "any", + "optional": false, + "rest": false + }, + { + "name": "either", + "type": "any", + "optional": false, + "rest": false + } + ] + } + ], + "untyped": false + }, + { + "name": "maybe", + "signatures": [ + { + "parameters": [ + { + "name": "value", + "type": { + "kind": "union", + "members": ["None", "str"] + }, + "optional": true, + "rest": false + }, + { + "name": "rest", + "type": "any", + "optional": false, + "rest": true + } + ] + } + ], + "untyped": false + }, + { + "name": "Color", + "signatures": [], + "untyped": false + }, + { + "name": "Level", + "signatures": [], + "untyped": false + }, + { + "name": "Widget", + "signatures": [], + "untyped": false + }, + { + "name": "aliased", + "signatures": [ + { + "parameters": [] + } + ], + "untyped": false + }, + { + "name": "fromReexported", + "signatures": [ + { + "parameters": [] + } + ], + "untyped": false + } + ], + "script.js": [] +} diff --git a/packages/reflex-frontend/tests/fixtures/signatures/typed.ts b/packages/reflex-frontend/tests/fixtures/signatures/typed.ts new file mode 100644 index 00000000000..397cdefdf06 --- /dev/null +++ b/packages/reflex-frontend/tests/fixtures/signatures/typed.ts @@ -0,0 +1,9 @@ +export function pick(mode: "a" | "b", maybe: string | null): void {} + +export function overloaded(n: number): void; +export function overloaded(): void; +export function overloaded(n?: number): void {} + +export function tupleRest(...args: [number, string]): void {} + +export function defaultFirst(prefix = "", count: number): void {} diff --git a/packages/reflex-frontend/tests/fixtures/signatures/types.ts b/packages/reflex-frontend/tests/fixtures/signatures/types.ts new file mode 100644 index 00000000000..d02a830b0bc --- /dev/null +++ b/packages/reflex-frontend/tests/fixtures/signatures/types.ts @@ -0,0 +1,59 @@ +import type { Shape } from "./shapes"; + +export enum Color { + Red = "red", + Green = "green", +} + +export enum Level { + Low, + High, +} + +export interface Options { + size: number; +} + +export type Count = number; + +export class Widget {} + +export * from "./reexported.js"; +export { fromReexported as aliased } from "./reexported.js"; +export type { Shape } from "./shapes"; + +export function literals( + color: Color, + level: Level, + mode: "b" | "a" | 2 | true, + ratio: 1.5 | 2.5, + flag: true, + shape: Shape, +): void {} + +export function containers( + names: string[], + pair: [number, string], + empty: [], + counts: Record, + options: Options, + inline: { a: number }, + partial: Partial, + keyed: Record<"x" | "y", number>, + nested: Array, +): void {} + +export function noJsonForm( + tags: Set, + lookup: Map, + big: bigint, + later: Promise, + make: () => string[], + element: HTMLElement, + widget: Widget, + anything: object, + both: Options & { b: number }, + either: string | HTMLElement, +): void {} + +export function maybe(value?: string | null, ...rest: [number, string]): void {} diff --git a/packages/reflex-frontend/tests/script_signatures.test.js b/packages/reflex-frontend/tests/script_signatures.test.js new file mode 100644 index 00000000000..9abe9cb5cc6 --- /dev/null +++ b/packages/reflex-frontend/tests/script_signatures.test.js @@ -0,0 +1,85 @@ +// @vitest-environment node +/** + * `script-signatures.js` prints the call signatures of the functions modules + * export, which Reflex renders as Python type stubs. It is a command run in an + * app's .web, so this sets its arguments, imports it and reads what it prints. + * `fixtures/signatures/signatures.json` is that output, which the Python tests + * render, so both sides are held to the same contract. + */ +import { readFileSync } from "node:fs"; +import { basename } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test, vi } from "vitest"; + +const fixtures = fileURLToPath( + new URL("./fixtures/signatures/", import.meta.url), +); + +/** + * Run the script on fixture files. + * @param {string[]} names The fixtures' file names. + * @returns {Promise>} Its output, by file name. + */ +const signatures = async (names) => { + const argv = process.argv; + process.argv = [ + argv[0], + "script-signatures.js", + ...names.map((name) => fixtures + name), + ]; + const write = vi + .spyOn(process.stdout, "write") + .mockImplementation(() => true); + try { + vi.resetModules(); + await import("$/script-signatures.js"); + const output = JSON.parse(String(write.mock.calls[0][0])); + return Object.fromEntries( + Object.entries(output).map(([file, exports]) => [ + basename(file), + exports, + ]), + ); + } finally { + write.mockRestore(); + process.argv = argv; + } +}; + +test("classifies each parameter's type into the JSON value Python sends", async () => { + // types.ts declares `mode: "b" | "a" | 2 | true`; the union comes out sorted. + expect( + await signatures(["jsdoc.js", "typed.ts", "types.ts", "script.js"]), + ).toEqual(JSON.parse(readFileSync(`${fixtures}signatures.json`, "utf-8"))); +}); + +test("a type that contains itself stops at its first repeat", async () => { + // The repeat has no finite JSON shape, so it accepts any value. + expect(await signatures(["recursive.ts"])).toEqual({ + "recursive.ts": [ + { + name: "grow", + signatures: [ + { + parameters: [ + { + name: "tree", + type: { kind: "mapping", value: "any" }, + optional: false, + rest: false, + }, + { + name: "nested", + type: { kind: "list", item: "any" }, + optional: false, + rest: false, + }, + { name: "depth", type: "float", optional: false, rest: false }, + ], + }, + ], + untyped: false, + }, + ], + }); +}); diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index cdf2f4e052c..caaba141b1e 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1590,9 +1590,12 @@ def add_save_task( ), utils._compile_bundled_libraries(), ]) - script_module_outputs, script_module_imports, script_module_copies = ( - utils.compile_script_modules() - ) + ( + script_module_outputs, + script_module_imports, + script_module_copies, + script_module_sources, + ) = utils.compile_script_modules() compile_results.extend(script_module_outputs) progress.advance(task) @@ -1620,6 +1623,10 @@ def add_save_task( app._get_frontend_packages( utils.merge_imports(all_imports, script_module_imports) ) + # Stubs are for type checkers during development: a production build leaves + # the source tree alone. After the install: reading types needs typescript. + if not is_prod_mode(): + compile_results.extend(utils.compile_script_stubs(script_module_sources)) frontend_skeleton.update_react_router_config( prerender_routes=prerender_routes, diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index a942dc9ffd0..324d126b09a 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -9,9 +9,10 @@ import json import operator import os +import subprocess import tempfile import traceback -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Collection, Iterable, Mapping, Sequence from dataclasses import MISSING from datetime import datetime from pathlib import Path @@ -32,11 +33,22 @@ MemoParamKind, ) from reflex_base.constants.state import FIELD_MARKER -from reflex_base.js.modules import _package_imports, _render_stub +from reflex_base.js.modules import _package_imports from reflex_base.js.source import ScriptSource, module_files +from reflex_base.js.stubs import ( + _SIGNATURES_SCRIPT, + _TYPESCRIPT_PACKAGE, + Export, + _exports_by_file, + _is_generated, + _module_digest, + _read_header, + _render_stub, + _stub_header, +) from reflex_base.registry import RegistrationContext from reflex_base.style import Style -from reflex_base.utils import format, imports, memo_paths, serializers, types +from reflex_base.utils import console, format, imports, memo_paths, serializers, types from reflex_base.utils.exceptions import ReflexRuntimeError, VarTypeError from reflex_base.utils.imports import ImportVar, ParsedImportDict from reflex_base.vars.base import Field, Var, VarData @@ -67,6 +79,9 @@ merge_imports = imports.merge_imports write_file = path_ops.write_file +# How long reading the types of the app's JavaScript modules may take. +_SIGNATURES_TIMEOUT = 120 + def compile_import_statement(fields: list[ImportVar]) -> tuple[str, list[str]]: """Compile an import statement. @@ -304,7 +319,11 @@ def _compile_bundled_libraries() -> tuple[str, str]: def _script_module_files( modules_dir: Path, -) -> tuple[dict[Path, tuple[Path, ScriptSource | None]], dict[str, Path]]: +) -> tuple[ + dict[Path, tuple[Path, ScriptSource | None]], + dict[str, Path], + dict[Path, dict[Path, ScriptSource | None]], +]: """Find the files of the JavaScript modules the app imports from Python. Args: @@ -312,7 +331,8 @@ def _script_module_files( Returns: Each resolved file with its target and its script, or None if it is - not a script, and each module's file, by module name. + not a script; each module's file, by module name; and each module's + files with their scripts, by the module's file. Raises: ReflexRuntimeError: If a module imports a file outside its package's @@ -320,11 +340,13 @@ def _script_module_files( """ files: dict[Path, tuple[Path, ScriptSource | None]] = {} modules: dict[str, Path] = {} + sources: dict[Path, dict[Path, ScriptSource | None]] = {} for name, path in sorted( RegistrationContext.ensure_context()._script_modules.items() ): package_dir = path.resolve().parents[name.count(".") - 1] - for file, script in module_files(path).items(): + module_sources = module_files(path) + for file, script in module_sources.items(): if not file.is_relative_to(package_dir): msg = ( f"The JavaScript module {name} imports {file}, which is " @@ -338,11 +360,15 @@ def _script_module_files( file, (modules_dir / file.relative_to(package_dir.parent), script) ) modules[name] = path.resolve() - return files, modules + sources[modules[name]] = module_sources + return files, modules, sources def compile_script_modules() -> tuple[ - list[tuple[str, str]], dict[str, list[ImportVar]], list[tuple[Path, Path]] + list[tuple[str, str]], + dict[str, list[ImportVar]], + list[tuple[Path, Path]], + dict[Path, dict[Path, ScriptSource | None]], ]: """Compile the JavaScript modules the app imports from Python. @@ -350,16 +376,19 @@ def compile_script_modules() -> tuple[ under ``utils/script_modules`` at their paths from the directory that holds the app's package, so relative imports resolve as they do in the source tree. ``utils/scripts.js`` maps each module's Python name to a function - that loads it, and a type stub is written next to each module, for type - checkers. + that loads it. Returns: - The (path, code) outputs, the npm packages the scripts import, to - install, and the (source, target) copies of the other files they - reference, such as stylesheets and images, which may be binary. + The (path, code) outputs; the npm packages to install, which are those + the scripts import and, in development, typescript, which reads their + types for ``compile_script_stubs``; the (source, target) copies of the + other files they reference, such as stylesheets and images, which may + be binary; and each module's files, by the module's file. """ web_dir = get_web_dir() - files, modules = _script_module_files(web_dir / constants.Dirs.SCRIPT_MODULES_PATH) + files, modules, sources = _script_module_files( + web_dir / constants.Dirs.SCRIPT_MODULES_PATH + ) outputs: list[tuple[str, str]] = [] copies: list[tuple[Path, Path]] = [] package_imports: dict[str, list[ImportVar]] = {} @@ -372,19 +401,17 @@ def compile_script_modules() -> tuple[ (package, [ImportVar(tag=None, render=False)]) for package in _package_imports(file, script.imports) ) + if modules and not is_prod_mode(): + package_imports[_TYPESCRIPT_PACKAGE] = [ImportVar(tag=None, render=False)] loaders = "" for name, path in modules.items(): - target, script = files[path] - # The module itself is a .js or .ts file, so it is a script. - functions = cast("ScriptSource", script).functions - outputs.append((str(path.with_suffix(".pyi")), _render_stub(path, functions))) - specifier = json.dumps(f"$/{target.relative_to(web_dir).as_posix()}") + specifier = json.dumps(f"$/{files[path][0].relative_to(web_dir).as_posix()}") loaders += f" {json.dumps(name)}: () => import({specifier}),\n" outputs.append(( str(web_dir / (constants.Dirs.SCRIPTS_PATH + constants.Ext.JS)), f"export default {{\n{loaders}}};\n" if loaders else "export default {};\n", )) - return outputs, package_imports, copies + return outputs, package_imports, copies, sources def copy_files(copies: Iterable[tuple[Path, Path]]) -> None: @@ -401,6 +428,108 @@ def copy_files(copies: Iterable[tuple[Path, Path]]) -> None: target.write_bytes(data) +def _read_script_signatures( + paths: Collection[Path], +) -> dict[Path, dict[str, Export]] | None: + """Read the signatures of the functions JavaScript modules export. + + Runs ``script-signatures.js`` in ``.web``, where typescript is installed. + + Args: + paths: The modules. + + Returns: + The exports of each module by resolved path, then by name, or None if + they could not be read, which this warns about. + """ + web_dir = get_web_dir() + runtime = path_ops.get_node_path() or path_ops.get_bun_path() + if runtime is None: + cause = "reading them needs Node.js or Bun, and neither was found" + fix = ( + f"Install Node.js {constants.Node.MIN_VERSION} or newer, then run " + "the app again." + ) + else: + try: + process = subprocess.run( + [str(runtime), _SIGNATURES_SCRIPT, *map(str, paths)], + cwd=web_dir, + capture_output=True, + encoding="utf-8", + timeout=_SIGNATURES_TIMEOUT, + check=True, + ) + return _exports_by_file(json.loads(process.stdout)) + except subprocess.CalledProcessError as error: + detail = error.stderr.strip() + except (OSError, subprocess.TimeoutExpired, ValueError) as error: + detail = str(error) + cause = f"{_SIGNATURES_SCRIPT} failed: {detail}" + fix = ( + f"If this repeats, delete {web_dir / 'node_modules'} so that the " + "next run reinstalls typescript, or report the error at " + "https://github.com/reflex-dev/reflex/issues." + ) + console.warn( + "Reflex could not read the argument types of the JavaScript modules " + f"{', '.join(map(str, paths))}, so their type stubs accept any " + f"arguments: {cause}. The app still runs. {fix}" + ) + return None + + +def compile_script_stubs( + sources: Mapping[Path, Mapping[Path, ScriptSource | None]], +) -> list[tuple[str, str]]: + """Write a type stub next to each JavaScript module the app imports. + + Argument types come from the module's TypeScript annotations or JSDoc. A + stub whose header records the digest of what it is made from is kept, so + typescript only runs for a module when that changed; when the types cannot + be read, the stub has untyped arguments and is retried on the next compile. + A stub Reflex wrote for a module the app no longer imports is removed. + + Args: + sources: Each module's files, by the module's file, from + ``compile_script_modules``. + + Returns: + The (path, code) outputs: the stubs to rewrite, and the list of stubs, + so the next compile knows which to remove. + """ + web_dir = get_web_dir() + # A package upgrade can change the types a module uses. + dependencies = ( + (web_dir / constants.PackageJson.PATH).read_bytes() if sources else b"" + ) + stale: dict[Path, str] = {} + for path, files in sources.items(): + digest = _module_digest(path, files, dependencies) + if _read_header(path.with_suffix(".pyi")) != _stub_header(path, digest): + stale[path] = digest + stubs = [str(path.with_suffix(".pyi")) for path in sources] + record = web_dir / constants.Dirs.SCRIPT_STUBS + if record.is_file(): + for stub in set(json.loads(record.read_text(encoding="utf-8"))) - set(stubs): + if _is_generated(_read_header(Path(stub))): + Path(stub).unlink() + outputs = [(str(record), json.dumps(stubs))] + if not stale: + return outputs + signatures = _read_script_signatures(stale) + for path, digest in stale.items(): + # The module itself is a .js or .ts file, so it is a script. + functions = cast("ScriptSource", sources[path][path]).functions + if signatures is None: + # Never current, so the next compile reads the types again. + stub = _render_stub(path, "untyped", functions, None) + else: + stub = _render_stub(path, digest, functions, signatures[path]) + outputs.append((str(path.with_suffix(".pyi")), stub)) + return outputs + + def _restore_bundled_libraries() -> bool: """Restore the registry saved by the most recent compile. diff --git a/reflex/utils/exec.py b/reflex/utils/exec.py index 6a5603b234d..ebe5da2a4f4 100644 --- a/reflex/utils/exec.py +++ b/reflex/utils/exec.py @@ -781,6 +781,8 @@ def run_uvicorn_backend(host: str, port: int, loglevel: LogLevel): "bash", "log", "db", + # The type stubs Reflex writes next to JavaScript modules. + "pyi", ) HOTRELOAD_IGNORE_PATTERNS = ( diff --git a/tests/integration/tests_playwright/test_call_script.py b/tests/integration/tests_playwright/test_call_script.py index c8bf5d3730f..05a8f182911 100644 --- a/tests/integration/tests_playwright/test_call_script.py +++ b/tests/integration/tests_playwright/test_call_script.py @@ -2,7 +2,10 @@ from __future__ import annotations +import re +import sys from collections.abc import Generator +from pathlib import Path import pytest from playwright.sync_api import Page, expect @@ -44,9 +47,11 @@ def CallScript(): # A JavaScript module beside the app, imported from Python. Path(__file__).with_name("page_script.js").write_text( 'import { reflex } from "$/utils/reflex";\n' + "/** @param {string} id @param {string} text */\n" "export function setText(id, text) {\n" " document.getElementById(id).textContent = text;\n" "}\n" + "/** @param {string} handler @param {string} value */\n" "export async function report(handler, value) {\n" " await Promise.resolve();\n" " reflex.addEvents([reflex.event(handler, { value })]);\n" @@ -709,3 +714,35 @@ def test_js_module(call_script_page: Page): expect(call_script_page.locator("#reflex_result")).to_have_value( "written by the module" ) + + +def test_js_module_stub(call_script: AppHarness): + """A typed stub is written next to the module from its JSDoc. + + Args: + call_script: harness for CallScript app. + """ + # The module Python imported: a harness that reloads an app package of the + # same name keeps the package's first directory. + spec = sys.modules[f"{call_script.app_name}.page_script"].__spec__ + assert spec is not None + assert spec.origin is not None + module = Path(spec.origin) + handler = "_str | _Var[_str] | _EventCallback[*_tuple[_Any, ...]]" + # The digest depends on the app's package.json; the rest is exact. + stub = re.sub( + r"^(# Generated by Reflex from page_script\.js \()[0-9a-f]{16}(\))", + r"\1\2", + module.with_suffix(".pyi").read_text(), + ) + assert stub == ( + "# Generated by Reflex from page_script.js (); rewritten when it changes.\n" + "from builtins import str as _str, tuple as _tuple\n" + "from typing import Any as _Any\n" + "\n" + "from reflex_base.event import EventCallback as _EventCallback, EventSpec as _EventSpec\n" + "from reflex_base.vars.base import Var as _Var\n" + "\n" + f"def setText(id: {handler}, text: {handler}, /) -> _EventSpec: ...\n" + f"def report(handler: {handler}, value: {handler}, /) -> _EventSpec: ...\n" + ) diff --git a/tests/units/compiler/test_compiler.py b/tests/units/compiler/test_compiler.py index 0f42e0ea870..83248628278 100644 --- a/tests/units/compiler/test_compiler.py +++ b/tests/units/compiler/test_compiler.py @@ -16,6 +16,7 @@ from reflex_base.constants.base import LiteralColorMode from reflex_base.constants.compiler import PageNames from reflex_base.environment import environment +from reflex_base.js.stubs import _TYPESCRIPT_PACKAGE from reflex_base.plugins import CompileContext, CompilerHooks, PageContext from reflex_base.plugins.sitemap import SitemapPlugin from reflex_base.registry import RegistrationContext @@ -83,6 +84,60 @@ def __init__(self): assert task["current"] == task["total"] +@pytest.mark.parametrize("prod", [False, True]) +def test_compile_writes_stubs_only_in_development( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mocker: MockerFixture, + clean_registration_context: RegistrationContext, + prod: bool, +): + """A production build installs no typescript and leaves the source tree alone. + + Args: + tmp_path: The temporary application directory. + monkeypatch: Fixture for selecting the application directory and mode. + mocker: Fixture for avoiding frontend dependency installation. + clean_registration_context: A fresh registry of imported modules. + prod: Whether to compile in production mode. + """ + monkeypatch.chdir(tmp_path) + if prod: + monkeypatch.setenv("REFLEX_ENV_MODE", constants.Env.PROD.value) + monkeypatch.setattr(utils.path_ops, "get_node_path", lambda: None) + monkeypatch.setattr(utils.path_ops, "get_bun_path", lambda: None) + monkeypatch.setattr(utils.console, "warn", lambda message: None) + mocker.patch( + "reflex_base.config._get_config", + return_value=rx.Config(app_name="stub_test", plugins=[]), + ) + party = tmp_path / "stub_test" / "party.js" + party.parent.mkdir() + party.write_text("export function go(n) {}\n") + clean_registration_context._script_modules["stub_test.party"] = party + app = rx.App(enable_state=False) + app.add_page(lambda: rx.el.div("hello"), route="index") + install = mocker.patch.object(app, "_get_frontend_packages") + web_dir = prerequisites.get_web_dir() + web_dir.mkdir() + (web_dir / "package.json").write_text("{}") + + compiler.compile_app(app, use_rich=False) + + (installed,) = install.call_args.args + stub = party.with_suffix(".pyi") + assert (_TYPESCRIPT_PACKAGE in installed, stub.is_file()) == (not prod, not prod) + if not prod: + assert stub.read_text() == ( + "# Generated by Reflex from party.js (untyped); rewritten when it changes.\n" + "from typing import Any as _Any\n" + "\n" + "from reflex_base.event import EventSpec as _EventSpec\n" + "\n" + "def go(*args: _Any) -> _EventSpec: ...\n" + ) + + @pytest.mark.parametrize("remove_assets_directory", [False, True]) @pytest.mark.usefixtures("clean_registration_context") def test_compile_removes_deleted_app_assets( diff --git a/tests/units/compiler/test_compiler_utils.py b/tests/units/compiler/test_compiler_utils.py index 7d49d6c5386..da8ecf83e9f 100644 --- a/tests/units/compiler/test_compiler_utils.py +++ b/tests/units/compiler/test_compiler_utils.py @@ -4,6 +4,7 @@ import json import os import shutil +import sys from copy import deepcopy from pathlib import Path from typing import Annotated, Literal, TypedDict @@ -11,6 +12,8 @@ import pytest from reflex_base import constants from reflex_base.components.memo import create_passthrough_component_memo +from reflex_base.js.source import module_files, read_script +from reflex_base.js.stubs import _TYPESCRIPT_PACKAGE, _module_digest from reflex_base.registry import RegistrationContext from reflex_base.utils.exceptions import ReflexRuntimeError from reflex_components_core.base.fragment import Fragment @@ -863,29 +866,15 @@ def test_compile_script_modules( 'app.my-"widget"': widget, }) - outputs, package_imports, copies = utils.compile_script_modules() + outputs, package_imports, copies, sources = utils.compile_script_modules() modules_dir = web_dir / "utils" / "script_modules" / "app" - - def stub(name: str, function: str) -> str: - return ( - f"# Generated by Reflex from {name}; rewritten when it changes.\n" - "from typing import Any\n" - "\n" - "from reflex_base.event import EventSpec\n" - "\n" - f"def {function}(*args: Any) -> EventSpec: ...\n" - ) - assert outputs == [ (str(modules_dir / 'my-"widget".js'), widget.read_text()), (str(modules_dir / "scripts" / "party.js"), party.read_text()), (str(modules_dir / "scripts" / "helpers.js"), helpers.read_text()), (str(modules_dir / "scripts" / "formatters" / "count.ts"), count.read_text()), (str(modules_dir / "typed.ts"), typed.read_text()), - (str(tmp_path / "app" / 'my-"widget".pyi'), stub('my-"widget".js', "show")), - (str(scripts / "party.pyi"), stub("party.js", "celebrate")), - (str(tmp_path / "app" / "typed.pyi"), stub("typed.ts", "typed")), ( str(web_dir / "utils" / "scripts.js"), ( @@ -900,6 +889,7 @@ def stub(name: str, function: str) -> str: assert package_imports == { "canvas-confetti": [rx.ImportVar(tag=None, render=False)], "dayjs": [rx.ImportVar(tag=None, render=False)], + _TYPESCRIPT_PACKAGE: [rx.ImportVar(tag=None, render=False)], } # Stylesheets and binary files are copied byte for byte, with what they reference. assert copies == [ @@ -911,6 +901,24 @@ def stub(name: str, function: str) -> str: utils.copy_files(copies) assert (modules_dir / "scripts" / "logo.png").read_bytes() == logo.read_bytes() assert (modules_dir / "scripts" / "bg.png").read_bytes() == background.read_bytes() + # Each module's files, for its stub, with each script read once. + assert {path: list(files) for path, files in sources.items()} == { + widget.resolve(): [widget.resolve()], + party.resolve(): [ + party.resolve(), + helpers.resolve(), + css.resolve(), + logo.resolve(), + theme.resolve(), + background.resolve(), + count.resolve(), + ], + typed.resolve(): [typed.resolve()], + } + assert ( + sources[party.resolve()][helpers.resolve()], + sources[party.resolve()][css.resolve()], + ) == (read_script(helpers), None) def test_compile_script_modules_rejects_files_outside_the_package( @@ -962,6 +970,267 @@ def test_compile_script_modules_without_modules( [(str(tmp_path / "utils" / "scripts.js"), "export default {};\n")], {}, [], + {}, + ) + + +@pytest.fixture +def stub_app( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + clean_registration_context: RegistrationContext, +) -> Path: + """An app whose `.web` reads types with Python running a fake signatures script. + + The script is Python code under the JavaScript script's name, so the real + subprocess call runs it on every platform. + + Args: + tmp_path: pytest tmp_path fixture. + monkeypatch: pytest monkeypatch fixture. + clean_registration_context: a fresh registry of imported modules. + + Returns: + The app's package directory, holding ``party.js``. + """ + web_dir = tmp_path / ".web" + web_dir.mkdir() + (web_dir / "package.json").write_text('{"dependencies": {"a": "1"}}') + monkeypatch.setattr(utils, "get_web_dir", lambda: web_dir) + monkeypatch.setattr(utils.path_ops, "get_node_path", lambda: Path(sys.executable)) + package = tmp_path / "app" + package.mkdir() + party = package / "party.js" + party.write_text("/** @param {number} n */\nexport function go(n) {}\n") + clean_registration_context._script_modules["app.party"] = party + return package + + +def _fake_signatures(web_dir: Path, body: str): + """Make the fake signatures script run some Python, logging each run. + + Args: + web_dir: The app's ``.web``. + body: The Python code, which can use ``sys``. + """ + (web_dir / "script-signatures.js").write_text( + "import sys\n" + "with open(sys.argv[0] + '.log', 'a') as log:\n" + " log.write(' '.join(sys.argv[1:]) + '\\n')\n" + body + ) + + +def _compile_stubs() -> list[tuple[str, str]]: + """Compile the app's modules and their stubs, writing the outputs to disk. + + Returns: + The outputs of ``compile_script_stubs``. + """ + sources = utils.compile_script_modules()[3] + outputs = utils.compile_script_stubs(sources) + for path, code in outputs: + Path(path).write_text(code) + return outputs + + +def test_compile_script_stubs(stub_app: Path): + """A stub is typed from the module's signatures, and kept while nothing changed. + + Args: + stub_app: the app's package directory. + """ + web_dir = utils.get_web_dir() + party = stub_app / "party.js" + exports = [ + { + "name": "go", + "signatures": [ + { + "parameters": [ + {"name": "n", "type": "float", "optional": False, "rest": False} + ] + } + ], + "untyped": False, + } + ] + _fake_signatures( + web_dir, f"print({json.dumps({str(party.resolve()): exports})!r})\n" + ) + record = ( + str(web_dir / "script_stubs.json"), + json.dumps([str(stub_app / "party.pyi")]), + ) + + def stub() -> str: + digest = _module_digest( + party.resolve(), + module_files(party), + (web_dir / "package.json").read_bytes(), + ) + return ( + f"# Generated by Reflex from party.js ({digest}); rewritten when it changes.\n" + "from builtins import float as _float\n" + "\n" + "from reflex_base.event import EventSpec as _EventSpec\n" + "from reflex_base.vars.base import Var as _Var\n" + "\n" + "def go(n: _float | _Var[_float], /) -> _EventSpec: ...\n" + ) + + assert _compile_stubs() == [record, (str(stub_app / "party.pyi"), stub())] + # Unchanged sources keep their stub without reading the types again. + assert _compile_stubs() == [record] + # A package upgrade can change the types: they are read again. + (web_dir / "package.json").write_text('{"dependencies": {"a": "2"}}') + assert _compile_stubs() == [record, (str(stub_app / "party.pyi"), stub())] + assert (web_dir / "script-signatures.js.log").read_text() == ( + f"{party.resolve()}\n{party.resolve()}\n" + ) + + +def test_compile_script_stubs_removes_stubs_of_modules_no_longer_imported( + stub_app: Path, clean_registration_context: RegistrationContext +): + """Only stubs Reflex wrote for a module the app stopped importing are removed. + + Args: + stub_app: the app's package directory. + clean_registration_context: the registry of imported modules. + """ + web_dir = utils.get_web_dir() + party = (stub_app / "party.js").resolve() + _fake_signatures(web_dir, f"print({json.dumps({str(party): []})!r})\n") + party_stub = stub_app / "party.pyi" + handwritten = stub_app / "handwritten.pyi" + handwritten.write_text("def go() -> None: ...\n") + (web_dir / "script_stubs.json").write_text( + json.dumps([str(handwritten), str(stub_app / "gone.pyi")]) + ) + _compile_stubs() + digest = _module_digest( + party, module_files(party), (web_dir / "package.json").read_bytes() + ) + assert party_stub.read_text() == ( + f"# Generated by Reflex from party.js ({digest}); rewritten when it changes.\n" + ) + + clean_registration_context._script_modules.clear() + assert _compile_stubs() == [(str(web_dir / "script_stubs.json"), "[]")] + assert (party_stub.exists(), handwritten.read_text()) == ( + False, + "def go() -> None: ...\n", + ) + + +@pytest.mark.parametrize( + ("body", "cause"), + [ + ( + "sys.stderr.write(\"Error: Cannot find module 'typescript'\\n\")\nsys.exit(1)\n", + "script-signatures.js failed: Error: Cannot find module 'typescript'", + ), + ( + "print('not JSON')\n", + "script-signatures.js failed: Expecting value: line 1 column 1 (char 0)", + ), + ], +) +def test_compile_script_stubs_warns_when_the_types_cannot_be_read( + stub_app: Path, monkeypatch: pytest.MonkeyPatch, body: str, cause: str +): + """The stub accepts any arguments, the warning says why, and the next compile retries. + + Args: + stub_app: the app's package directory. + monkeypatch: pytest monkeypatch fixture. + body: the fake signatures script's code. + cause: the cause the warning gives. + """ + web_dir = utils.get_web_dir() + _fake_signatures(web_dir, body) + warnings: list[str] = [] + monkeypatch.setattr(utils.console, "warn", warnings.append) + party = stub_app / "party.js" + untyped = ( + "# Generated by Reflex from party.js (untyped); rewritten when it changes.\n" + "from typing import Any as _Any\n" + "\n" + "from reflex_base.event import EventSpec as _EventSpec\n" + "\n" + "def go(*args: _Any) -> _EventSpec: ...\n" + ) + assert _compile_stubs()[1:] == [(str(stub_app / "party.pyi"), untyped)] + assert _compile_stubs()[1:] == [(str(stub_app / "party.pyi"), untyped)] + warning = ( + "Reflex could not read the argument types of the JavaScript modules " + f"{party.resolve()}, so their type stubs accept any arguments: {cause}. " + f"The app still runs. If this repeats, delete {web_dir / 'node_modules'} " + "so that the next run reinstalls typescript, or report the error at " + "https://github.com/reflex-dev/reflex/issues." + ) + assert warnings == [warning, warning] + + +def test_compile_script_stubs_warns_when_reading_times_out( + stub_app: Path, monkeypatch: pytest.MonkeyPatch +): + """A signatures script that runs too long is stopped, and the stub is untyped. + + Args: + stub_app: the app's package directory. + monkeypatch: pytest monkeypatch fixture. + """ + web_dir = utils.get_web_dir() + _fake_signatures(web_dir, "import time\ntime.sleep(30)\n") + monkeypatch.setattr(utils, "_SIGNATURES_TIMEOUT", 0.5) + warnings: list[str] = [] + monkeypatch.setattr(utils.console, "warn", warnings.append) + party = stub_app / "party.js" + _compile_stubs() + command = [sys.executable, "script-signatures.js", str(party.resolve())] + assert warnings == [ + ( + "Reflex could not read the argument types of the JavaScript modules " + f"{party.resolve()}, so their type stubs accept any arguments: " + f"script-signatures.js failed: Command '{command}' timed out after 0.5 " + "seconds. The app still runs. If this repeats, delete " + f"{web_dir / 'node_modules'} so that the next run reinstalls typescript, " + "or report the error at https://github.com/reflex-dev/reflex/issues." + ) + ] + + +def test_compile_script_stubs_warns_without_node_or_bun( + stub_app: Path, monkeypatch: pytest.MonkeyPatch +): + """Without a JavaScript runtime the stub is untyped, and the warning says what to install. + + Args: + stub_app: the app's package directory. + monkeypatch: pytest monkeypatch fixture. + """ + monkeypatch.setattr(utils.path_ops, "get_node_path", lambda: None) + monkeypatch.setattr(utils.path_ops, "get_bun_path", lambda: None) + warnings: list[str] = [] + monkeypatch.setattr(utils.console, "warn", warnings.append) + _compile_stubs() + assert warnings == [ + ( + "Reflex could not read the argument types of the JavaScript modules " + f"{(stub_app / 'party.js').resolve()}, so their type stubs accept any " + "arguments: reading them needs Node.js or Bun, and neither was found. " + f"The app still runs. Install Node.js {constants.Node.MIN_VERSION} or " + "newer, then run the app again." + ) + ] + assert (stub_app / "party.pyi").read_text() == ( + "# Generated by Reflex from party.js (untyped); rewritten when it changes.\n" + "from typing import Any as _Any\n" + "\n" + "from reflex_base.event import EventSpec as _EventSpec\n" + "\n" + "def go(*args: _Any) -> _EventSpec: ...\n" ) diff --git a/tests/units/reflex_base/js/test_modules.py b/tests/units/reflex_base/js/test_modules.py index f706a043f99..3b105fafbd9 100644 --- a/tests/units/reflex_base/js/test_modules.py +++ b/tests/units/reflex_base/js/test_modules.py @@ -4,7 +4,7 @@ from pathlib import Path import pytest -from reflex_base.js.modules import _JsFunction, _package_imports, _render_stub +from reflex_base.js.modules import _JsFunction, _package_imports from reflex_base.utils.exceptions import ReflexRuntimeError from reflex_base.vars.base import LiteralVar, Var @@ -91,18 +91,6 @@ def test_var_arguments_are_compiled_into_the_event(): ) -def test_render_stub_skips_names_python_cannot_import(): - """The stub leaves out functions whose names are not Python identifiers.""" - assert _render_stub(Path("party.js"), ["celebrate", "$private", "class"]) == ( - "# Generated by Reflex from party.js; rewritten when it changes.\n" - "from typing import Any\n" - "\n" - "from reflex_base.event import EventSpec\n" - "\n" - "def celebrate(*args: Any) -> EventSpec: ...\n" - ) - - def test_a_function_is_an_event_trigger(app_package: Path): """A function works in an event trigger, called or uncalled. diff --git a/tests/units/reflex_base/js/test_source.py b/tests/units/reflex_base/js/test_source.py index 4574b579061..a893d6a6956 100644 --- a/tests/units/reflex_base/js/test_source.py +++ b/tests/units/reflex_base/js/test_source.py @@ -8,6 +8,7 @@ module_files, parse_script, read_script, + type_files, ) SOURCE = r""" @@ -285,3 +286,64 @@ def test_find_stylesheet_references(): "fonts.css", "./bg.png", ] + + +def test_parse_script_type_imports(): + """Local files read only for types: type-only imports, JSDoc and triple-slash references. + + A runtime import is not one, nor is a package's types. + """ + script = parse_script( + '/// \n' + "/// \n" + 'import type { A } from "./a";\n' + 'export type { B } from "./b.js";\n' + 'import type { P } from "party-types";\n' + 'import { run } from "./run";\n' + '/** @param {import("./c").C} c @param {import("pkg").T} t */\n' + '/** @returns {import("./run").R} */\n' + "export function go(c, t) {}\n" + ) + assert (script.imports, script.type_imports) == ( + ["./run"], + ["./a", "./b.js", "globals.d.ts", "./more.d.ts", "./c"], + ) + + +def test_type_files(tmp_path: Path): + """Files a module reads types from are found transitively, beside its imports or not. + + Files it loads, files that do not exist and files that are not scripts are + left out. + + Args: + tmp_path: pytest tmp_path fixture. + """ + module = tmp_path / "party.js" + module.write_text( + 'import { help } from "./helpers.js";\n' + 'import confetti from "canvas-confetti";\n' + 'import type { Count } from "./types";\n' + 'import type { Shape } from "./shapes";\n' + 'import type { Data } from "./data.json";\n' + 'import type { Gone } from "./gone";\n' + ) + (tmp_path / "helpers.js").write_text("export const help = 1;\n") + (tmp_path / "helpers.d.ts").write_text('import { Unit } from "./units";\n') + (tmp_path / "units.ts").write_text("export type Unit = string;\n") + (tmp_path / "types.d.ts").write_text("export type Count = number;\n") + (tmp_path / "shapes").mkdir() + (tmp_path / "shapes" / "index.d.ts").write_text("export type Shape = string;\n") + (tmp_path / "data.json").write_text("{}\n") + files = module_files(module) + found = type_files(files) + assert list(files) == [module.resolve(), (tmp_path / "helpers.js").resolve()] + assert found == { + file.resolve(): read_script(file) + for file in [ + tmp_path / "helpers.d.ts", + tmp_path / "types.d.ts", + tmp_path / "shapes" / "index.d.ts", + tmp_path / "units.ts", + ] + } diff --git a/tests/units/reflex_base/js/test_stubs.py b/tests/units/reflex_base/js/test_stubs.py new file mode 100644 index 00000000000..d7855e4a07e --- /dev/null +++ b/tests/units/reflex_base/js/test_stubs.py @@ -0,0 +1,379 @@ +"""Tests for the type stubs of JavaScript modules imported from Python.""" + +import hashlib +import json +from pathlib import Path + +import pytest +from reflex_base import constants +from reflex_base.js import stubs +from reflex_base.js.source import module_files, read_script +from reflex_base.js.stubs import ( + _TYPESCRIPT_PACKAGE, + Export, + JsonType, + Parameter, + _exports_by_file, + _module_digest, + _render_stub, + _render_type, + _renderer_digest, +) + +FRONTEND = constants.Templates.Dirs.WEB_TEMPLATE.parent +# What script-signatures.js prints for the fixtures, which its vitest checks. +SIGNATURE_FIXTURES = FRONTEND / "tests" / "fixtures" / "signatures" + +HANDLER = "_EventCallback[*_tuple[_Any, ...]]" +STR = f"_str | _Var[_str] | {HANDLER}" +FLOAT = "_float | _Var[_float]" +# The imports of a stub with string parameters. +IMPORTS = ( + "from builtins import str as _str, tuple as _tuple\n" + "from typing import Any as _Any\n" + "\n" + "from reflex_base.event import EventCallback as _EventCallback, EventSpec as _EventSpec\n" + "from reflex_base.vars.base import Var as _Var\n" +) + + +def _fixture_stub(name: str) -> str: + """Render a fixture module's stub from the real output of script-signatures.js. + + Args: + name: The fixture's file name. + + Returns: + The stub, with the digest ``abc``. + """ + output = json.loads((SIGNATURE_FIXTURES / "signatures.json").read_text()) + module = SIGNATURE_FIXTURES / name + exports = _exports_by_file({str(module): output[name]})[module.resolve()] + return _render_stub(Path(name), "abc", read_script(module).functions, exports) + + +def test_render_stub_from_jsdoc(): + """JSDoc types, an optional and a rest parameter; untyped arguments are Any.""" + assert _fixture_stub("jsdoc.js") == ( + "# Generated by Reflex from jsdoc.js (abc); rewritten when it changes.\n" + "from builtins import bool as _bool, float as _float, str as _str, tuple as _tuple\n" + "from typing import Any as _Any\n" + "\n" + "from reflex_base.event import EventCallback as _EventCallback, EventSpec as _EventSpec\n" + "from reflex_base.vars.base import Var as _Var\n" + "\n" + f"def celebrate(count: {FLOAT}, elementId: {STR}, " + "loud: _bool | _Var[_bool] = ..., /) -> _EventSpec: ...\n" + f"def many(*names: {STR}) -> _EventSpec: ...\n" + "def untyped(a: _Any, b: _Any, /) -> _EventSpec: ...\n" + "def anything(*args: _Any) -> _EventSpec: ...\n" + ) + + +def test_render_stub_from_typescript(): + """String literals, null, overloads, a tuple rest and a default before a required parameter.""" + literal = '_Literal["a", "b"]' + assert _fixture_stub("typed.ts") == ( + "# Generated by Reflex from typed.ts (abc); rewritten when it changes.\n" + "from builtins import float as _float, str as _str, tuple as _tuple\n" + "from typing import Any as _Any, Literal as _Literal, overload as _overload\n" + "\n" + "from reflex_base.event import EventCallback as _EventCallback, EventSpec as _EventSpec\n" + "from reflex_base.vars.base import Var as _Var\n" + "\n" + f"def pick(mode: {literal} | _Var[{literal}], " + f"maybe: _str | None | _Var[_str | None] | {HANDLER}, /) -> _EventSpec: ...\n" + "@_overload\n" + f"def overloaded(n: {FLOAT}, /) -> _EventSpec: ...\n" + "@_overload\n" + "def overloaded() -> _EventSpec: ...\n" + "def tupleRest(*args: _Any) -> _EventSpec: ...\n" + f"def defaultFirst(prefix: {STR}, count: {FLOAT}, /) -> _EventSpec: ...\n" + ) + + +def test_render_stub_maps_types_without_json_form_to_any(): + """Enums are their values; containers keep item types; Set, Map, bigint and the like are Any. + + Types, classes, enums and ``export *`` names are not functions the module + exports, so the stub declares none of them; a function re-exported by + name is one. + """ + + def var(value: str) -> str: + return f"{value} | _Var[{value}]" + + color = var('_Literal["green", "red"]') + mode = var('_Literal["a", "b", 2, True]') + shape = var('_Literal["circle", "square"]') + + assert _fixture_stub("types.ts") == ( + "# Generated by Reflex from types.ts (abc); rewritten when it changes.\n" + "from builtins import float as _float, str as _str, tuple as _tuple\n" + "from collections.abc import Mapping as _Mapping, Sequence as _Sequence\n" + "from typing import Any as _Any, Literal as _Literal\n" + "\n" + "from reflex_base.event import EventCallback as _EventCallback, EventSpec as _EventSpec\n" + "from reflex_base.vars.base import Var as _Var\n" + "\n" + "def aliased() -> _EventSpec: ...\n" + "def literals(" + f"color: {color}, " + f"level: {var('_Literal[0, 1]')}, " + f"mode: {mode}, " + f"ratio: {FLOAT}, " + f"flag: {var('_Literal[True]')}, " + f"shape: {shape}, /) -> _EventSpec: ...\n" + "def containers(" + f"names: {var('_Sequence[_str]')}, " + f"pair: {var('_Sequence[_float | _str]')}, " + f"empty: {var('_Sequence[_Any]')}, " + f"counts: {var('_Mapping[_str, _float]')}, " + f"options: {var('_Mapping[_str, _Any]')}, " + f"inline: {var('_Mapping[_str, _Any]')}, " + f"partial: {var('_Mapping[_str, _Any]')}, " + f"keyed: {var('_Mapping[_str, _Any]')}, " + f"nested: {var('_Sequence[_str | None]')}, /) -> _EventSpec: ...\n" + "def noJsonForm(tags: _Any, lookup: _Any, big: _Any, later: _Any, " + "make: _Any, element: _Any, widget: _Any, anything: _Any, both: _Any, " + "either: _Any, /) " + "-> _EventSpec: ...\n" + f"def maybe(value: _str | None | _Var[_str | None] | {HANDLER} = ..., /, " + "*rest: _Any) -> _EventSpec: ...\n" + ) + + +@pytest.mark.parametrize( + ("json_type", "rendered", "used"), + [ + ("None", "None", set()), + ( + {"kind": "literal", "value": 'say "hi" 🎉'}, + '_Literal["say \\"hi\\" 🎉"]', + {"Literal"}, + ), + ( + { + "kind": "union", + "members": [ + "None", + "str", + {"kind": "list", "item": "bool"}, + {"kind": "literal", "value": -1}, + {"kind": "literal", "value": False}, + ], + }, + "_str | _Sequence[_bool] | _Literal[-1, False] | None", + {"str", "Sequence", "bool", "Literal"}, + ), + ], +) +def test_render_type(json_type: JsonType, rendered: str, used: set[str]): + """A union's literals merge into one Literal before None; string literals are escaped. + + Args: + json_type: The JSON type. + rendered: Its Python type. + used: The names it makes the stub import. + """ + names: set[str] = set() + assert (_render_type(json_type, names), names) == (rendered, used) + + +def _param(name: str, type_: JsonType, optional: bool = False) -> Parameter: + """Build a parameter as script-signatures.js prints it. + + Args: + name: Its name. + type_: Its JSON type. + optional: Whether it is optional. + + Returns: + The parameter. + """ + return {"name": name, "type": type_, "optional": optional, "rest": False} + + +def _export(name: str, *parameters: Parameter) -> Export: + """Build an export with one signature, as script-signatures.js prints it. + + Args: + name: Its name. + *parameters: The parameters of its signature. + + Returns: + The export. + """ + return { + "name": name, + "signatures": [{"parameters": list(parameters)}], + "untyped": False, + } + + +def test_render_stub_renames_parameters_python_cannot_name(): + """A parameter renamed from a non-identifier never takes another's name.""" + send = _export( + "send", _param("$value", "str"), _param("arg0", "str"), _param("arg0_", "str") + ) + assert _render_stub(Path("send.js"), "abc", ["send"], {"send": send}) == ( + "# Generated by Reflex from send.js (abc); rewritten when it changes.\n" + + IMPORTS + + "\n" + f"def send(arg0__: {STR}, arg0: {STR}, arg0_: {STR}, /) -> _EventSpec: ...\n" + ) + + +def test_render_stub_only_trailing_optional_parameters_have_defaults(): + """A default that a required parameter follows cannot be omitted in Python.""" + send = _export( + "send", + _param("prefix", "str", optional=True), + _param("count", "float"), + _param("suffix", "str", optional=True), + ) + assert _render_stub(Path("send.ts"), "abc", ["send"], {"send": send}) == ( + "# Generated by Reflex from send.ts (abc); rewritten when it changes.\n" + "from builtins import float as _float, str as _str, tuple as _tuple\n" + "from typing import Any as _Any\n" + "\n" + "from reflex_base.event import EventCallback as _EventCallback, EventSpec as _EventSpec\n" + "from reflex_base.vars.base import Var as _Var\n" + "\n" + f"def send(prefix: {STR}, count: {FLOAT}, suffix: {STR} = ..., /) -> _EventSpec: ...\n" + ) + + +def test_render_stub_declares_only_importable_functions(): + """Python cannot import a keyword, a non-identifier or a name the stub uses for a type. + + An export named like a builtin type, such as ``str``, keeps the stub's + annotations intact, since those use private aliases. A name TypeScript + does not list is only a type, and a name with no signature is no function. + """ + exports = { + "str": _export("str", _param("value", "str")), + "class": _export("class"), + "_str": _export("_str"), + "constant": {"name": "constant", "signatures": [], "untyped": False}, + } + functions = ["str", "class", "$private", "_str", "OnlyAType", "constant"] + assert _render_stub(Path("m.js"), "abc", functions, exports) == ( + "# Generated by Reflex from m.js (abc); rewritten when it changes.\n" + + IMPORTS + + "\n" + f"def str(value: {STR}, /) -> _EventSpec: ...\n" + ) + + +def test_render_stub_without_types(): + """When the types could not be read every function takes any arguments. + + A module with no function to declare gets a stub with only its header. + """ + assert _render_stub(Path("party.ts"), "untyped", ["go", "class"], None) == ( + "# Generated by Reflex from party.ts (untyped); rewritten when it changes.\n" + "from typing import Any as _Any\n" + "\n" + "from reflex_base.event import EventSpec as _EventSpec\n" + "\n" + "def go(*args: _Any) -> _EventSpec: ...\n" + ) + assert _render_stub(Path("empty.js"), "abc", [], None) == ( + "# Generated by Reflex from empty.js (abc); rewritten when it changes.\n" + ) + go = _export("go") + assert _render_stub(Path("go.js"), "abc", ["go"], {"go": go}) == ( + "# Generated by Reflex from go.js (abc); rewritten when it changes.\n" + "from reflex_base.event import EventSpec as _EventSpec\n" + "\n" + "def go() -> _EventSpec: ...\n" + ) + + +def test_renderer_digest(): + """The renderer's digest is that of this module and script-signatures.js.""" + assert ( + _renderer_digest() + == hashlib.sha256( + Path(stubs.__file__).read_bytes() + + (FRONTEND / "src" / "script-signatures.js").read_bytes() + ).digest() + ) + + +def test_module_digest(tmp_path: Path): + """The digest covers the module's scripts and the files it reads types from. + + That is a declaration beside an import, a type-only import, a JSDoc + ``import("…")`` (even outside the package) and a triple-slash reference, + by their paths relative to the module, and the app's packages; not a + stylesheet or a file nothing reads. + + Args: + tmp_path: pytest tmp_path fixture. + """ + app = tmp_path / "app" + shared = tmp_path / "shared" + app.mkdir() + shared.mkdir() + module = app / "party.js" + module.write_text( + '/// \n' + 'import { help } from "./helpers.js";\n' + 'import "./party.css";\n' + '/** @param {import("../shared/count").Count} n */\n' + "export function go(n) {}\n" + ) + files = { + module: module.read_text(), + app + / "helpers.js": 'import type { Shape } from "./shapes";\nexport const help = 1;\n', + app / "party.css": "body {}\n", + app / "unused.ts": "export const u = 1;\n", + app / "globals.d.ts": "declare const VERSION: string;\n", + app / "helpers.d.ts": "export declare const help: number;\n", + app / "shapes.ts": "export type Shape = string;\n", + shared / "count.d.ts": "export type Count = number;\n", + } + for file, text in files.items(): + file.write_text(text) + expected = hashlib.sha256(_renderer_digest()) + for name, file in [ + ("party.js", module), + ("helpers.js", app / "helpers.js"), + ("shapes.ts", app / "shapes.ts"), + ("helpers.d.ts", app / "helpers.d.ts"), + ("globals.d.ts", app / "globals.d.ts"), + (str(Path("..", "shared", "count.d.ts")), shared / "count.d.ts"), + ]: + expected.update(name.encode() + b"\0" + files[file].encode() + b"\0") + expected.update(b'{"a": "1"}') + assert ( + _module_digest(module, module_files(module), b'{"a": "1"}') + == (expected.hexdigest()[:16]) + ) + + +def test_module_digest_is_the_same_on_every_machine(tmp_path: Path): + """The same sources in different directories give the same digest. + + Args: + tmp_path: pytest tmp_path fixture. + """ + digests = set() + for checkout in ("one", "two"): + module = tmp_path / checkout / "party.js" + module.parent.mkdir() + module.write_text('import "./helpers.js";\nexport function go() {}\n') + module.with_name("helpers.js").write_text("export const help = 1;\n") + digests.add(_module_digest(module, module_files(module), b"")) + assert len(digests) == 1 + + +def test_typescript_matches_the_frontend_package(): + """The typescript installed to read types is the one reflex-frontend uses.""" + dev_dependencies = json.loads((FRONTEND / "package.json").read_text())[ + "devDependencies" + ] + assert f"typescript@{dev_dependencies['typescript']}" == _TYPESCRIPT_PACKAGE diff --git a/tests/units/utils/test_exec.py b/tests/units/utils/test_exec.py index 4eeff83f785..68985f33041 100644 --- a/tests/units/utils/test_exec.py +++ b/tests/units/utils/test_exec.py @@ -5,6 +5,7 @@ import logging import multiprocessing import os +import re import socket import subprocess import sys @@ -1520,3 +1521,15 @@ def test_get_package_json_and_hash_reads_utf8(tmp_path: Path): ) content, _ = exec_utils.get_package_json_and_hash(package_json_path) assert content == {"description": "\u0101", "dependencies": {}} + + +def test_hot_reload_ignores_type_stubs(): + """Writing a type stub next to a JavaScript module does not reload the backend.""" + ignored = [ + name + for name in ("party.pyi", "party.py") + if any( + re.match(pattern, name) for pattern in exec_utils.HOTRELOAD_IGNORE_PATTERNS + ) + ] + assert ignored == ["party.pyi"]