diff --git a/doc/api/single-executable-applications.md b/doc/api/single-executable-applications.md index 6f35f28d4b9d..dcef69a7ee95 100644 --- a/doc/api/single-executable-applications.md +++ b/doc/api/single-executable-applications.md @@ -7,6 +7,9 @@ added: - v19.7.0 - v18.16.0 changes: + - version: REPLACEME + pr-url: https://github.com/nodejs/node/pull/66768 + description: Allowed "useCodeCache" together with "useVfs". - version: v25.5.0 pr-url: https://github.com/nodejs/node/pull/61167 description: Added built-in single executable application generation via the CLI flag `--build-sea`. @@ -317,13 +320,25 @@ with `"vfsArchive"`: `--build-sea` rejects it in `"execArgv"`, and the executable rejects it at startup when it comes from `--node-options` or from the `execArgv` of a worker. -#### Snapshot and code caching limitations +#### Code cache for the modules in the VFS -`"useVfs": true` cannot be used together with `"useSnapshot": true` or -`"useCodeCache": true`. The code cache limitation is due to incomplete -implementation, not a technical impossibility. Consider bundling the -application if startup performance matters and do not rely on module loading -from the VFS in that case. +When `"useCodeCache": true` is combined with `"useVfs": true`, the code cache +is not limited to the main script: `--build-sea` compiles every JavaScript +module in the virtual file system (and every TypeScript module, when type +stripping is enabled), including the main script and the modules under +`node_modules`, and embeds their V8 code cache into the preparation blob. +The modules are compiled without being run, so the build has no side +effects. A module that fails to compile is left out of the cache and fails +the same way at run time if the application loads it. + +The cache is keyed by the path of each module inside the mount and verified +against its source, so it is used by the executable whenever the modules are +loaded from the VFS, and ignored if they changed. The cache is +platform-specific, like the code cache of the main script. + +#### Snapshot limitations + +`"useVfs": true` cannot be used together with `"useSnapshot": true`. #### Native addon limitations @@ -373,6 +388,9 @@ executable application is launched, instead of compiling the `main` script from scratch, Node.js would use the code cache to speed up the compilation, then execute the script, which would improve the startup performance. +When `useVfs` is also set to `true`, the code cache covers every module in the +virtual file system, see [Code cache for the modules in the VFS][]. + ### Execution arguments The `execArgv` field can be used to specify Node.js-specific @@ -798,6 +816,7 @@ Suggestions for other resource injection tools/workflows are welcomed. Please start a discussion at to help us document them. +[Code cache for the modules in the VFS]: #code-cache-for-the-modules-in-the-vfs [CommonJS]: modules.md#modules-commonjs-modules [ECMAScript Modules]: esm.md#modules-ecmascript-modules [ELF]: https://en.wikipedia.org/wiki/Executable_and_Linkable_Format diff --git a/doc/api/vfs.md b/doc/api/vfs.md index ecadd05d75b8..300c86eaa983 100644 --- a/doc/api/vfs.md +++ b/doc/api/vfs.md @@ -571,8 +571,10 @@ ESM entry points (`"mainFormat": "module"`) are supported: the main module is loaded from inside the mount through the ESM loader, and `import.meta.dirname` points at the mount root. -`"useVfs"` cannot be used together with `"useSnapshot"` or `"useCodeCache"`. -The SEA configuration parser will error if either combination is detected. +`"useVfs"` cannot be used together with `"useSnapshot"`; the SEA +configuration parser will error if the combination is detected. With +`"useCodeCache"`, every module in the mount is compiled at build time and its +code cache is embedded into the executable. Instead of listing individual `"assets"`, the SEA configuration can point `"vfsArchive"` at a prebuilt ZIP archive; the mount is then backed by a diff --git a/lib/internal/vfs/providers/sea.js b/lib/internal/vfs/providers/sea.js index 4a03246c271b..fa8df1b8f799 100644 --- a/lib/internal/vfs/providers/sea.js +++ b/lib/internal/vfs/providers/sea.js @@ -44,6 +44,11 @@ const { }, } = internalBinding('constants'); +// Assets under this prefix are reserved by --build-sea (the compile cache of +// the modules in the virtual file system) and are not files of the +// application. Must match src/node_sea.cc. +const kReservedAssetPrefix = 'node:sea:'; + // Private symbols const kAssets = Symbol('kAssets'); const kExtraFiles = Symbol('kExtraFiles'); @@ -91,6 +96,7 @@ class SEAProvider extends VirtualProvider { const keys = getAssetKeys() || []; for (let i = 0; i < keys.length; i++) { const key = keys[i]; + if (StringPrototypeStartsWith(key, kReservedAssetPrefix)) continue; this[kAssets].set(this.#normalizePath(key), key); } const extraFiles = options.extraFiles; diff --git a/lib/internal/vfs/sea.js b/lib/internal/vfs/sea.js index 4bf2273c8bbc..c6aa19314533 100644 --- a/lib/internal/vfs/sea.js +++ b/lib/internal/vfs/sea.js @@ -83,11 +83,13 @@ function initSeaVfsInWorker() { /** * The name under which the SEA main script is placed at the mount point root. + * @param {string} [mainPath] The "main" of the SEA configuration; defaults + * to the one recorded in the running executable * @returns {string} */ -function getSeaMainName() { +function getSeaMainName(mainPath = mainCodePath || process.execPath) { const { basename } = require('path'); - return basename(mainCodePath || process.execPath); + return basename(mainPath); } // The reserved asset key under which --build-sea stores the ZIP archive diff --git a/lib/internal/vfs/sea_compile_cache.js b/lib/internal/vfs/sea_compile_cache.js new file mode 100644 index 000000000000..1d77e54f2855 --- /dev/null +++ b/lib/internal/vfs/sea_compile_cache.js @@ -0,0 +1,234 @@ +'use strict'; + +// Build-time counterpart of the SEA virtual file system: run by --build-sea +// (src/node_sea.cc) in a regular Node.js instance to compile every module of +// the virtual file system and collect their compile cache, which the +// executable then carries as assets. The assets are mounted the way the +// executable mounts them at startup, so the modules are compiled from the +// same paths and sources that the loaders compile them from in the +// executable, and the cache is keyed and hashed exactly as the loaders look +// it up there. + +const { + ArrayPrototypeIncludes, + StringPrototypeEndsWith, + StringPrototypeReplaceAll, +} = primordials; + +const { Buffer } = require('buffer'); +const fs = require('fs'); +const path = require('path'); +const { pathToFileURL } = require('internal/url'); +const { getOptionValue } = require('internal/options'); +const { + enableCompileCache: enableNativeCompileCache, +} = internalBinding('modules'); +const { + constants: { compileCacheStatus }, + stringify, +} = require('internal/modules/helpers'); +const { + codes: { + ERR_INVALID_STATE, + }, +} = require('internal/errors'); + +let debug = require('internal/util/debuglog').debuglog('sea', (fn) => { + debug = fn; +}); + +// Only files the module loaders compile are cached. Everything else in the +// mount is data. +const kScriptExtensions = ['.js', '.cjs', '.mjs']; +const kTypeScriptExtensions = ['.ts', '.cts', '.mts']; + +/** + * Mounts the assets of the SEA configuration the way the executable mounts + * them: the main script at the mount point root, at the layer reserved for + * --vfs-load when they come from an archive (see lib/internal/vfs/sea.js). + * @param {object} options The SEA configuration, see buildSeaCompileCache() + * @param {string} mainName The name of the main script at the mount root + * @returns {string} The mount point + */ +function mountAssets(options, mainName) { + const { VirtualFileSystem, kLoadLayer } = require('internal/vfs/file_system'); + const isArchive = options.vfsArchive !== undefined; + + let provider; + if (isArchive) { + const { ZipBuffer } = require('internal/zip'); + const { ZipProvider } = require('internal/vfs/providers/ziparchive'); + const zip = new ZipBuffer(fs.readFileSync(options.vfsArchive)); + zip.addSync(mainName, Buffer.from(options.mainSource), + { __proto__: null, method: 'store' }); + provider = new ZipProvider(zip); + } else { + const { MemoryProvider } = require('internal/vfs/providers/memory'); + provider = new MemoryProvider(); + } + + const vfs = new VirtualFileSystem(provider, { + __proto__: null, + emitExperimentalWarning: false, + [kLoadLayer]: isArchive, + }); + + if (!isArchive) { + // Every asset is a file at its key, as the SEAProvider serves them. + const { assetKeys, assetPaths } = options; + for (let i = 0; i < assetKeys.length; i++) { + const target = path.posix.normalize( + '/' + StringPrototypeReplaceAll(assetKeys[i], '\\', '/')); + const dir = path.posix.dirname(target); + if (dir !== '/') { + vfs.mkdirSync(dir, { __proto__: null, recursive: true }); + } + vfs.writeFileSync(target, fs.readFileSync(assetPaths[i])); + } + vfs.writeFileSync('/' + mainName, options.mainSource); + provider.setReadOnly(); + } + + return vfs.mount(); +} + +/** + * Compiles a module without running it, the way the loaders compile it when + * it is loaded from the mount, so that its compile cache entry is created. + * @param {string} filename The path of the module inside the mount + * @returns {boolean} Whether the file is a module the loaders compile + */ +function compileModule(filename) { + const { Module, wrapSafe } = require('internal/modules/cjs/loader'); + const { + defaultGetFormatWithoutErrors, + } = require('internal/modules/esm/get_format'); + const { + compileSourceTextModule, + SourceTextModuleTypes: { kUser }, + } = require('internal/modules/esm/utils'); + + const url = pathToFileURL(filename); + const buffer = fs.readFileSync(filename); + let format = defaultGetFormatWithoutErrors(url, { + __proto__: null, + source: buffer, + }); + + let source; + switch (format) { + case 'module': + // The ESM loader decodes the source it loads. + source = stringify(buffer); + break; + case 'commonjs': + case 'commonjs-typescript': + case 'module-typescript': + case 'typescript': + // The CommonJS loader reads the source as a UTF-8 string. + source = buffer.toString('utf8'); + break; + default: + return false; + } + + if (StringPrototypeEndsWith(format, 'typescript')) { + const { stripTypeScriptModuleTypes } = require('internal/modules/typescript'); + source = stripTypeScriptModuleTypes(source, filename, url.href); + switch (format) { + case 'commonjs-typescript': + format = 'commonjs'; + break; + case 'module-typescript': + format = 'module'; + break; + default: + // Detected from the type-stripped source by wrapSafe(), like + // Module.prototype._compile() does. + format = undefined; + break; + } + } + + if (format !== 'module') { + const result = wrapSafe(filename, source, new Module(filename, null), format); + if (!result.canParseAsESM) { + return true; + } + } + compileSourceTextModule(url.href, source, kUser); + return true; +} + +/** + * Compiles every module under a directory of the mount. + * @param {string} dir The directory + * @param {boolean} stripTypes Whether TypeScript files are modules too + * @returns {number} The number of modules compiled + */ +function compileTree(dir, stripTypes) { + let compiled = 0; + const dirents = fs.readdirSync(dir, { __proto__: null, withFileTypes: true }); + for (let i = 0; i < dirents.length; i++) { + const dirent = dirents[i]; + const filename = path.join(dir, dirent.name); + if (dirent.isDirectory()) { + compiled += compileTree(filename, stripTypes); + continue; + } + if (!dirent.isFile()) continue; + const ext = path.extname(filename); + if (!ArrayPrototypeIncludes(kScriptExtensions, ext) && + !(stripTypes && ArrayPrototypeIncludes(kTypeScriptExtensions, ext))) { + continue; + } + // A module that does not compile is left out of the cache: it fails the + // same way at run time if the application ever loads it. + try { + if (compileModule(filename)) compiled++; + } catch (err) { + debug('not caching %s: %s', filename, err?.message); + } + } + return compiled; +} + +/** + * Compiles the modules of the SEA virtual file system into the compile cache + * of this Node.js instance, from which --build-sea collects them. The cache + * is keyed relative to the mount point, which is the same in the executable. + * @param {object} options + * @param {string} options.main The "main" of the SEA configuration + * @param {string} options.mainSource The source of the main script + * @param {string} [options.vfsArchive] The "vfsArchive" of the configuration + * @param {string[]} [options.assetKeys] The keys of the "assets" + * @param {string[]} [options.assetPaths] The paths of the "assets" + * @returns {number} The number of modules compiled + */ +function buildSeaCompileCache(options) { + const { getSeaMainName } = require('internal/vfs/sea'); + const mountPoint = mountAssets(options, getSeaMainName(options.main)); + + // Enabling the compile cache takes a directory, but nothing is persisted + // to it: the entries are collected from memory by --build-sea. + const cacheDir = fs.mkdtempSync( + path.join(require('os').tmpdir(), 'node-sea-compile-cache-')); + try { + const { 0: status, 1: message } = enableNativeCompileCache( + cacheDir, true /* portable */, false /* readOnly */, mountPoint); + if (status !== compileCacheStatus.ENABLED) { + throw new ERR_INVALID_STATE( + `cannot enable the compile cache for ${mountPoint}: ${message}`); + } + const compiled = compileTree(mountPoint, getOptionValue('--strip-types')); + debug('compiled %d modules in %s for the compile cache', + compiled, mountPoint); + return compiled; + } finally { + fs.rmSync(cacheDir, { __proto__: null, recursive: true, force: true }); + } +} + +module.exports = { + buildSeaCompileCache, +}; diff --git a/src/compile_cache.cc b/src/compile_cache.cc index 6e891619f42b..ae3a23e61e39 100644 --- a/src/compile_cache.cc +++ b/src/compile_cache.cc @@ -265,12 +265,12 @@ CompileCacheEntry* CompileCacheHandler::GetOrInsert(Local code, return nullptr; } std::string relative_path = - GetRelativePath(normalized_file_path, normalized_compile_cache_dir_); + GetRelativePath(normalized_file_path, normalized_key_root_); if (!relative_path.empty()) { file_path = relative_path; Debug("[compile cache] using relative path %s from %s\n", file_path, - compile_cache_dir_); + normalized_key_root_); } } uint32_t key = GetCacheKey(file_path, type); @@ -435,20 +435,8 @@ void CompileCacheHandler::Persist() { continue; } - DCHECK_EQ(entry->cache->buffer_policy, - ScriptCompiler::CachedData::BufferOwned); - char* cache_ptr = - reinterpret_cast(const_cast(entry->cache->data)); - uint32_t cache_size = static_cast(entry->cache->length); - uint32_t cache_hash = GetHash(cache_ptr, cache_size); - - // Generating headers. - std::vector headers(kHeaderCount); - headers[kMagicNumberOffset] = kCacheMagicNumber; - headers[kCodeSizeOffset] = entry->code_size; - headers[kCacheSizeOffset] = cache_size; - headers[kCodeHashOffset] = entry->code_hash; - headers[kCacheHashOffset] = cache_hash; + std::vector data = SerializeEntry(entry); + const uint32_t* headers = reinterpret_cast(data.data()); // Generate the temporary filename. // The temporary file should be placed in a location like: @@ -489,16 +477,13 @@ void CompileCacheHandler::Persist() { headers[kCacheHashOffset]); // Write to the temporary file. - uv_buf_t headers_buf = uv_buf_init(reinterpret_cast(headers.data()), - headers.size() * sizeof(uint32_t)); - uv_buf_t data_buf = uv_buf_init(cache_ptr, entry->cache->length); - uv_buf_t bufs[] = {headers_buf, data_buf}; + uv_buf_t buf = uv_buf_init(data.data(), data.size()); uv_fs_t write_req; auto cleanup_write = OnScopeLeave([&write_req]() { uv_fs_req_cleanup(&write_req); }); err = uv_fs_write( - nullptr, &write_req, mkstemp_req.result, bufs, 2, 0, nullptr); + nullptr, &write_req, mkstemp_req.result, &buf, 1, 0, nullptr); if (err < 0) { Debug("failed: %s\n", uv_strerror(err)); continue; @@ -543,6 +528,47 @@ void CompileCacheHandler::Persist() { compiler_cache_store_.clear(); } +// Serializes an entry in the layout of a cache file, see Persist(). +std::vector CompileCacheHandler::SerializeEntry( + const CompileCacheEntry* entry) { + DCHECK_NOT_NULL(entry->cache); + DCHECK_EQ(entry->cache->buffer_policy, + ScriptCompiler::CachedData::BufferOwned); + const char* cache_ptr = reinterpret_cast(entry->cache->data); + uint32_t cache_size = static_cast(entry->cache->length); + + uint32_t headers[kHeaderCount]; + headers[kMagicNumberOffset] = kCacheMagicNumber; + headers[kCodeSizeOffset] = entry->code_size; + headers[kCacheSizeOffset] = cache_size; + headers[kCodeHashOffset] = entry->code_hash; + headers[kCacheHashOffset] = GetHash(cache_ptr, cache_size); + + const char* headers_ptr = reinterpret_cast(headers); + std::vector data; + data.reserve(sizeof(headers) + cache_size); + data.insert(data.end(), headers_ptr, headers_ptr + sizeof(headers)); + data.insert(data.end(), cache_ptr, cache_ptr + cache_size); + return data; +} + +std::vector>> +CompileCacheHandler::SerializeEntries() const { + std::vector>> result; + for (const auto& pair : compiler_cache_store_) { + const CompileCacheEntry* entry = pair.second.get(); + if (entry->cache == nullptr) { + Debug("[compile cache] skip serializing %s %s because the cache was " + "not initialized\n", + entry->type_name(), + entry->source_filename); + continue; + } + result.emplace_back(Uint32ToHex(entry->cache_key), SerializeEntry(entry)); + } + return result; +} + CompileCacheHandler::CompileCacheHandler(Environment* env) : isolate_(env->isolate()), is_debug_( @@ -553,9 +579,11 @@ CompileCacheHandler::CompileCacheHandler(Environment* env) // - $NODE_VERSION-$ARCH-$CACHE_DATA_VERSION_TAG-$UID // ($UID is omitted for a portable cache) // - $FILENAME_AND_MODULE_TYPE_HASH.cache: a hash of filename + module type -CompileCacheEnableResult CompileCacheHandler::Enable(Environment* env, - const std::string& dir, - EnableOption option) { +CompileCacheEnableResult CompileCacheHandler::Enable( + Environment* env, + const std::string& dir, + EnableOption option, + const std::string& key_root) { std::string cache_tag = GetCacheVersionTag(option); std::string absolute_cache_dir_base = PathResolve(env, {dir}); std::string cache_dir_with_tag = @@ -627,8 +655,8 @@ CompileCacheEnableResult CompileCacheHandler::Enable(Environment* env, portable_ = option; read_only_ = read_only; if (HasOption(option, EnableOption::PORTABLE)) { - normalized_compile_cache_dir_ = - NormalizeFileURLOrPath(env, compile_cache_dir_); + normalized_key_root_ = NormalizeFileURLOrPath( + env, key_root.empty() ? compile_cache_dir_ : key_root); } result.status = CompileCacheEnableStatus::ENABLED; return result; diff --git a/src/compile_cache.h b/src/compile_cache.h index 0fbce3624a61..a5c5d044c107 100644 --- a/src/compile_cache.h +++ b/src/compile_cache.h @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include "v8.h" namespace node { @@ -80,11 +82,18 @@ inline constexpr bool HasOption(EnableOption value, EnableOption flag) { class CompileCacheHandler { public: explicit CompileCacheHandler(Environment* env); + // With PORTABLE, the cache keys are computed from paths relative to + // key_root, or to the cache directory when key_root is empty. CompileCacheEnableResult Enable(Environment* env, const std::string& dir, - EnableOption option = EnableOption::DEFAULT); + EnableOption option = EnableOption::DEFAULT, + const std::string& key_root = std::string()); void Persist(); + // Every entry holding a cache in the layout of a cache file, paired with + // the name of the file Persist() would write it to. + std::vector>> SerializeEntries() + const; CompileCacheEntry* GetOrInsert(v8::Local code, v8::Local filename, @@ -101,6 +110,7 @@ class CompileCacheHandler { private: void ReadCacheFile(CompileCacheEntry* entry); + static std::vector SerializeEntry(const CompileCacheEntry* entry); template void MaybeSaveImpl(CompileCacheEntry* entry, @@ -121,7 +131,7 @@ class CompileCacheHandler { bool is_debug_ = false; std::string compile_cache_dir_; - std::string normalized_compile_cache_dir_; + std::string normalized_key_root_; EnableOption portable_ = EnableOption::DEFAULT; bool read_only_ = false; std::unordered_map> diff --git a/src/env.cc b/src/env.cc index 2da45b82f1a2..8f49efaacd2e 100644 --- a/src/env.cc +++ b/src/env.cc @@ -1385,7 +1385,9 @@ void Environment::InitializeCompileCache() { } CompileCacheEnableResult Environment::EnableCompileCache( - const std::string& cache_dir, EnableOption option) { + const std::string& cache_dir, + EnableOption option, + const std::string& key_root) { CompileCacheEnableResult result; std::string disable_env; if (credentials::SafeGetenv( @@ -1402,7 +1404,7 @@ CompileCacheEnableResult Environment::EnableCompileCache( if (!compile_cache_handler_) { std::unique_ptr handler = std::make_unique(this); - result = handler->Enable(this, cache_dir, option); + result = handler->Enable(this, cache_dir, option, key_root); if (result.status == CompileCacheEnableStatus::ENABLED) { compile_cache_handler_ = std::move(handler); AtExit( diff --git a/src/env.h b/src/env.h index 0349add75442..b737b7eb6e9e 100644 --- a/src/env.h +++ b/src/env.h @@ -1170,8 +1170,10 @@ class Environment final : public MemoryRetainer { void InitializeCompileCache(); // Enable built-in compile cache if it has not yet been enabled. // The cache will be persisted to disk on exit. - CompileCacheEnableResult EnableCompileCache(const std::string& cache_dir, - EnableOption option); + CompileCacheEnableResult EnableCompileCache( + const std::string& cache_dir, + EnableOption option, + const std::string& key_root = std::string()); void FlushCompileCache(); void RunAndClearNativeImmediates(bool only_refed = false); diff --git a/src/node_modules.cc b/src/node_modules.cc index ec4f6bdd79a8..441051fcf045 100644 --- a/src/node_modules.cc +++ b/src/node_modules.cc @@ -585,9 +585,16 @@ void EnableCompileCache(const FunctionCallbackInfo& args) { if (args.Length() > 2 && args[2]->IsTrue()) { option = option | EnableOption::READ_ONLY; } + // Internal: the directory the keys of a portable cache are relative to. + std::string key_root; + if (args.Length() > 3 && args[3]->IsString()) { + Utf8Value key_root_value(isolate, args[3]); + key_root = key_root_value.ToString(); + } Utf8Value value(isolate, args[0]); - CompileCacheEnableResult result = env->EnableCompileCache(*value, option); + CompileCacheEnableResult result = + env->EnableCompileCache(*value, option, key_root); Local values[3]; values[0] = v8::Integer::New(isolate, static_cast(result.status)); if (ToV8Value(context, result.message).ToLocal(&values[1]) && diff --git a/src/node_sea.cc b/src/node_sea.cc index 975df52d6efa..bb6f23b6230b 100644 --- a/src/node_sea.cc +++ b/src/node_sea.cc @@ -8,6 +8,7 @@ #include "node_external_reference.h" #include "node_internals.h" #include "node_options.h" +#include "node_realm-inl.h" #include "node_snapshot_builder.h" #include "node_union_bytes.h" #include "node_v8_platform-inl.h" @@ -47,6 +48,12 @@ namespace sea { // The reserved asset key under which the ZIP archive named by "vfsArchive" // is stored. Must match the name used by lib/internal/vfs/sea.js. constexpr std::string_view kVfsArchiveAssetName = "node:sea:vfs.zip"; +// The prefix of the reserved asset keys holding the compile cache of the +// modules in the virtual file system, one asset per module named by its +// compile cache key. Must match the prefix skipped by +// lib/internal/vfs/providers/sea.js. +constexpr std::string_view kVfsCompileCacheAssetPrefix = + "node:sea:compile-cache/"; namespace { @@ -613,11 +620,6 @@ std::optional ParseSingleExecutableConfig( "\"useVfs\" is not supported when \"useSnapshot\" is true\n"); return std::nullopt; } - if (static_cast(result.flags & SeaFlags::kUseCodeCache)) { - FPrintF(stderr, - "\"useVfs\" is not supported when \"useCodeCache\" is true\n"); - return std::nullopt; - } } if (static_cast(result.flags & SeaFlags::kVfsArchive)) { @@ -792,6 +794,127 @@ std::optional GenerateCodeCache(std::string_view main_path, return code_cache; } +// Compiles every module in the virtual file system of the single executable +// application and collects their compile cache, to be embedded as the assets +// under kVfsCompileCacheAssetPrefix. The modules are compiled by a Node.js +// instance that mounts the assets the way the executable does at startup +// (see lib/internal/vfs/sea_compile_cache.js), so the cache is keyed and +// hashed exactly as the module loaders look it up from the mount. +ExitCode GenerateVfsCompileCache( + const SeaConfig& config, + const std::vector& args, + const std::vector& exec_args, + std::string_view main_script, + std::unordered_map* assets) { + std::vector errors; + std::vector env_args = {args[0]}; + std::unique_ptr setup = + CommonEnvironmentSetup::Create( + per_process::v8_platform.Platform(), &errors, env_args, exec_args); + if (!setup) { + for (const std::string& err : errors) { + FPrintF(stderr, "%s: %s\n", args[0], err); + } + return ExitCode::kBootstrapFailure; + } + + Isolate* isolate = setup->isolate(); + v8::Locker locker(isolate); + Isolate::Scope isolate_scope(isolate); + HandleScope handle_scope(isolate); + Local context = setup->context(); + Context::Scope context_scope(context); + Environment* env = setup->env(); + + // The cache is collected from this instance, which must neither read nor + // write the compile cache the build process itself may be configured with. + env->set_env_vars(env->env_vars()->Clone(isolate)); + for (const char* name : {"NODE_COMPILE_CACHE", + "NODE_COMPILE_CACHE_PORTABLE", + "NODE_COMPILE_CACHE_READONLY", + "NODE_DISABLE_COMPILE_CACHE"}) { + env->env_vars()->Delete(isolate, OneByteString(isolate, name)); + } + + errors::PrinterTryCatch bootstrapCatch( + isolate, errors::PrinterTryCatch::kPrintSourceLine); + + Local options = Object::New(isolate); + auto set_option = [&](const char* key, auto&& value) { + Local v8_value; + return ToV8Value(context, value).ToLocal(&v8_value) && + options->Set(context, OneByteString(isolate, key), v8_value) + .IsJust(); + }; + if (!set_option("main", config.main_path) || + !set_option("mainSource", main_script)) { + return ExitCode::kGenericUserError; + } + if (static_cast(config.flags & SeaFlags::kVfsArchive)) { + if (!set_option("vfsArchive", config.vfs_archive_path)) { + return ExitCode::kGenericUserError; + } + } else { + std::vector asset_keys; + std::vector asset_paths; + for (auto const& [key, path] : config.assets) { + asset_keys.push_back(key); + asset_paths.push_back(path); + } + if (!set_option("assetKeys", asset_keys) || + !set_option("assetPaths", asset_paths)) { + return ExitCode::kGenericUserError; + } + } + + Local compiled; + bool loaded = + LoadEnvironment( + env, + [&](const StartExecutionCallbackInfo& info) -> MaybeLocal { + Local id = FIXED_ONE_BYTE_STRING( + isolate, "internal/vfs/sea_compile_cache"); + Local builder; + if (!env->principal_realm() + ->builtin_module_require() + ->Call(context, Undefined(isolate), 1, &id) + .ToLocal(&builder)) { + return MaybeLocal(); + } + Local fn; + if (!builder.As() + ->Get(context, + FIXED_ONE_BYTE_STRING(isolate, + "buildSeaCompileCache")) + .ToLocal(&fn)) { + return MaybeLocal(); + } + Local argv[] = {options}; + return fn.As()->Call( + context, Undefined(isolate), arraysize(argv), argv); + }) + .ToLocal(&compiled); + if (!loaded) { + FPrintF(stderr, + "Cannot generate the compile cache of the virtual file system\n"); + return ExitCode::kGenericUserError; + } + + CHECK(env->use_compile_cache()); + size_t entries = 0; + for (auto& [name, data] : env->compile_cache_handler()->SerializeEntries()) { + assets->emplace(std::string(kVfsCompileCacheAssetPrefix) + name, + std::string(data.begin(), data.end())); + entries++; + } + per_process::Debug(DebugCategory::SEA, + "Generated %zu compile cache entries for %d modules in " + "the virtual file system\n", + entries, + compiled.As()->Value()); + return ExitCode::kNoFailure; +} + int BuildAssets(const std::unordered_map& config, std::unordered_map* assets) { for (auto const& [key, path] : config) { @@ -875,6 +998,17 @@ ExitCode GenerateSingleExecutableBlob( } assets.emplace(std::string(kVfsArchiveAssetName), std::move(archive)); } + SeaFlags flags = config.flags; + if (static_cast(flags & SeaFlags::kEnableVfs) && + static_cast(flags & SeaFlags::kUseCodeCache)) { + ExitCode exit_code = GenerateVfsCompileCache( + config, args, exec_args, main_script, &assets); + if (exit_code != ExitCode::kNoFailure) { + return exit_code; + } + // The cache is embedded as assets even when the configuration has none. + flags |= SeaFlags::kIncludeAssets; + } std::unordered_map assets_view; for (auto const& [key, content] : assets) { assets_view.emplace(key, content); @@ -884,7 +1018,7 @@ ExitCode GenerateSingleExecutableBlob( exec_argv_view.emplace_back(arg); } SeaResource sea{ - config.flags, + flags, config.exec_argv_extension, config.main_path, builds_snapshot_from_main diff --git a/test/fixtures/sea/vfs-esm/sea-config-code-cache.json b/test/fixtures/sea/vfs-esm/sea-config-code-cache.json new file mode 100644 index 000000000000..26bd6b0f6f4a --- /dev/null +++ b/test/fixtures/sea/vfs-esm/sea-config-code-cache.json @@ -0,0 +1,15 @@ +{ + "main": "main.mjs", + "mainFormat": "module", + "output": "sea-prep.blob", + "useVfs": true, + "useCodeCache": true, + "assets": { + "config.json": "config.json", + "data/greeting.txt": "greeting.txt", + "modules/math.mjs": "math.mjs", + "modules/calculator.cjs": "calculator.cjs", + "node_modules/test-esm-pkg/package.json": "test-esm-pkg-package.json", + "node_modules/test-esm-pkg/index.mjs": "test-esm-pkg-index.mjs" + } +} diff --git a/test/fixtures/sea/vfs-zip/sea-config-code-cache.json b/test/fixtures/sea/vfs-zip/sea-config-code-cache.json new file mode 100644 index 000000000000..3a8791cd4aee --- /dev/null +++ b/test/fixtures/sea/vfs-zip/sea-config-code-cache.json @@ -0,0 +1,7 @@ +{ + "main": "sea.js", + "output": "sea-prep.blob", + "useVfs": true, + "useCodeCache": true, + "vfsArchive": "assets.zip" +} diff --git a/test/fixtures/sea/vfs/sea-config-code-cache.json b/test/fixtures/sea/vfs/sea-config-code-cache.json new file mode 100644 index 000000000000..7627849eb9de --- /dev/null +++ b/test/fixtures/sea/vfs/sea-config-code-cache.json @@ -0,0 +1,16 @@ +{ + "main": "sea.js", + "output": "sea-prep.blob", + "useVfs": true, + "useCodeCache": true, + "assets": { + "config.json": "config.json", + "data/greeting.txt": "greeting.txt", + "modules/math.js": "math.js", + "modules/calculator.js": "calculator.js", + "node_modules/test-pkg/package.json": "test-pkg-package.json", + "node_modules/test-pkg/index.js": "test-pkg-index.js", + "node_modules/test-exports-pkg/package.json": "test-exports-pkg-package.json", + "node_modules/test-exports-pkg/lib/entry.js": "test-exports-pkg-entry.js" + } +} diff --git a/test/sea/test-build-sea-vfs-code-cache-errors.js b/test/sea/test-build-sea-vfs-code-cache-errors.js new file mode 100644 index 000000000000..b672d16587ec --- /dev/null +++ b/test/sea/test-build-sea-vfs-code-cache-errors.js @@ -0,0 +1,227 @@ +'use strict'; + +// This tests the error conditions of generating the compile cache of the +// virtual file system when "useCodeCache" is combined with "useVfs": a module +// that does not compile is left out of the cache, while a broken archive, a +// broken main script or a build environment without a usable temporary +// directory fail the build. + +const common = require('../common'); + +const { + buildSEA, + skipIfBuildSEAIsNotSupported, +} = require('../common/sea'); + +skipIfBuildSEAIsNotSupported(); + +const assert = require('assert'); +const fs = require('fs'); +const tmpdir = require('../common/tmpdir'); +const fixtures = require('../common/fixtures'); +const { spawnSyncAndAssert } = require('../common/child_process'); + +const kCompileCachePrefix = 'node:sea:compile-cache/'; +const kBlob = 'sea-prep.blob'; + +const hasTypeScript = process.features.typescript === 'strip'; + +function countCompileCacheEntries(blobPath) { + const blob = fs.readFileSync(blobPath).toString('latin1'); + return blob.split(kCompileCachePrefix).length - 1; +} + +function writeConfig(name, config) { + fs.writeFileSync(tmpdir.resolve(name), JSON.stringify(config)); +} + +// Generates the preparation blob of the configuration in the working +// directory and returns the child process result. +function generateBlob(configName, env, expectations) { + return spawnSyncAndAssert( + process.execPath, + ['--experimental-sea-config', configName], + { + cwd: tmpdir.path, + env: { + ...process.env, + NODE_DEBUG: 'sea', + NODE_DEBUG_NATIVE: 'SEA', + ...env, + }, + }, + expectations); +} + +// Test: a module that does not compile is skipped, the rest is cached, and +// the executable runs with the broken module failing when it is loaded. +{ + tmpdir.refresh(); + fs.writeFileSync(tmpdir.resolve('main.js'), ` + const path = require('path'); + try { + require(path.join(__dirname, 'modules', 'broken.js')); + } catch (err) { + console.log('broken:', err.name); + } + console.log('good:', require(path.join(__dirname, 'modules', 'good.js'))); + if (process.features.typescript === 'strip') { + console.log('ts:', require(path.join(__dirname, 'modules', 'good.ts'))); + } + `); + fs.writeFileSync(tmpdir.resolve('good.js'), 'module.exports = 42;\n'); + fs.writeFileSync(tmpdir.resolve('broken.js'), 'module.exports = {\n'); + const assets = { + 'modules/good.js': 'good.js', + 'modules/broken.js': 'broken.js', + }; + // main.js, good.js and, with type stripping, good.ts, which has two + // entries: the stripped source and the code cache compiled from it. + let modules = 2; + let entries = 2; + if (hasTypeScript) { + fs.writeFileSync(tmpdir.resolve('good.ts'), + 'const n: number = 42;\nmodule.exports = n;\n'); + // Enums are not supported by type stripping. + fs.writeFileSync(tmpdir.resolve('enum.ts'), 'enum E { A }\n'); + assets['modules/good.ts'] = 'good.ts'; + assets['modules/enum.ts'] = 'enum.ts'; + modules++; + entries += 2; + } + const config = { + main: 'main.js', + output: kBlob, + useVfs: true, + useCodeCache: true, + assets, + }; + writeConfig('sea-config-blob.json', config); + writeConfig('sea-config.json', { ...config, output: 'sea' }); + + generateBlob('sea-config-blob.json', {}, { + stderr: common.mustCall((stderr) => { + assert.match(stderr, /not caching \S*broken\.js: /); + if (hasTypeScript) { + assert.match(stderr, /not caching \S*enum\.ts: /); + } + assert.match(stderr, new RegExp( + `Generated ${entries} compile cache entries for ${modules} modules`)); + }), + }); + assert.strictEqual(countCompileCacheEntries(tmpdir.resolve(kBlob)), entries); + + const outputFile = buildSEA(tmpdir.path); + spawnSyncAndAssert( + outputFile, + { + env: { + ...process.env, + NODE_DEBUG: undefined, + NODE_DEBUG_NATIVE: undefined, + }, + }, + { + stdout: common.mustCall((stdout) => { + assert.match(stdout, /^broken: SyntaxError$/m); + assert.match(stdout, /^good: 42$/m); + if (hasTypeScript) { + assert.match(stdout, /^ts: 42$/m); + } + }), + stderr: common.mustCall((stderr) => { + assert.doesNotMatch(stderr, /Code cache data rejected/); + }), + }); +} + +// Test: an ES module that does not compile is skipped too. +{ + tmpdir.refresh(); + fs.cpSync(fixtures.path('sea', 'vfs-esm'), tmpdir.path, { recursive: true }); + fs.writeFileSync(tmpdir.resolve('broken.mjs'), 'export {\n'); + const config = JSON.parse( + fs.readFileSync(tmpdir.resolve('sea-config-code-cache.json'), 'utf8')); + config.assets['modules/broken.mjs'] = 'broken.mjs'; + writeConfig('sea-config-code-cache.json', config); + + generateBlob('sea-config-code-cache.json', {}, { + stderr: common.mustCall((stderr) => { + assert.match(stderr, /not caching \S*broken\.mjs: /); + assert.match(stderr, /Generated 4 compile cache entries for 4 modules/); + }), + }); + assert.strictEqual(countCompileCacheEntries(tmpdir.resolve(kBlob)), 4); +} + +// Test: a "vfsArchive" that passes the signature check but cannot be parsed +// fails the build when the compile cache is generated from it. +{ + tmpdir.refresh(); + fs.writeFileSync(tmpdir.resolve('main.js'), 'console.log("hello");\n'); + fs.writeFileSync(tmpdir.resolve('assets.zip'), + Buffer.concat([Buffer.from('PK\x03\x04'), Buffer.alloc(64)])); + writeConfig('sea-config.json', { + main: 'main.js', + output: kBlob, + useVfs: true, + useCodeCache: true, + vfsArchive: 'assets.zip', + }); + + generateBlob('sea-config.json', {}, { + status: 1, + stderr: common.mustCall((stderr) => { + assert.match(stderr, /ERR_ZIP_INVALID_ARCHIVE/); + assert.match( + stderr, /Cannot generate the compile cache of the virtual file system/); + }), + }); + assert.strictEqual(fs.existsSync(tmpdir.resolve(kBlob)), false); +} + +// Test: a main script that does not compile fails the build. +{ + tmpdir.refresh(); + fs.writeFileSync(tmpdir.resolve('main.js'), 'console.log(\n'); + writeConfig('sea-config.json', { + main: 'main.js', + output: kBlob, + useVfs: true, + useCodeCache: true, + }); + + generateBlob('sea-config.json', {}, { + status: 1, + stderr: /Cannot generate V8 code cache/, + }); + assert.strictEqual(fs.existsSync(tmpdir.resolve(kBlob)), false); +} + +// Test: the compile cache is collected through a temporary directory, so the +// build fails without a usable one. +{ + tmpdir.refresh(); + fs.writeFileSync(tmpdir.resolve('main.js'), 'console.log("hello");\n'); + writeConfig('sea-config.json', { + main: 'main.js', + output: kBlob, + useVfs: true, + useCodeCache: true, + }); + + const missing = tmpdir.resolve('missing'); + generateBlob('sea-config.json', { + TMPDIR: missing, + TMP: missing, + TEMP: missing, + }, { + status: 1, + stderr: common.mustCall((stderr) => { + assert.match(stderr, /ENOENT/); + assert.match( + stderr, /Cannot generate the compile cache of the virtual file system/); + }), + }); + assert.strictEqual(fs.existsSync(tmpdir.resolve(kBlob)), false); +} diff --git a/test/sea/test-build-sea-vfs-incompatible-options.js b/test/sea/test-build-sea-vfs-incompatible-options.js index bd51ee12c166..e2ef11a01458 100644 --- a/test/sea/test-build-sea-vfs-incompatible-options.js +++ b/test/sea/test-build-sea-vfs-incompatible-options.js @@ -54,28 +54,6 @@ skipIfBuildSEAIsNotSupported(); }); } -// Test: "useVfs" with "useCodeCache" -{ - tmpdir.refresh(); - const config = tmpdir.resolve('vfs-code-cache.json'); - writeFileSync(config, ` -{ - "main": "bundle.js", - "output": "sea", - "useVfs": true, - "useCodeCache": true -} - `, 'utf8'); - spawnSyncAndAssert( - process.execPath, - ['--build-sea', config], { - cwd: tmpdir.path, - }, { - status: 1, - stderr: /"useVfs" is not supported when "useCodeCache" is true/, - }); -} - // Test: "vfsArchive" without "useVfs" { tmpdir.refresh(); diff --git a/test/sea/test-single-executable-application-vfs-code-cache.js b/test/sea/test-single-executable-application-vfs-code-cache.js new file mode 100644 index 000000000000..1ba4a5e8903c --- /dev/null +++ b/test/sea/test-single-executable-application-vfs-code-cache.js @@ -0,0 +1,128 @@ +'use strict'; + +// This tests that "useCodeCache" combined with "useVfs" embeds a compile +// cache entry for every module in the virtual file system into the +// preparation blob, and that the resulting executable runs. + +const common = require('../common'); + +const { + buildSEA, + skipIfBuildSEAIsNotSupported, +} = require('../common/sea'); + +skipIfBuildSEAIsNotSupported(); + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const { createWriteStream } = require('fs'); +const { pipeline } = require('stream/promises'); +const { zipFiles } = require('zlib'); +const tmpdir = require('../common/tmpdir'); +const fixtures = require('../common/fixtures'); +const { spawnSyncAndAssert } = require('../common/child_process'); + +// Every entry is stored as an asset under this reserved prefix, see +// src/node_sea.cc. +const kCompileCachePrefix = 'node:sea:compile-cache/'; + +function countCompileCacheEntries(blobPath) { + const blob = fs.readFileSync(blobPath).toString('latin1'); + return blob.split(kCompileCachePrefix).length - 1; +} + +// Generates the preparation blob of a fixture with "useCodeCache" and returns +// the number of compile cache entries it contains. +async function generateBlob(fixtureDir, prepare) { + tmpdir.refresh(); + fs.cpSync(fixtureDir, tmpdir.path, { recursive: true }); + await prepare?.(); + spawnSyncAndAssert( + process.execPath, + ['--experimental-sea-config', 'sea-config-code-cache.json'], + { + cwd: tmpdir.path, + env: { + ...process.env, + NODE_DEBUG_NATIVE: 'SEA', + }, + }, + { + stderr: /Generated \d+ compile cache entries for \d+ modules/, + }); + return countCompileCacheEntries(tmpdir.resolve('sea-prep.blob')); +} + +async function main() { + // CommonJS: sea.js, modules/math.js, modules/calculator.js and the two + // packages under node_modules; config.json and greeting.txt are data. + assert.strictEqual(await generateBlob(fixtures.path('sea', 'vfs')), 5); + + // ESM: main.mjs, modules/math.mjs, modules/calculator.cjs and the package. + assert.strictEqual(await generateBlob(fixtures.path('sea', 'vfs-esm')), 4); + + // From a ZIP archive: sea.js, modules/math.js and the package. + const zipFixture = (...args) => fixtures.path('sea', 'vfs-zip', ...args); + assert.strictEqual(await generateBlob(zipFixture(), () => pipeline( + zipFiles([ + [zipFixture('config.json'), 'config.json'], + [zipFixture('greeting.txt'), 'data/greeting.txt'], + [zipFixture('math.js'), 'modules/math.js'], + [zipFixture('test-pkg-package.json'), 'node_modules/test-pkg/package.json'], + [zipFixture('test-pkg-index.js'), 'node_modules/test-pkg/index.js'], + ]), + createWriteStream(tmpdir.resolve('assets.zip')), + )), 3); + + // The build is not affected by the compile cache configuration of the + // build process itself: the cache is generated although NODE_COMPILE_CACHE + // points at an unusable location and the compile cache is disabled. + tmpdir.refresh(); + fs.cpSync(fixtures.path('sea', 'vfs'), tmpdir.path, { recursive: true }); + spawnSyncAndAssert( + process.execPath, + ['--experimental-sea-config', 'sea-config-code-cache.json'], + { + cwd: tmpdir.path, + env: { + ...process.env, + NODE_COMPILE_CACHE: path.join(tmpdir.path, 'config.json', 'nope'), + NODE_DISABLE_COMPILE_CACHE: '1', + NODE_DEBUG_NATIVE: undefined, + }, + }, + { + stderr: /Wrote single executable preparation blob/, + }); + assert.strictEqual( + countCompileCacheEntries(tmpdir.resolve('sea-prep.blob')), 5); + + // The executable runs with the cache embedded, and the reserved assets + // are not visible as files of the virtual file system. + tmpdir.refresh(); + const outputFile = buildSEA(fixtures.path('sea', 'vfs'), { + configPath: 'sea-config-code-cache.json', + }); + spawnSyncAndAssert( + outputFile, + { + env: { + ...process.env, + NODE_DEBUG_NATIVE: undefined, + }, + }, + { + stdout(stdout) { + assert.match(stdout, /All SEA VFS tests passed!/); + assert.doesNotMatch(stdout, /node:sea:/); + }, + stderr(stderr) { + assert.doesNotMatch(stderr, /Code cache data rejected/); + assert.doesNotMatch(stderr, /ExperimentalWarning: VirtualFileSystem/); + }, + }, + ); +} + +main().then(common.mustCall());