diff --git a/.changeset/oauth-registration-infer-native.md b/.changeset/oauth-registration-infer-native.md new file mode 100644 index 0000000000..7c596899ac --- /dev/null +++ b/.changeset/oauth-registration-infer-native.md @@ -0,0 +1,5 @@ +--- +"@executor-js/mcp-auth": patch +--- + +Dynamic client registrations that omit `application_type` are native when a callback is an RFC 8252 loopback or private-use one, and every callback is held to the native rules. A client that registers a web callback beside a loopback or private-use one is no longer refused. diff --git a/.github/workflows/fork-ci.yml b/.github/workflows/fork-ci.yml index 690d2bc121..0822748f0a 100644 --- a/.github/workflows/fork-ci.yml +++ b/.github/workflows/fork-ci.yml @@ -42,3 +42,28 @@ jobs: bun run "e2e:$suite" echo "::endgroup::" done + + # Self-host scenarios for scoped-connection widening, OAuth registration, app + # User-Agent, and load-failure containment. Motel's collector build requires + # Bun 1.4.4, so this job does not share check's 1.3.11 pin. No extra secrets: + # the runner mints its own API key. CI disables the e2e prepare cache. + security-e2e: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.4.4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + - run: bun install --frozen-lockfile + - name: Install Playwright Chromium + run: bunx playwright install --with-deps chromium + - run: bun run e2e:prepare + - name: Security scenarios + run: | + bun run e2e:self-host -- --test-name 'application_type|fetch without a User-Agent names Executor|cannot create an all-apps connection|cannot widen a connection or mint|fails to load fails only its own reads' diff --git a/AGENTS.md b/AGENTS.md index 7ee0052389..84ebd19ba6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -304,9 +304,7 @@ A pull request that changes the release paths (the `paths` in `.github/workflows/release-artifacts.yml`) also runs "Executor releases". Gate its merge on that workflow as on CI: wait for both (`ci-wait --workflow CI --workflow "Executor releases"`) and do not merge while -either fails. Its packaging jobs fail when a native archive exceeds the npm budget -(`npmArchiveBudgetBytes` in `scripts/releases/config.ts`); an archive over it cannot be -published, so that failure blocks every later release until it is fixed. +either fails. A failed e2e job uploads raw reports and server logs. Product database files, runtime dependencies and private `actors.json` sessions are excluded. diff --git a/RELEASING.md b/RELEASING.md index ed5cf941ae..55b350f50d 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -108,19 +108,26 @@ The workflow refuses a channel that does not match the committed version. 1. Build and test every native runtime, desktop installer and Docker architecture. 2. Export the filtered source snapshot to the public repository's `v2-releases` branch. Create `executor@` on that public commit, never a private SHA. -3. Publish native npm variants, then the launcher with exact optional aliases. - Poll each public registry archive and compare its integrity before continuing. -4. Verify a clean `executor@` installation from npm. -5. Upload installers, native packages, bundled Git source and SHA256SUMS. -6. Publish the tested Docker manifest as `:` and `:beta`, then make the +3. Upload installers, native archives, the npm launcher, bundled Git source and + SHA256SUMS to the draft. +4. Publish the tested Docker manifest as `:` and `:beta`, then make the GitHub prerelease public with `latest=false`. -7. Point the desktop `beta` update channel at the public release assets. +5. Publish the npm launcher, which downloads the native archives from that public + release. It records each archive's SHA-256; publication checks them against + the build and the release's SHA256SUMS, then polls the registry archive and + compares its integrity. +6. Verify a clean `executor@` installation from npm, including its first + download. +7. Point the Homebrew formula and the desktop `beta` update channel at the public + release assets. The npm `latest` tag, Docker `latest`, and Executor 1 desktop updater stay unchanged. A draft public release blocks accidental repeat publication of the same version. After a partial failure, re-run the failed publish job. It resumes -only its own draft, whose target is this source's unchanged public snapshot, and -skips npm versions whose registry integrity matches the local archive. Any other +only its own release, whose target is this source's unchanged public snapshot: +a draft gets its assets again, and a release already public keeps them, which +npm publication then checks. It skips an npm launcher whose registry integrity +matches the local archive. Any other existing release or npm version stops publication for inspection. Never republish an accepted immutable npm version merely because its registry entry is still propagating. diff --git a/apps/cli/CHANGELOG.md b/apps/cli/CHANGELOG.md index a5708ccfdf..811c84f4cd 100644 --- a/apps/cli/CHANGELOG.md +++ b/apps/cli/CHANGELOG.md @@ -1,5 +1,9 @@ # executor +## 2.0.0-beta.36 + +Nightly beta of the Executor local runtime, CLI, desktop, and self-host Docker image. + ## 2.0.0-beta.35 Nightly beta of the Executor local runtime, CLI, desktop, and self-host Docker image. diff --git a/apps/cli/package.json b/apps/cli/package.json index 402c82a907..5713e1e820 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "executor", - "version": "2.0.0-beta.35", + "version": "2.0.0-beta.36", "private": true, "description": "Run Executor locally and connect your agents to your apps.", "homepage": "https://executor.sh", diff --git a/apps/docs/content/run/cli.mdx b/apps/docs/content/run/cli.mdx index 1503cfdb05..e5dbfe4234 100644 --- a/apps/docs/content/run/cli.mdx +++ b/apps/docs/content/run/cli.mdx @@ -22,10 +22,11 @@ browser and exits. Executor keeps running until you stop it; see with its runtime, dashboard, Git and workerd beside it; no source checkout, Node or Bun is needed to run it. -The npm package is a small launcher: npm installs the executable for your -platform as an optional dependency. If `executor` reports that its native -runtime is missing, npm skipped optional dependencies; install again with -`--include=optional`. +The npm package is a small launcher. The first time a version starts, it +downloads that release's archive for your platform from the public GitHub +release, checks it against the SHA-256 the package records, and keeps it in +`~/.executor/v2/launcher/`. Set `EXECUTOR_DOWNLOAD_URL` to use a mirror +of the releases. The install script keeps each release in `~/.executor/v2/runtime/versions/`, points `~/.executor/v2/runtime/current` at the one in use and links diff --git a/apps/docs/release-commands.ts b/apps/docs/release-commands.ts index d50832ece7..b031bc5d90 100644 --- a/apps/docs/release-commands.ts +++ b/apps/docs/release-commands.ts @@ -16,8 +16,8 @@ export const ReleaseCommandProduct = Schema.Literals([ /** Concrete instructions generated from the same identity as the published artifacts. */ export const releaseCommands = { cli: { - description: `With npm, on Node ${release.minimumNodeVersion} or newer. Executor carries its own runtime, so Node only starts it. Check that executor --version prints ${release.version} before starting. Supports macOS on Apple Silicon and Intel, Linux (glibc) on ARM64 and x64, and Windows on x64.`, - command: `${release.npmInstall} --include=optional\nexecutor --version\nexecutor`, + description: `With npm, on Node ${release.minimumNodeVersion} or newer. The package is a launcher that downloads Executor's own runtime for your platform on first run, so Node only starts it. Check that executor --version prints ${release.version} before starting. Supports macOS on Apple Silicon and Intel, Linux (glibc) on ARM64 and x64, and Windows on x64.`, + command: `${release.npmInstall}\nexecutor --version\nexecutor`, }, "cli-script": { description: diff --git a/apps/hosted/server/src/contracts/workflows.ts b/apps/hosted/server/src/contracts/workflows.ts index 4cd813ce89..e945a0e684 100644 --- a/apps/hosted/server/src/contracts/workflows.ts +++ b/apps/hosted/server/src/contracts/workflows.ts @@ -12,6 +12,7 @@ import { StartWorkflowFields, ListWorkflowRuns, WorkflowErrors, + WorkflowDefinitionErrors, } from "@executor-js/sdk/core"; import { OrganizationReference, @@ -21,6 +22,7 @@ import { const app = { organization: OrganizationReference, app: AppId }; const run = { ...app, run: WorkflowRunId }; const errors = [...WorkflowErrors, OrganizationForbidden] as const; +const definitionErrors = [...WorkflowDefinitionErrors, OrganizationForbidden] as const; const path = "/api/organizations/:organization/apps/:app"; /** * Members who can use the app read definitions and start runs with their accounts. Runs belong to @@ -34,7 +36,7 @@ export const HostedWorkflows = HttpApiGroup.make("workflows") withLegacyKeys({ version: "deployment" }), ), success: Schema.Array(HostedWorkflow), - error: errors, + error: definitionErrors, }), ) .add( diff --git a/apps/local/server/src/contracts/resources.ts b/apps/local/server/src/contracts/resources.ts index ed4f8fa529..704a210f19 100644 --- a/apps/local/server/src/contracts/resources.ts +++ b/apps/local/server/src/contracts/resources.ts @@ -11,6 +11,7 @@ import { StartWorkflowFields, ListWorkflowRuns, WorkflowErrors, + WorkflowDefinitionErrors, WebhookSubscription, WebhookErrors, WebhookId, @@ -34,7 +35,7 @@ export const DashboardWorkflows = HttpApiGroup.make("workflows") withLegacyKeys({ version: "deployment" }), ), success: Schema.Array(HostedWorkflow), - error: WorkflowErrors, + error: WorkflowDefinitionErrors, }), ) .add( diff --git a/apps/local/server/src/implementation/upgrade.ts b/apps/local/server/src/implementation/upgrade.ts index 2674522c27..4bb952e73b 100644 --- a/apps/local/server/src/implementation/upgrade.ts +++ b/apps/local/server/src/implementation/upgrade.ts @@ -72,14 +72,14 @@ export interface UpgradeOptions { } /** - * Each package manager's global install. npm can skip the optional dependency that carries the - * native runtime, which leaves a launcher that cannot start, so it must be asked to include it. + * Each package manager's global install. It installs only the launcher, which downloads the + * release's runtime the first time it starts. */ const installArgs = (method: GlobalInstaller, version: string): ReadonlyArray => { const spec = `executor@${version}`; switch (method) { case "npm": - return ["install", "-g", spec, "--include=optional"]; + return ["install", "-g", spec]; case "pnpm": return ["add", "-g", spec]; case "bun": @@ -402,7 +402,7 @@ const rollback = (options: UpgradeOptions, host: UpgradeHost) => if (Option.isNone(install)) return yield* failed( "not-standalone", - "Only an install from the Executor install script keeps a previous version to roll back to. Install the version you want with the package manager you used, such as: npm install -g executor@ --include=optional. Nothing was changed.", + "Only an install from the Executor install script keeps a previous version to roll back to. Install the version you want with the package manager you used, such as: npm install -g executor@. Nothing was changed.", ); const previous = yield* linkedVersion(install.value.root, "previous"); if (Option.isNone(previous)) @@ -472,7 +472,7 @@ export const upgrade = (options: UpgradeOptions, host: UpgradeHost) => if (Option.isNone(current)) return yield* failed( "development-build", - "This is a development build of Executor without a release version, so it cannot upgrade itself. Install a release with: npm install -g executor@beta --include=optional", + "This is a development build of Executor without a release version, so it cannot upgrade itself. Install a release with: npm install -g executor@beta", ); const registry = yield* npmRegistry; @@ -530,7 +530,7 @@ export const upgrade = (options: UpgradeOptions, host: UpgradeHost) => if (Option.isNone(install)) return yield* failed( "not-standalone", - `${host.executable} is not in an install script's versions directory, so executor upgrade cannot replace it. Reinstall with the Executor install script, or with npm: npm install -g executor@${target} --include=optional. Nothing was changed.`, + `${host.executable} is not in an install script's versions directory, so executor upgrade cannot replace it. Reinstall with the Executor install script, or with npm: npm install -g executor@${target}. Nothing was changed.`, ); if (compareReleaseVersions(target, current.value) < 0) yield* confirm( @@ -642,10 +642,14 @@ export const upgrade = (options: UpgradeOptions, host: UpgradeHost) => "verify-failed", `${method} installed Executor v${target}, but ${detail} Run \`${host.platform === "win32" ? "where.exe executor" : "type -a executor"}\` to see which executable your shell starts, or reinstall with: ${command}`, ); + // The new launcher's first start downloads and checks its release's runtime. + yield* Console.log( + `Starting Executor v${target}. Its first start downloads the runtime for this platform.`, + ); const check = yield* runInstalled( onPath(host.platform), ["--version"], - Duration.seconds(60), + Duration.minutes(15), ).pipe(Effect.result); if (Result.isFailure(check)) return yield* verify("`executor --version` could not be started."); diff --git a/bun.lock b/bun.lock index e8823e1265..36454e7954 100644 --- a/bun.lock +++ b/bun.lock @@ -40,7 +40,7 @@ }, "apps/cli": { "name": "executor", - "version": "2.0.0-beta.35", + "version": "2.0.0-beta.36", }, "apps/docs": { "name": "@executor-js/docs", diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index 970e0faa5d..52bbe8dfc6 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -1437,6 +1437,17 @@ export const scenarios = plan({ cloud: na("Hosted Tools pages are covered through organization routes."), }, }, + appLoadFailureContainment: { + fixtures: "actors", + file: "app-load-failure-containment.spec.ts", + title: + "An app whose code fails to load fails only its own reads and names its error in every preview", + targets: { + "self-host": scheduled, + cloud: scheduled, + local: na("The hosted app overview and dashboard batches own this presentation."), + }, + }, toolsErrorState: { fixtures: "actors", file: "tools-error-state.spec.ts", @@ -2755,6 +2766,16 @@ export const scenarios = plan({ local: na("Exercises the hosted import and account connection APIs."), }, }, + appFetchUserAgent: { + fixtures: "actors", + file: "app-fetch-user-agent.spec.ts", + title: "An app's fetch without a User-Agent names Executor and keeps an app's own", + targets: { + "self-host": scheduled, + cloud: na("Uses a loopback upstream to exercise the shared outbound network."), + local: na("Exercises the hosted app deploy, account check and tool call APIs."), + }, + }, mcpUserAgent: { fixtures: "actors", file: "mcp-user-agent.spec.ts", @@ -4321,6 +4342,12 @@ export const scenarios = plan({ title: "OAuth resources are provisioned before client registration", targets: { local: scheduled, "self-host": scheduled, cloud: scheduled }, }, + oauthRegistrationApplicationType: { + file: "oauth-registration-application-type.spec.ts", + title: + "a registration without application_type is native for a loopback or private-use callback, held to the native rules, and web otherwise", + targets: { local: scheduled, "self-host": scheduled, cloud: scheduled }, + }, apiDocumentPatterns: { file: "api-document-patterns.spec.ts", title: "the published API document keeps the string patterns requests must match", diff --git a/e2e/tests/app-fetch-user-agent.spec.ts b/e2e/tests/app-fetch-user-agent.spec.ts new file mode 100644 index 0000000000..5e3a7e96df --- /dev/null +++ b/e2e/tests/app-fetch-user-agent.spec.ts @@ -0,0 +1,177 @@ +/** + * App code's own `fetch` goes out through Executor's outbound network. Workers send no User-Agent, + * and GitHub's API refuses a request without one, so the outbound names Executor when app code + * names no client and keeps the one it does. + */ +import { expect, layer } from "@effect/vitest"; +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; +import { Effect, Layer, Schema } from "effect"; +import { HttpRouter, HttpServer, HttpServerRequest, HttpServerResponse } from "effect/http"; +import { createServer } from "node:http"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { appsManifest } from "../support/apps-release.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { App, Resource } from "../support/contracts.ts"; +import { connectionsPath } from "../support/app-accounts.ts"; +import { Evidence } from "../support/evidence.ts"; +import { scenarios } from "../test-plan.ts"; + +const token = "synthetic-user-agent-token"; + +const Health = Schema.Struct({ + info: Schema.NullOr(Schema.Struct({ username: Schema.optional(Schema.String) })), + apps: Schema.Array( + Schema.Struct({ + app: Schema.String, + check: Schema.NullOr( + Schema.Struct({ status: Schema.String, message: Schema.optionalKey(Schema.String) }), + ), + }), + ), +}); + +/** + * The check reads the current user as GitHub's catalog app does, naming no client. The tool sends + * the User-Agent it is given, or none, and returns the one the service saw. + */ +const source = (origin: string) => `import { + decodeJson, defineApp, defineProvider, object, query, router, secrets, string, +} from "apps"; +const User = object({ login: string() }); +const Client = object({ userAgent: string() }); +const service = defineProvider({ + name: "User-Agent service", + auth: { token: secrets({ label: "Token", fields: object({ token: string() }) }) }, + async health({ account, fetch, signal }) { + const response = await fetch(${JSON.stringify(`${origin}/user`)}, { + signal, + headers: { Authorization: "Bearer " + account.fields.token, Accept: "application/json" }, + }); + const me = await decodeJson(response, User); + return { accountInfo: { username: me.login } }; + }, +}); +export default defineApp({ user: { accounts: { service } } }, { + tools: router({ + client: query({ input: object({ userAgent: string().optional() }) }, async (_ctx, { userAgent }) => { + const response = await fetch(${JSON.stringify(`${origin}/client`)}, { + headers: userAgent === undefined ? {} : { "User-Agent": userAgent }, + }); + return decodeJson(response, Client); + }), + }), +});`; + +layer(HostedLive, { excludeTestServices: true })("App fetch User-Agent", (it) => { + it.effect(scenarios.appFetchUserAgent.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors, + evidence = yield* Evidence; + // Answers as GitHub's API does: 403 with an HTML page for a request without a User-Agent. + const identified = ( + respond: ( + request: HttpServerRequest.HttpServerRequest, + userAgent: string, + ) => ReturnType, + ) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const userAgent = request.headers["user-agent"]; + if (!userAgent) + return HttpServerResponse.text( + "Request forbidden by administrative rules. Please make sure your request has a User-Agent header", + { status: 403, contentType: "text/html; charset=utf-8" }, + ); + return yield* respond(request, userAgent); + }); + const routes = Layer.mergeAll( + HttpRouter.add( + "GET", + "/user", + identified((request) => + request.headers.authorization === `Bearer ${token}` + ? HttpServerResponse.json({ login: "synthetic-user" }) + : HttpServerResponse.json({ message: "Bad credentials" }, { status: 401 }), + ), + ), + HttpRouter.add( + "GET", + "/client", + identified((_request, userAgent) => HttpServerResponse.json({ userAgent })), + ), + ); + const services = yield* Layer.build( + HttpRouter.serve(routes, { disableLogger: true, disableListenLog: true }).pipe( + Layer.provideMerge(NodeHttpServer.layer(createServer, { host: "127.0.0.1", port: 0 })), + ), + ); + const server = yield* HttpServer.HttpServer.pipe(Effect.provideContext(services)); + if (!("port" in server.address)) return yield* Effect.die("Fixture must listen on TCP"); + const origin = `http://127.0.0.1:${server.address.port}`; + + const prefix = `/api/organizations/${actors.organization.id}`; + const deployed = yield* api.request(actors.owner, "POST", `${prefix}/apps/deploy`, { + name: "User-Agent proof", + files: [{ path: "index.ts", content: source(origin) }, appsManifest], + }); + expect(deployed.status, JSON.stringify(deployed.body)).toBe(200); + const app = yield* body(App, deployed); + const path = `${prefix}/apps/${app.id}`; + let account: string | undefined; + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + yield* api.request(actors.owner, "DELETE", path); + if (account !== undefined) + yield* api.request(actors.owner, "DELETE", `${prefix}/accounts/${account}`); + }).pipe(Effect.orDie), + ); + const connection = yield* body( + Resource, + yield* api.request(actors.owner, "POST", connectionsPath(path, "user", "service"), {}), + ); + const saved = yield* api.request( + actors.owner, + "POST", + `${prefix}/connections/${connection.id}/submit`, + { method: "token", fields: { token } }, + ); + expect(saved.status, JSON.stringify(saved.body)).toBe(200); + account = (yield* body(Resource, saved)).id; + + // The check that names no client passes and reports the service's username. + const checked = yield* api.request( + actors.owner, + "POST", + `${prefix}/accounts/${account}/health`, + ); + yield* evidence.json("health.json", checked); + expect(checked.status, JSON.stringify(checked.body)).toBe(200); + const health = yield* body(Health, checked); + const check = health.apps.find((entry) => entry.app === app.id)?.check; + expect(check?.status, JSON.stringify(check)).toBe("healthy"); + expect(health.info?.username).toBe("synthetic-user"); + + // A tool's request names Executor by default and keeps the client app code names. + for (const userAgent of [undefined, "Custom client/1.0"]) { + const called = yield* api.request(actors.owner, "POST", `${path}/tools/call`, { + tool: "client", + kind: "query", + input: userAgent === undefined ? {} : { userAgent }, + }); + yield* evidence.json( + userAgent === undefined ? "default-client.json" : "custom-client.json", + called, + ); + expect(called.status, JSON.stringify(called.body)).toBe(200); + expect(yield* body(Schema.Struct({ userAgent: Schema.String }), called)).toEqual({ + userAgent: userAgent ?? "Executor", + }); + } + }), + ), + ); +}); diff --git a/e2e/tests/app-load-failure-containment.spec.ts b/e2e/tests/app-load-failure-containment.spec.ts new file mode 100644 index 0000000000..61da0556b7 --- /dev/null +++ b/e2e/tests/app-load-failure-containment.spec.ts @@ -0,0 +1,196 @@ +/** + * An app whose code throws while it loads fails only that app's reads. Its workflow listing + * reports the app's own evaluation error, as its tool and skill listings do, instead of an opaque + * workflow failure. A dashboard batch that carries its reads still answers every other read, + * including another app's, and the other app's page still loads. + */ +import { expect, layer } from "@effect/vitest"; +import { Effect, Schema } from "effect"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { appsManifest } from "../support/apps-release.ts"; +import { Browser } from "../support/browser.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { openThroughBrowser } from "../support/in-app-navigation.ts"; +import { batchPath } from "../support/read-batches.ts"; +import { scenarios } from "../test-plan.ts"; + +const App = Schema.Struct({ id: Schema.String }); +const Workflows = Schema.Array(Schema.Struct({ name: Schema.String })); +const EvaluationFailure = Schema.Struct({ + _tag: Schema.Literal("AppEvaluationFailed"), + app: Schema.String, + failure: Schema.Struct({ + source: Schema.String, + errorName: Schema.String, + message: Schema.String, + }), +}); +const Answers = Schema.Array( + Schema.Struct({ id: Schema.Number, status: Schema.Number, text: Schema.optional(Schema.String) }), +); + +const broken = `import { defineApp } from "apps"; +export default defineApp({}, async () => { + throw new Error("SYNTHETIC_LOAD_FAILURE"); +});`; +const healthy = `import { defineApp, query, object, router, string, workflow } from "apps"; +const record = workflow( + { description: "Record a label", input: object({ label: string() }) }, + async (_ctx, input) => input.label, +); +export default defineApp({}, { + tools: router({ + status: query({ description: "Service status", input: object({}) }, async () => "ready"), + }), + workflows: { record }, +});`; + +layer(HostedLive, { excludeTestServices: true })("App load failure containment", (it) => { + it.effect(scenarios.appLoadFailureContainment.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const actors = yield* Actors; + const api = yield* Api; + const browser = yield* Browser; + const organization = actors.organization.id; + const prefix = `/api/organizations/${organization}`; + const deploy = (name: string, content: string) => + Effect.gen(function* () { + const response = yield* api.request(actors.owner, "POST", `${prefix}/apps/deploy`, { + name, + files: [{ path: "index.ts", content }, appsManifest], + }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + const app = yield* body(App, response); + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", `${prefix}/apps/${app.id}`).pipe(Effect.orDie), + ); + return app; + }); + const failing = yield* deploy("Failing loader", broken); + const working = yield* deploy("Working reports", healthy); + + // Listing the failing app's workflows reports its own error, as listing its tools does. + const failed = yield* api.request( + actors.owner, + "GET", + `${prefix}/apps/${failing.id}/workflows`, + ); + expect(failed.status, JSON.stringify(failed.body)).toBe(502); + expect(yield* body(EvaluationFailure, failed)).toEqual({ + _tag: "AppEvaluationFailed", + app: failing.id, + failure: { source: "app", errorName: "Error", message: "SYNTHETIC_LOAD_FAILURE" }, + }); + const listed = yield* api.request( + actors.owner, + "GET", + `${prefix}/apps/${working.id}/workflows`, + ); + expect(listed.status, JSON.stringify(listed.body)).toBe(200); + expect((yield* body(Workflows, listed)).map(({ name }) => name)).toEqual(["record"]); + + yield* browser.login(actors.owner); + yield* browser.use("Open the dashboard", (page) => + page.goto(`/org/${actors.organization.slug}/apps`), + ); + // One batch, as the dashboard sends it, with the failing app's reads among the others. + const reads = [ + { group: "organization", endpoint: "inventory", app: undefined }, + { group: "workflows", endpoint: "definitions", app: failing.id }, + { group: "skills", endpoint: "list", app: failing.id }, + { group: "tools", endpoint: "index", app: failing.id }, + { group: "workflows", endpoint: "definitions", app: working.id }, + { group: "tools", endpoint: "index", app: working.id }, + ].map(({ app, ...read }, id) => ({ + id, + ...read, + params: app === undefined ? { organization } : { organization, app }, + query: {}, + })); + const text = yield* browser.use("Send one batch from the signed-in page", (page) => + page.evaluate( + ([path, payload]) => + fetch(path, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ reads: payload }), + }).then((response) => + response.status === 200 + ? response.text() + : Promise.reject(new Error(`The batch answered ${response.status}`)), + ), + [batchPath, reads] as const, + ), + ); + const answers = Schema.decodeUnknownSync(Answers)( + text + .split("\n") + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line)), + ); + const label = (id: number) => { + const read = reads[id]; + if (read === undefined) return `unknown read ${id}`; + const app = "app" in read.params ? read.params.app : "organization"; + return `${read.group}.${read.endpoint} ${app}`; + }; + const statuses = Object.fromEntries(answers.map(({ id, status }) => [label(id), status])); + expect(statuses).toEqual({ + "organization.inventory organization": 200, + [`workflows.definitions ${failing.id}`]: 502, + [`skills.list ${failing.id}`]: 502, + [`tools.index ${failing.id}`]: 502, + [`workflows.definitions ${working.id}`]: 200, + [`tools.index ${working.id}`]: 200, + }); + for (const answer of answers.filter(({ status }) => status === 502)) + expect(answer.text, label(answer.id)).toContain("SYNTHETIC_LOAD_FAILURE"); + + yield* openThroughBrowser( + "Open the failing app's overview", + `/org/${actors.organization.slug}/apps/${failing.id}`, + ); + const workflows = yield* browser.use( + "The workflows preview names the app's error", + (page) => + page + .getByRole("region", { name: "App workflows preview", exact: true }) + .getByRole("alert", { name: "Tools could not be loaded", exact: true }) + .innerText(), + ); + expect(workflows).toContain("The app threw Error: SYNTHETIC_LOAD_FAILURE"); + expect(workflows).not.toContain("Workflow request failed"); + yield* browser.use("The skills preview names the same error", (page) => + page + .getByRole("region", { name: "App skills preview", exact: true }) + .getByRole("alert", { name: "Tools could not be loaded", exact: true }) + .waitFor(), + ); + yield* browser.use("The rest of the app's page still renders", (page) => + page.getByText("Failing loader", { exact: true }).first().waitFor(), + ); + yield* browser.checkpoint("Failing app overview"); + + yield* openThroughBrowser( + "Open the working app's overview in the same session", + `/org/${actors.organization.slug}/apps/${working.id}`, + ); + yield* browser.use("The working app's workflows still load", (page) => + page + .getByRole("region", { name: "App workflows preview", exact: true }) + .getByText("record", { exact: true }) + .waitFor(), + ); + expect( + yield* browser.use("The working app's overview shows no failure", (page) => + page.getByRole("alert").count(), + ), + ).toBe(0); + yield* browser.checkpoint("Working app overview"); + }), + ), + ); +}); diff --git a/e2e/tests/local-upgrade.spec.ts b/e2e/tests/local-upgrade.spec.ts index d0c0f10849..c3a48f2af9 100644 --- a/e2e/tests/local-upgrade.spec.ts +++ b/e2e/tests/local-upgrade.spec.ts @@ -376,11 +376,10 @@ layer(TestLive, { excludeTestServices: true })("Local upgrade", (it) => { return yield* Effect.die("Registry must listen on TCP"); const registryUrl = `http://127.0.0.1:${registry.address.port}`; - const platformPackage = `executor-${process.platform}-${process.arch}`; /** * A launcher at `location` that starts the CLI, as a package manager's install would: the - * npm launcher with the release executable's package beside it, or a script that imports - * the source entry. + * npm launcher, whose runtime for this version is already downloaded (the release + * executable's directory), or a script that imports the source entry. */ const installedAt = (location: string) => Effect.gen(function* () { @@ -401,14 +400,15 @@ layer(TestLive, { excludeTestServices: true })("Local upgrade", (it) => { name: "executor", version: "2.0.0-beta.2", type: "module", - optionalDependencies: { [platformPackage]: "*" }, + // Never checked: the runtime below is already in place. + runtimes: { [`${process.platform}-${process.arch}`]: "0".repeat(64) }, }), ); - yield* fs.makeDirectory(path.join(directory, "node_modules")); - yield* fs.symlink( - path.dirname(entry), - path.join(directory, "node_modules", platformPackage), - ); + const runtime = path.join(root, ".executor", "v2", "launcher", "2.0.0-beta.2"); + if (!(yield* fs.exists(runtime))) { + yield* fs.makeDirectory(path.dirname(runtime), { recursive: true }); + yield* fs.symlink(path.dirname(entry), runtime); + } return launcher; }); const data = path.join(root, "data"); @@ -420,6 +420,8 @@ layer(TestLive, { excludeTestServices: true })("Local upgrade", (it) => { SystemRoot: process.env.SystemRoot ?? "", ComSpec: process.env.ComSpec ?? "", PATHEXT: process.env.PATHEXT ?? "", + // The npm launcher keeps its runtimes in the home directory, as on macOS and Linux. + USERPROFILE: root, } : {}), HOME: root, @@ -474,14 +476,14 @@ layer(TestLive, { excludeTestServices: true })("Local upgrade", (it) => { published = { beta: "2.999.0-beta.1", latest: "2.999.0" }; const upgraded = `Upgraded Executor from v${version} to v2.999.0.`; - // npm installs globally and includes the optional dependency with the native runtime. + // npm installs the launcher globally; it downloads the native runtime when it first starts. const byNpm = yield* run(npm, ["upgrade"]); yield* evidence.json("npm.json", byNpm); expect(byNpm.code, byNpm.stderr).toBe(0); expect(byNpm.stdout).toContain("installed executor@2.999.0 with fake npm"); expect(byNpm.stdout).toContain(upgraded); expect(yield* calls).toEqual([ - { name: "npm", args: ["install", "-g", "executor@2.999.0", "--include=optional"] }, + { name: "npm", args: ["install", "-g", "executor@2.999.0"] }, ]); // Each global package manager is detected from where it installs packages. @@ -525,7 +527,7 @@ layer(TestLive, { excludeTestServices: true })("Local upgrade", (it) => { "Executor could not tell which package manager installed it", ); for (const command of [ - "npm install -g executor@2.999.0 --include=optional", + "npm install -g executor@2.999.0", "pnpm add -g executor@2.999.0", "bun add -g executor@2.999.0", "yarn global add executor@2.999.0", @@ -579,7 +581,7 @@ layer(TestLive, { excludeTestServices: true })("Local upgrade", (it) => { `Upgraded Executor from v${version} to v2.0.0-beta.1.`, ); expect(yield* calls).toEqual([ - { name: "npm", args: ["install", "-g", "executor@2.0.0-beta.1", "--include=optional"] }, + { name: "npm", args: ["install", "-g", "executor@2.0.0-beta.1"] }, ]); // A background server for this data directory, which every scenario below stops. diff --git a/e2e/tests/npm-launcher.spec.ts b/e2e/tests/npm-launcher.spec.ts index 768a264ab0..fe7876855b 100644 --- a/e2e/tests/npm-launcher.spec.ts +++ b/e2e/tests/npm-launcher.spec.ts @@ -1,19 +1,39 @@ /** - * The `executor` npm package is a launcher: it starts the executable from the platform package npm - * installs as an optional dependency, passes signals on and exits as the executable exits. + * The `executor` npm package is a launcher. It holds no runtime: the first start of a version + * downloads that release's archive for this platform, checks it against the SHA-256 the launcher's + * package.json records, and unpacks it into ~/.executor/v2/launcher/. Later starts run it + * from there, pass signals on and exit as the executable exits. * * Package managers create a command shim from the launcher's declared bin path. Executor 1 declared * `bin/executor`; Executor 2 declares `bin.mjs`. An upgrade that replaces the package without * relinking, such as `bun add executor@beta` run inside Bun's global directory, leaves the old shim * starting `bin/executor`, so the launcher must keep that path working. The launcher needs no * particular Node: set EXECUTOR_E2E_OLD_NODE to the oldest Node its package.json accepts to start - * it there too. + * it there too, including its first download. + * + * Through 2.0.0-beta.36 each runtime was published as a version of the `executor` package itself, + * such as `2.0.0-beta.36-win32-x64`, which semver ranks above the launcher it belongs to. A + * `bun update -g` then replaced the launcher with a runtime archive, which has no `bin.mjs`. Those + * versions are deprecated, so package managers skip them; the second scenario installs and updates + * the launcher with Bun from a registry that still lists one. + * + * EXECUTOR_E2E_LAUNCHER_ARCHIVE is the packed launcher (`bun run release:wrapper`) and + * EXECUTOR_E2E_RUNTIME_ARCHIVE this platform's release archive, which the launcher records. */ import { expect, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { Config, Deferred, Effect, FileSystem, Option, Path, Schema, Stream } from "effect"; +import { createHash } from "node:crypto"; +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; +import { createServer } from "node:http"; +import { Config, Deferred, Effect, FileSystem, Layer, Option, Path, Schema, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; -import { FetchHttpClient } from "effect/http"; +import { + FetchHttpClient, + HttpRouter, + HttpServer, + HttpServerRequest, + HttpServerResponse, +} from "effect/http"; import { localNpmRegistry } from "../support/npm-registry.ts"; import { connectionRefused, freePort } from "../support/ports.ts"; @@ -22,87 +42,189 @@ const Manifest = Schema.fromJsonString( version: Schema.NonEmptyString, bin: Schema.Struct({ executor: Schema.Literal("bin.mjs") }), engines: Schema.Struct({ node: Schema.String }), + runtimes: Schema.Record(Schema.String, Schema.String), }), ); +const target = `${process.platform}-${process.arch}`; +const executableName = process.platform === "win32" ? "executor.exe" : "executor"; + +/** + * Serve `files` by URL path on loopback, counting requests per path, for the scenario's scope: + * the public release's downloads and, for Bun, the npm registry. + */ +const serve = (files: Map) => + Effect.gen(function* () { + const requests = new Map(); + const handler = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const url = decodeURIComponent(request.url.split("?")[0] ?? "/"); + requests.set(url, (requests.get(url) ?? 0) + 1); + const body = files.get(url); + return body === undefined + ? HttpServerResponse.empty({ status: 404 }) + : HttpServerResponse.uint8Array(body); + }); + const services = yield* Layer.build( + HttpRouter.serve(HttpRouter.add("GET", "/*", handler), { + disableLogger: true, + disableListenLog: true, + }).pipe( + Layer.provideMerge(NodeHttpServer.layer(createServer, { host: "127.0.0.1", port: 0 })), + ), + ); + const server = yield* HttpServer.HttpServer.pipe(Effect.provideContext(services)); + if (!("port" in server.address)) return yield* Effect.die("The server must listen on TCP"); + return { port: server.address.port, requests }; + }); + +const scenario = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const processes = yield* ChildProcessSpawner.ChildProcessSpawner; + const launcherArchive = path.resolve(yield* Config.String("EXECUTOR_E2E_LAUNCHER_ARCHIVE")); + const runtimeArchive = path.resolve(yield* Config.String("EXECUTOR_E2E_RUNTIME_ARCHIVE")); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "executor-npm-launcher-" }); + const home = path.join(root, "home"); + yield* fs.makeDirectory(home); + + const run = ( + command: string, + args: ReadonlyArray, + cwd: string, + env: Readonly> = {}, + ) => + Effect.scoped( + Effect.gen(function* () { + const child = yield* processes.spawn( + ChildProcess.make(command, args, { + cwd, + // Release scenarios never send product analytics, even from a build with a baked key. + // The launcher keeps its runtimes in the home directory. + env: { DO_NOT_TRACK: "1", HOME: home, USERPROFILE: home, ...env }, + extendEnv: true, + stdout: "pipe", + stderr: "pipe", + }), + ); + const [code, stdout, stderr] = yield* Effect.all( + [ + child.exitCode, + child.stdout.pipe(Stream.decodeText(), Stream.mkString), + child.stderr.pipe(Stream.decodeText(), Stream.mkString), + ], + { concurrency: "unbounded" }, + ).pipe(Effect.timeout("150 seconds")); + return { code: Number(code), stdout: stdout.trim(), stderr }; + }), + ); + // Unpack as npm lays out an install: the archive's package/ becomes `directory`. + const unpack = Effect.fnUntraced(function* (archive: string, directory: string) { + const staging = path.join(root, `unpack-${path.basename(directory)}`); + yield* fs.makeDirectory(staging); + yield* fs.copyFile(archive, path.join(staging, "package.tgz")); + // A relative archive path: GNU tar on Windows reads a drive letter as a remote host. + const result = yield* run("tar", ["-xzf", "package.tgz"], staging); + expect(result.code, result.stderr).toBe(0); + yield* fs.makeDirectory(path.dirname(directory), { recursive: true }); + yield* fs.rename(path.join(staging, "package"), directory); + return directory; + }); + const launcher = yield* unpack(launcherArchive, path.join(root, "node_modules", "executor")); + const manifest = yield* fs + .readFileString(path.join(launcher, "package.json")) + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Manifest))); + const runtimeBytes = yield* fs.readFile(runtimeArchive); + expect(manifest.runtimes[target]).toBe(createHash("sha256").update(runtimeBytes).digest("hex")); + const releaseFile = `/executor@${manifest.version}/executor-${manifest.version}-${target}.tgz`; + const runtimes = path.join(home, ".executor", "v2", "launcher"); + const installed = path.join(runtimes, manifest.version); + return { + fs, + path, + processes, + root, + home, + run, + launcher, + manifest, + runtimeBytes, + releaseFile, + runtimes, + installed, + }; +}); + it.live( - "the npm launcher starts the platform executable through every entry point, passes on its exit code and signals, and explains a missing platform package", + "the npm launcher downloads and checks its runtime once, starts it through every entry point, passes on its exit code and signals, and refuses a download that does not match", () => Effect.scoped( Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const processes = yield* ChildProcessSpawner.ChildProcessSpawner; - const launcherArchive = path.resolve(yield* Config.String("EXECUTOR_E2E_LAUNCHER_ARCHIVE")); - const runtimeArchive = path.resolve(yield* Config.String("EXECUTOR_E2E_RUNTIME_ARCHIVE")); + const { + fs, + path, + processes, + root, + home, + run, + launcher, + manifest, + runtimeBytes, + releaseFile, + runtimes, + installed, + } = yield* scenario; const oldNode = yield* Config.NonEmptyString("EXECUTOR_E2E_OLD_NODE").pipe(Config.option); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "executor-npm-launcher-" }); - const modules = path.join(root, "node_modules"); - yield* fs.makeDirectory(modules); - - const run = ( - command: string, - args: ReadonlyArray, - cwd: string, - env: Readonly> = {}, - ) => - Effect.scoped( - Effect.gen(function* () { - const child = yield* processes.spawn( - ChildProcess.make(command, args, { - cwd, - // Release scenarios never send product analytics, even from a build with a baked key. - env: { DO_NOT_TRACK: "1", ...env }, - extendEnv: true, - stdout: "pipe", - stderr: "pipe", - }), - ); - const [code, stdout, stderr] = yield* Effect.all( - [ - child.exitCode, - child.stdout.pipe(Stream.decodeText(), Stream.mkString), - child.stderr.pipe(Stream.decodeText(), Stream.mkString), - ], - { concurrency: "unbounded" }, - ).pipe(Effect.timeout("60 seconds")); - return { code: Number(code), stdout: stdout.trim(), stderr }; - }), - ); - // Unpack as npm lays out an install: each archive's package/ becomes node_modules/. - const unpack = Effect.fnUntraced(function* (archive: string, name: string) { - const staging = path.join(root, `unpack-${name}`); - yield* fs.makeDirectory(staging); - yield* fs.copyFile(archive, path.join(staging, "package.tgz")); - // A relative archive path: GNU tar on Windows reads a drive letter as a remote host. - const result = yield* run("tar", ["-xzf", "package.tgz"], staging); - expect(result.code, result.stderr).toBe(0); - yield* fs.rename(path.join(staging, "package"), path.join(modules, name)); - return path.join(modules, name); - }); - - const launcher = yield* unpack(launcherArchive, "executor"); - // The launcher's optional dependency alias for this machine's native runtime. - const platformPackage = `executor-${process.platform}-${process.arch}`; - const runtime = yield* unpack(runtimeArchive, platformPackage); - const manifest = yield* fs - .readFileString(path.join(launcher, "package.json")) - .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Manifest))); + const files = new Map([[releaseFile, runtimeBytes]]); + const releases = yield* serve(files); + const download = { EXECUTOR_DOWNLOAD_URL: `http://127.0.0.1:${releases.port}/` }; const entry = path.join(launcher, "bin.mjs"); - const executable = path.join( - runtime, - process.platform === "win32" ? "executor.exe" : "executor", + + // Versions from earlier downloads: the newest stays, as the version a background Executor + // started before an update may run from; older ones and abandoned downloads go. + const day = 24 * 60 * 60 * 1000; + const now = Date.now(); + for (const [name, age] of [ + ["2.0.0-beta.1", 2 * day], + ["2.0.0-beta.2", day], + [".download-1-abandoned", 2 * day], + ] as const) { + yield* fs.makeDirectory(path.join(runtimes, name), { recursive: true }); + const time = new Date(now - age); + yield* fs.utimes(path.join(runtimes, name), time, time); + } + + // The first start downloads, on the oldest Node the launcher supports when one is given. + const first = yield* run( + Option.getOrElse(oldNode, () => process.execPath), + [entry, "--version"], + root, + download, + ); + expect(first.code, first.stderr).toBe(0); + expect(first.stdout).toBe(`executor v${manifest.version}`); + expect(first.stderr).toContain( + `Downloading Executor ${manifest.version} for ${process.platform}-${process.arch}`, + ); + expect(releases.requests.get(releaseFile)).toBe(1); + expect( + yield* fs.exists( + path.join(installed, process.platform === "win32" ? "executor.exe" : "executor"), + ), + ).toBe(true); + expect((yield* fs.readDirectory(runtimes)).sort()).toEqual( + ["2.0.0-beta.2", manifest.version].sort(), ); for (const file of ["bin.mjs", "bin/executor"]) { const script = path.join(launcher, file); // Windows shims (npm's .cmd and .ps1, Bun's .exe) start the entry with Node. - const viaNode = yield* run(process.execPath, [script, "--version"], root); + const viaNode = yield* run(process.execPath, [script, "--version"], root, download); expect(viaNode.code, viaNode.stderr).toBe(0); expect(viaNode.stdout).toBe(`executor v${manifest.version}`); // Unix shims are symlinks to the entry, started through its shebang. if (process.platform !== "win32") { - const viaShebang = yield* run(script, ["--version"], root); + const viaShebang = yield* run(script, ["--version"], root, download); expect(viaShebang.code, viaShebang.stderr).toBe(0); expect(viaShebang.stdout).toBe(`executor v${manifest.version}`); } @@ -110,12 +232,12 @@ it.live( if (Option.isSome(oldNode)) { const old = yield* run(oldNode.value, ["--version"], root); expect(manifest.engines.node).toBe(`>=${old.stdout.replace(/^v/u, "")}`); - const viaOldNode = yield* run(oldNode.value, [entry, "--version"], root); - expect(viaOldNode.code, viaOldNode.stderr).toBe(0); - expect(viaOldNode.stdout).toBe(`executor v${manifest.version}`); } + // Every later start runs the downloaded runtime. + expect(releases.requests.get(releaseFile)).toBe(1); // A failing command exits with the executable's code and message. + const executable = path.join(installed, executableName); const direct = yield* run(executable, ["web"], root); const launched = yield* run(process.execPath, [entry, "web"], root); expect(direct.code).toBe(1); @@ -133,6 +255,7 @@ it.live( cwd: root, env: { DO_NOT_TRACK: "1", + HOME: home, EXECUTOR_DATA_DIR: path.join(root, "data"), EXECUTOR_PORT: String(port), EXECUTOR_KEY_STORAGE: "file", @@ -173,14 +296,189 @@ it.live( expect(yield* connectionRefused(port)).toBe(true); } - // npm skips optional dependencies with --omit=optional; the launcher says how to fix it. - yield* fs.remove(runtime, { recursive: true }); - const missing = yield* run(process.execPath, [entry, "--version"], root); + // A download that does not match the recorded SHA-256 installs nothing. + yield* fs.remove(installed, { recursive: true }); + files.set(releaseFile, new Uint8Array([...runtimeBytes.subarray(0, 1024), 0])); + const tampered = yield* run(process.execPath, [entry, "--version"], root, download); + expect(tampered.code).toBe(1); + expect(tampered.stderr).toContain("does not match the SHA-256 this package records"); + expect(tampered.stderr).toContain("Nothing was installed."); + expect(yield* fs.exists(installed)).toBe(false); + expect((yield* fs.readDirectory(runtimes)).filter((name) => name.startsWith("."))).toEqual( + [], + ); + + // A release that serves no archive says where it looked and how to use a mirror. + files.delete(releaseFile); + const missing = yield* run(process.execPath, [entry, "--version"], root, download); expect(missing.code).toBe(1); - expect(missing.stderr).toContain(`${platformPackage} is not installed`); expect(missing.stderr).toContain( - `npm i -g executor${manifest.version.includes("-beta.") ? "@beta" : ""} --include=optional`, + `could not download its runtime from http://127.0.0.1:${releases.port}${releaseFile}: HTTP 404`, + ); + expect(missing.stderr).toContain("EXECUTOR_DOWNLOAD_URL"); + expect(yield* fs.exists(installed)).toBe(false); + }), + ).pipe(Effect.provide(NodeServices.layer), Effect.provide(FetchHttpClient.layer)), +); + +it.live( + "bun add -g and bun update -g keep a working executor, while a background Executor runs and the registry still lists a deprecated runtime version", + () => + Effect.scoped( + Effect.gen(function* () { + const { fs, path, root, run, launcher, manifest, runtimeBytes, releaseFile, runtimes } = + yield* scenario; + const launcherBytes = yield* fs.readFile( + path.resolve(yield* Config.String("EXECUTOR_E2E_LAUNCHER_ARCHIVE")), + ); + // A later launcher for `bun update -g` to move to, which semver ranks above this one. + const next = manifest.version.includes("-") + ? `${manifest.version}.1` + : manifest.version.replace(/\d+$/u, (patch) => String(Number(patch) + 1)); + const repack = path.join(root, "repack"); + yield* fs.makeDirectory(repack); + yield* fs.copy(launcher, path.join(repack, "package")); + const nextManifest = path.join(repack, "package", "package.json"); + yield* fs.writeFileString( + nextManifest, + (yield* fs.readFileString(nextManifest)).replace( + `"version": "${manifest.version}"`, + `"version": "${next}"`, + ), + ); + const packed = yield* run("tar", ["-czf", "next.tgz", "package"], repack); + expect(packed.code, packed.stderr).toBe(0); + const nextBytes = yield* fs.readFile(path.join(repack, "next.tgz")); + // The same runtime serves as the later release's, so it still reports this version. + const nextReleaseFile = `/executor@${next}/executor-${next}-${target}.tgz`; + + const runtimeVersion = `${manifest.version}-${target}`; + const integrity = (bytes: Uint8Array) => + `sha512-${createHash("sha512").update(bytes).digest("base64")}`; + const files = new Map([ + [releaseFile, runtimeBytes], + [nextReleaseFile, runtimeBytes], + [`/executor/-/executor-${manifest.version}.tgz`, launcherBytes], + [`/executor/-/executor-${next}.tgz`, nextBytes], + [`/executor/-/executor-${runtimeVersion}.tgz`, runtimeBytes], + ]); + const server = yield* serve(files); + const base = `http://127.0.0.1:${server.port}`; + const launcherEntry = (version: string, bytes: Uint8Array) => ({ + name: "executor", + version, + bin: manifest.bin, + engines: manifest.engines, + dist: { + tarball: `${base}/executor/-/executor-${version}.tgz`, + integrity: integrity(bytes), + }, + }); + // As npm holds `executor`: the launcher on `beta`, and the runtime published as a version + // of the same package, which semver ranks above it, now deprecated. + const publish = (versions: Record, beta: string) => + files.set( + "/executor", + new TextEncoder().encode( + JSON.stringify({ + name: "executor", + "dist-tags": { latest: "1.6.11", beta }, + versions: { + ...versions, + [runtimeVersion]: { + name: "executor", + version: runtimeVersion, + os: [process.platform], + cpu: [process.arch], + deprecated: "A platform runtime published in error; install executor@beta.", + dist: { + tarball: `${base}/executor/-/executor-${runtimeVersion}.tgz`, + integrity: integrity(runtimeBytes), + }, + }, + }, + }), + ), + ); + publish( + { [manifest.version]: launcherEntry(manifest.version, launcherBytes) }, + manifest.version, + ); + + const registry = yield* localNpmRegistry; + const port = yield* freePort; + const environment = { + BUN_INSTALL: path.join(root, "bun"), + BUN_INSTALL_CACHE_DIR: path.join(root, "bun-cache"), + NPM_CONFIG_REGISTRY: `${base}/`, + EXECUTOR_DOWNLOAD_URL: `${base}/`, + EXECUTOR_DATA_DIR: path.join(root, "data"), + EXECUTOR_PORT: String(port), + EXECUTOR_KEY_STORAGE: "file", + EXECUTOR_NO_UPDATE_CHECK: "1", + EXECUTOR_NPM_REGISTRY: registry.url, + }; + const global = path.join(root, "bun", "install", "global"); + const entry = path.join(global, "node_modules", "executor", "bin.mjs"); + const executor = (args: ReadonlyArray) => + run(process.execPath, [entry, ...args], root, environment); + const started = Effect.gen(function* () { + const result = yield* executor(["--version"]); + expect(result.code, result.stderr).toBe(0); + expect(result.stdout).toBe(`executor v${manifest.version}`); + if (process.platform !== "win32") { + const shim = yield* run( + path.join(root, "bun", "bin", "executor"), + ["--version"], + root, + environment, + ); + expect(shim.code, shim.stderr).toBe(0); + expect(shim.stdout).toBe(`executor v${manifest.version}`); + } + }); + const installedVersion = fs + .readFileString(path.join(global, "node_modules", "executor", "package.json")) + .pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Manifest)), + Effect.map((installed) => installed.version), + ); + + const added = yield* run("bun", ["add", "-g", "executor@beta"], root, environment); + expect(added.code, added.stderr).toBe(0); + expect(yield* installedVersion).toBe(manifest.version); + yield* started; + const unchanged = yield* run("bun", ["update", "-g"], root, environment); + expect(unchanged.code, unchanged.stderr).toBe(0); + expect(yield* installedVersion).toBe(manifest.version); + yield* started; + expect(server.requests.get(releaseFile)).toBe(1); + + // The background Executor runs from the downloaded runtime, outside node_modules, so + // replacing the launcher package touches no file it holds open; Windows refuses that. + yield* Effect.addFinalizer(() => executor(["service", "stop"]).pipe(Effect.ignore)); + const service = yield* executor(["service", "start"]); + expect(service.code, service.stderr).toBe(0); + publish( + { + [manifest.version]: launcherEntry(manifest.version, launcherBytes), + [next]: launcherEntry(next, nextBytes), + }, + next, ); + const updated = yield* run("bun", ["update", "-g"], root, environment); + expect(updated.code, updated.stderr).toBe(0); + expect(yield* installedVersion).toBe(next); + yield* started; + // The new launcher downloaded its release's runtime once. The one the background Executor + // runs from stays installed beside it. + expect(server.requests.get(nextReleaseFile)).toBe(1); + expect((yield* fs.readDirectory(runtimes)).sort()).toEqual([manifest.version, next].sort()); + const status = yield* executor(["service", "status"]); + expect(status.code, status.stderr).toBe(0); + expect(status.stdout).toContain(`(version ${manifest.version},`); + const stopped = yield* executor(["service", "stop"]); + expect(stopped.code, stopped.stderr).toBe(0); }), ).pipe(Effect.provide(NodeServices.layer), Effect.provide(FetchHttpClient.layer)), ); diff --git a/e2e/tests/oauth-registration-application-type.spec.ts b/e2e/tests/oauth-registration-application-type.spec.ts new file mode 100644 index 0000000000..711461ef8f --- /dev/null +++ b/e2e/tests/oauth-registration-application-type.spec.ts @@ -0,0 +1,97 @@ +/** + * MCP clients register under RFC 7591, which has no `application_type`. A registration that omits + * it is a native client when a callback is an RFC 8252 loopback or private-use one, and every + * callback it names is then held to the native rules. Any other registration is a web client. + */ +import { expect, layer } from "@effect/vitest"; +import { Effect, Schema } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/http"; +import { scenarios } from "../test-plan.ts"; +import { TestLive, withCase } from "../support/case.ts"; +import { Target } from "../support/platform.ts"; + +const Registered = Schema.Struct({ + client_id: Schema.NonEmptyString, + application_type: Schema.Literals(["web", "native"]), + redirect_uris: Schema.Array(Schema.String), +}); +const Refused = Schema.Struct({ error: Schema.String, error_description: Schema.String }); + +/** Register a public client anonymously, as an MCP client does. */ +const register = (redirectUris: ReadonlyArray) => + Effect.gen(function* () { + const target = yield* Target, + http = yield* HttpClient.HttpClient; + const origin = target.metadata.origin; + return yield* Effect.scoped( + Effect.gen(function* () { + const request = yield* HttpClientRequest.post(`${origin}/api/auth/oauth2/register`).pipe( + HttpClientRequest.setHeaders({ origin }), + HttpClientRequest.bodyJson({ + client_name: "Registration fixture", + redirect_uris: redirectUris, + token_endpoint_auth_method: "none", + grant_types: ["authorization_code", "refresh_token"], + response_types: ["code"], + }), + ); + const response = yield* http.execute(request); + return { status: response.status, body: yield* response.json }; + }), + ); + }); + +const registered = (redirectUris: ReadonlyArray) => + Effect.gen(function* () { + const { status, body } = yield* register(redirectUris); + expect(status).toBe(201); + return yield* Schema.decodeUnknownEffect(Registered)(body); + }); + +const refused = (redirectUris: ReadonlyArray) => + Effect.gen(function* () { + const { status, body } = yield* register(redirectUris); + expect(status).toBe(400); + return yield* Schema.decodeUnknownEffect(Refused)(body); + }); + +const web = "https://client.example.test/oauth/callback"; +const loopback = "http://localhost:8787/callback"; +const privateUse = "cursor://anysphere.cursor-mcp/oauth/callback"; + +layer(TestLive, { excludeTestServices: true })("OAuth registration application type", (it) => { + it.effect(scenarios.oauthRegistrationApplicationType.title, (context) => + withCase( + context, + Effect.gen(function* () { + // A web callback beside a loopback one, as Cursor registers, is one native client with both. + const both = yield* registered([web, loopback]); + expect(both.application_type).toBe("native"); + expect(both.redirect_uris).toEqual([web, loopback]); + + expect((yield* registered([privateUse])).application_type).toBe("native"); + expect((yield* registered(["http://127.0.0.1:43123/callback"])).application_type).toBe( + "native", + ); + expect((yield* registered([web])).application_type).toBe("web"); + + // Plain HTTP off the loopback host is refused whatever the client's type, and the refusal + // names the callback that broke the rule. + const mixed = yield* refused([loopback, "http://client.example.test/callback"]); + expect(mixed.error).toBe("invalid_redirect_uri"); + expect(mixed.error_description).toBe( + "redirect_uris[1]: Redirect URI must use HTTPS (HTTP allowed only for loopback hosts)", + ); + + // Without a loopback or private-use callback the client is a web client, which may not + // use a loopback host even over HTTPS. + const httpsLoopback = "https://localhost:8443/callback"; + const plain = yield* refused([httpsLoopback]); + expect(plain.error).toBe("invalid_redirect_uri"); + expect(plain.error_description).toBe( + `web clients require https redirect URIs on non-loopback hosts: ${httpsLoopback}`, + ); + }), + ), + ); +}); diff --git a/e2e/tests/oauth-token-telemetry.spec.ts b/e2e/tests/oauth-token-telemetry.spec.ts index 0e119c06d6..f2f5ae268d 100644 --- a/e2e/tests/oauth-token-telemetry.spec.ts +++ b/e2e/tests/oauth-token-telemetry.spec.ts @@ -472,11 +472,12 @@ layer(HostedLive, { excludeTestServices: true })("OAuth token telemetry", (it) = }; const registrationCases = [ { - // A private-use callback without `application_type` registers as a web client. + // Without an HTTP loopback or private-use callback, a client is a web client, which + // may not use a loopback host even over HTTPS. body: { ...grants, client_name: "Cursor", - redirect_uris: ["cursor://anysphere.cursor-retrieval/oauth/callback"], + redirect_uris: ["https://localhost:8443/oauth/callback"], }, expected: { status: 400, diff --git a/e2e/with-npm-registry.ts b/e2e/with-npm-registry.ts deleted file mode 100644 index a74a75a36e..0000000000 --- a/e2e/with-npm-registry.ts +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Run one command against the suite's loopback npm registry, which serves the `apps` package - * staged from this checkout and forwards every other request to the public registry; see - * support/npm-registry.ts. Release packaging uses it so installing an archive resolves the `apps` - * version this checkout ships before that version is published. - * - * Usage: `node e2e/with-npm-registry.ts -- [args...]` after `bun run apps:build` and - * `bun run e2e:apps`. Exits 1 and reports the command's exit code when it fails. - */ -import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; -import * as NodeServices from "@effect/platform-node/NodeServices"; -import { Console, Effect, Schema } from "effect"; -import { FetchHttpClient } from "effect/http"; -import { ChildProcess, ChildProcessSpawner } from "effect/process"; -import { localNpmRegistry } from "./support/npm-registry.ts"; - -class CommandFailed extends Schema.TaggedError()("CommandFailed", { - message: Schema.String, -}) {} - -NodeRuntime.runMain( - Effect.gen(function* () { - const separator = process.argv.indexOf("--"); - const [command, ...args] = separator === -1 ? [] : process.argv.slice(separator + 1); - if (command === undefined) - return yield* new CommandFailed({ message: "Usage: with-npm-registry.ts -- " }); - const registry = yield* localNpmRegistry; - yield* Console.log(`Serving apps@${registry.version} from this checkout at ${registry.url}`); - const processes = yield* ChildProcessSpawner.ChildProcessSpawner; - const code = yield* processes.exitCode( - ChildProcess.make(command, args, { - env: { NPM_CONFIG_REGISTRY: registry.url, BUN_CONFIG_REGISTRY: registry.url }, - extendEnv: true, - stdout: "inherit", - stderr: "inherit", - }), - ); - if (code !== 0) return yield* new CommandFailed({ message: `${command} exited ${code}` }); - }).pipe(Effect.scoped, Effect.provide([FetchHttpClient.layer, NodeServices.layer])), -); diff --git a/packages/app-templates/executor/skills/app-authoring/deploy.md b/packages/app-templates/executor/skills/app-authoring/deploy.md index d811ea03a3..f2f9a1fac2 100644 --- a/packages/app-templates/executor/skills/app-authoring/deploy.md +++ b/packages/app-templates/executor/skills/app-authoring/deploy.md @@ -574,8 +574,9 @@ types and a local type check do not catch: - `integrity` must be empty or omitted. `ctx.fetch` rejects an unsupported value with `FetchOptionUnsupported`, which -names the option and the values it accepts. Workers also send no `User-Agent`; -some APIs, such as GitHub's, answer 403 without one, so set it yourself. +names the option and the values it accepts. A request without a `User-Agent` +is sent with `User-Agent: Executor`, because some APIs, such as GitHub's, answer +403 without one. A `User-Agent` app code sets is sent unchanged. App code can reach public hosts with no allowlist, so a 403 or 404 from one is that service's answer. Executor refuses only requests it must not send: diff --git a/packages/app-templates/src/implementation/executor-skill-digests.gen.ts b/packages/app-templates/src/implementation/executor-skill-digests.gen.ts index d9264a8cd0..a54fc1e55c 100644 --- a/packages/app-templates/src/implementation/executor-skill-digests.gen.ts +++ b/packages/app-templates/src/implementation/executor-skill-digests.gen.ts @@ -4,7 +4,7 @@ export const executorSkillDigests: Readonly> = { "app-authoring/SKILL.md": "87118b0f1b98968a64fd6ba64dce39d1fde6c78f781f34ba85063ef4e44355a3", "app-authoring/accounts.md": "58cad0c32c94d719c50c63b53e798c0b6eae79a48f1942cc953a565ec73f9f91", - "app-authoring/deploy.md": "01938ca5b8a23c872bc6233c6c941c055adfb59ce2aca0744e873438f11095ff", + "app-authoring/deploy.md": "679ae66cb0b832e0fb9081ca402eaf4469128677366bfbf38570e1036dd916c8", "app-authoring/events.md": "8f7465e35c6c94b06978b89680181df60c741225de3ba8158f22f76ee7e91a5a", "app-authoring/integrations.md": "3f9cecb591465a71c95c98046dab960750000c05a900fde3550aac4ea3a0171a", "app-authoring/starter.md": "382bd769d7b10e9c09eaa2367755f23153a31a484aaca9c449ab4aac7d61e224", diff --git a/packages/mcp-auth/src/implementation/oauth.ts b/packages/mcp-auth/src/implementation/oauth.ts index e8db62f5a3..f790204dcc 100644 --- a/packages/mcp-auth/src/implementation/oauth.ts +++ b/packages/mcp-auth/src/implementation/oauth.ts @@ -89,10 +89,6 @@ const Claims = Schema.Struct({ aud: Schema.Union([Schema.String, Schema.Array(Schema.String)]), cnf: Schema.optionalKey(Schema.Unknown), }); -const Registration = Schema.Struct({ - application_type: Schema.optionalKey(Schema.String), - redirect_uris: Schema.Array(Schema.String), -}); /** Authorization parameters this plugin reads; every other parameter passes through unchanged. */ const AuthorizeRequest = Schema.StructWithRest( Schema.Struct({ @@ -237,6 +233,11 @@ export const grantOAuthPlugins = (settings: GrantOAuthOptions) => { allowDynamicClientRegistration: true, allowUnauthenticatedClientRegistration: true, clientRegistrationRequirePKCE: true, + // RFC 7591 has no `application_type`, so MCP clients often omit it. A registration with an + // RFC 8252 loopback or private-use callback is a native client, and every callback it names + // is checked against the native rules; any other stays a web client. The OpenID default of + // `web` refused such registrations, including Cursor's. + clientRegistrationDefaultApplicationType: "infer", // Hosted MCP clients such as Cursor register from a shared backend address, and Cursor // spends two requests per attempt. The default of 5 per minute rejected real sign-ins. // Better Auth's count resets only after a whole window without requests, so a device client @@ -1549,19 +1550,6 @@ export const grantOAuthPlugins = (settings: GrantOAuthOptions) => { }, hooks: { before: [ - { - matcher: (ctx: { path?: string }) => ctx.path === "/oauth2/register", - handler: createAuthMiddleware(async (ctx) => { - const input = Schema.decodeUnknownOption(Registration)(ctx.body); - if ( - Option.isSome(input) && - input.value.application_type === undefined && - input.value.redirect_uris.length > 0 && - input.value.redirect_uris.every(loopback) - ) - return { context: { body: { ...ctx.body, application_type: "native" } } }; - }), - }, { // Bind a request without `resource` before Better Auth signs and stores it, so consent, // the authorization code and its tokens all carry the same single audience. diff --git a/packages/sdk/src/contracts/workflows.ts b/packages/sdk/src/contracts/workflows.ts index e379356794..430975cdfe 100644 --- a/packages/sdk/src/contracts/workflows.ts +++ b/packages/sdk/src/contracts/workflows.ts @@ -25,6 +25,7 @@ import { import { AccountNotFound } from "./account.ts"; import { VersionNotFound } from "./version.ts"; import { OAuthReconnectRequired, OAuthRenewalFailed } from "./oauth.ts"; +import { AppEvaluationFailed, AppProviderFailed } from "./tools.ts"; export { HostedWorkflow, WorkflowRun, WorkflowRunId, WorkflowRunPage, WorkflowFailure }; /** @@ -85,6 +86,15 @@ export const WorkflowErrors = [ OAuthRenewalFailed, WorkflowFailure, ] as const; +/** + * Listing an app's workflows evaluates the app's definition, so it fails as listing its tools or + * skills does: with the app's own evaluation error or a service's refusal, attributed to the app. + */ +export const WorkflowDefinitionErrors = [ + ...WorkflowErrors, + AppEvaluationFailed, + AppProviderFailed, +] as const; const path = "/v1/apps/:app/workflow-runs"; /** Flat transport groups are projected beneath executor.apps in the public SDK. */ @@ -95,7 +105,7 @@ export const AppWorkflowsGroup = HttpApiGroup.make("appWorkflows").add( withLegacyKeys({ version: "deployment" }), ), success: Schema.Array(HostedWorkflow), - error: WorkflowErrors, + error: WorkflowDefinitionErrors, }), ); /** Lifecycle operations start quickly and return a retained run identity. */ diff --git a/packages/sdk/src/implementation/credential-handles.ts b/packages/sdk/src/implementation/credential-handles.ts index 30f851d3a0..91f1daa8c4 100644 --- a/packages/sdk/src/implementation/credential-handles.ts +++ b/packages/sdk/src/implementation/credential-handles.ts @@ -59,6 +59,7 @@ import { } from "apps/contracts"; import { isLoopbackHostname, isPrivateHostname } from "@executor-js/utils/url-policy"; import type { HostApiCheck } from "../contracts/host-api.ts"; +import { executorUserAgent } from "./user-agent.ts"; import { hostApiChangingHeader, hostApiChangingResponse, @@ -834,23 +835,27 @@ const forward = async (request: Request, outbound: CredentialOutbound): Promise< const send = (sent: Request) => outbound.send(sent).catch(sendFailed(request.signal)); const blocked = egressRefusal(url, outbound.egress); if (blocked !== undefined) return networkRefusalResponse(blocked); - const kind = request.body === null ? undefined : rewritable(request.headers.get("content-type")); + // Every app request names its sender, as app code's own User-Agent or as Executor. + const requestHeaders = new Headers(request.headers); + if (!requestHeaders.has("user-agent")) requestHeaders.set("user-agent", executorUserAgent); + const kind = request.body === null ? undefined : rewritable(requestHeaders.get("content-type")); const body = kind === undefined || request.body === null ? undefined : await readBody(request.body); const found = new Set([ ...handlesIn(url.href), ...[...url.searchParams.values()].flatMap(handlesWithin), - ...[...request.headers].flatMap(([, value]) => handlesWithin(value)), + ...[...requestHeaders].flatMap(([, value]) => handlesWithin(value)), ...(body?.complete === true ? handlesIn(body.text) : []), ]); if (found.size === 0) { - const headers = new Headers(request.headers); - stampCaller(headers, outbound); + // The default User-Agent is already on `requestHeaders`. Stamp the connection this Worker + // was loaded for onto that same copy: a scoped key still cannot widen by omitting User-Agent. + stampCaller(requestHeaders, outbound); const sent = body === undefined - ? new Request(request, { headers }) + ? new Request(request, { headers: requestHeaders }) : new Request(request, { - headers, + headers: requestHeaders, body: body.complete ? body.text : body.stream, ...(body.complete ? {} : { duplex: "half" }), }); @@ -905,12 +910,10 @@ const forward = async (request: Request, outbound: CredentialOutbound): Promise< // Managed handles go only in the operator's header, placed handles only where a placement // matches, and every other handle anywhere. const searched = body?.complete === true ? body.text : undefined; - const placed = Result.flatMap( - placeManaged(url, request.headers, searched, opened), - (headers) => - [...opened.values()].some((sealed) => sealed.request !== undefined) - ? placeCredentials(url, headers, searched, opened) - : Result.succeed({ href: url.href, headers }), + const placed = Result.flatMap(placeManaged(url, requestHeaders, searched, opened), (headers) => + [...opened.values()].some((sealed) => sealed.request !== undefined) + ? placeCredentials(url, headers, searched, opened) + : Result.succeed({ href: url.href, headers }), ); if (Result.isFailure(placed)) return networkRefusalResponse( diff --git a/packages/sdk/src/implementation/user-agent.ts b/packages/sdk/src/implementation/user-agent.ts index b14deb8762..a0d0660310 100644 --- a/packages/sdk/src/implementation/user-agent.ts +++ b/packages/sdk/src/implementation/user-agent.ts @@ -1,12 +1,15 @@ import { HttpClient, HttpClientRequest } from "effect/http"; /** - * Name Executor on requests that do not already carry a User-Agent. Some firewalls refuse requests - * without one, and Workers do not supply one by default. + * The User-Agent Executor sends when a request names none. Workers send no User-Agent, and some + * services refuse a request without one: GitHub's API answers 403 with an HTML page. */ +export const executorUserAgent = "Executor"; + +/** Name Executor on requests that do not already carry a User-Agent. */ export const withExecutorUserAgent = (client: HttpClient.HttpClient) => HttpClient.mapRequest(client, (request) => request.headers["user-agent"] === undefined - ? HttpClientRequest.setHeader(request, "user-agent", "Executor") + ? HttpClientRequest.setHeader(request, "user-agent", executorUserAgent) : request, ); diff --git a/packages/sdk/src/implementation/workflows.ts b/packages/sdk/src/implementation/workflows.ts index 7fa0ada8c7..0f7bdc5e37 100644 --- a/packages/sdk/src/implementation/workflows.ts +++ b/packages/sdk/src/implementation/workflows.ts @@ -6,6 +6,7 @@ import { Hex } from "effect/encoding"; import { UserFacingError } from "@executor-js/utils/user-facing-error"; import { HostedWorkflow, + ProviderError, WorkflowFailure, WorkflowRun, WorkflowRunId, @@ -37,6 +38,8 @@ import { callerContext, resolve, snapshot, type InvocationSnapshot } from "./too import { invocationScope, scopedSelection } from "./selection.ts"; import type { Declarations } from "./declarations.ts"; import { ownsDatabase, sharesRuntime } from "../contracts/apps.ts"; +import { AppEvaluationFailed, evaluationFailure } from "../contracts/tools.ts"; +import { appProviderFailure } from "./provider-error.ts"; const StoredRun = Schema.Struct({ id: WorkflowRunId, @@ -749,22 +752,32 @@ export const makeWorkflowRuns = ( definitions: (input: typeof WorkflowApp.Type) => Effect.gen(function* () { const state = yield* snapshot(db, input); + const identity = { app: state.app.id, version: state.version.id }; + // Listing workflows evaluates the app's definition, as listing its tools or skills does, + // so it keeps the same account of what failed: the app's own error, a service's refusal + // or the runtime failure Executor observed, attributed to this app alone. const value = yield* declarations.read("workflows", state, (context) => - safe( - runtime.workflow({ + runtime + .workflow({ app: input.app, build: state.version.build, sharedRuntime: sharesRuntime(state.version.requirements), hostApi: declarations.hostApi(state.version), ...context, command: { operation: "workflows" }, - }), - "execution", - ), + }) + .pipe( + Effect.mapError((error) => + Schema.is(ProviderError)(error) + ? appProviderFailure(state, error) + : evaluationFailure(identity, error, "Workflow evaluation failed"), + ), + ), ); - return yield* safe( - Schema.decodeUnknownEffect(Schema.Array(HostedWorkflow))(value), - "execution", + return yield* Schema.decodeUnknownEffect(Schema.Array(HostedWorkflow))(value).pipe( + Effect.mapError( + () => new AppEvaluationFailed({ ...identity, reason: "Invalid workflow catalog" }), + ), ); }).pipe(Effect.withSpan("sdk.workflows.definitions")), runs: { diff --git a/scripts/releases/README.md b/scripts/releases/README.md index 3b4162248e..8799cd7d84 100644 --- a/scripts/releases/README.md +++ b/scripts/releases/README.md @@ -1,7 +1,6 @@ # Executor 2 beta -Install with npm, `npm i -g executor@beta --include=optional`, on Node 14.18 -or newer. Without Node, run `curl -fsSL https://executor.sh/docs/install.sh | sh` +Install with npm, `npm i -g executor@beta`, on Node 14.18 or newer. Without Node, run `curl -fsSL https://executor.sh/docs/install.sh | sh` on macOS or Linux, or `irm https://executor.sh/docs/install.ps1 | iex` in PowerShell on Windows. Each release also attaches its archives and `SHA256SUMS`. Run `executor` to start the background Executor, or join the one already @@ -23,10 +22,13 @@ Executor is one executable, built with Bun, with the files it runs or reads beside it: the dashboard, the app framework and compiler references, workerd, Git, the native credential-store binding and the Motel collector. It needs no source checkout, Node, Bun or 1Password on the user machine. The `executor` npm -package is a launcher that starts the executable from the platform package npm -installs as an optional dependency, for macOS (Apple Silicon and Intel), Linux -(arm64 and x64, glibc) and Windows (x64). The same archive is the release -download the install scripts and Homebrew use. +package is a launcher. The first time a version starts, it downloads that +release's archive for the platform, for macOS (Apple Silicon and Intel), Linux +(arm64 and x64, glibc) or Windows (x64), from the public GitHub release, checks it +against the SHA-256 the package records, and keeps it in +`~/.executor/v2/launcher/` with the version before it. The same archive +is the release download the install scripts and Homebrew use. +`EXECUTOR_DOWNLOAD_URL` names a mirror of the releases for all of them. Beta uses `~/.executor/v2/cli` for its data. This path stays the same at the stable release. It is separate from Executor 1 data. One server runs per data @@ -79,20 +81,25 @@ Native artifacts are under `.local/releases/--/`. `release:cli` bundles the runtime twice. `package/` holds the JavaScript runtime the desktop app runs with its own Node. `standalone/` holds the release install: the `executor` executable (`bun build --compile`) and the files beside it, which -`executor---.tgz` packs as the platform's npm package -and release archive. The executable takes `runtime/cli.mjs` in its install -directory as every module's location, so resources, `require` and -`import.meta.resolve` find the same files beside it as from the npm package, -through any launcher symlink. Packages that must stay on disk load from +`executor---.tgz` packs as the platform's release +archive. The executable takes `runtime/cli.mjs` in its install directory as +every module's location, so resources, `require` and `import.meta.resolve` find +the same files beside it wherever the archive is unpacked, through any launcher +symlink. Packages that must stay on disk load from `runtime/node_modules`; Git is in `runtime/git`. -The npm launcher is under `.local/releases//wrapper/` and uses exact -native version aliases. These build commands do not publish anything. -The app compiler and framework are prepared for workerd at build time. No -runtime download is needed. Each compressed native archive must fit within -178 MiB before publication (`npmArchiveBudgetBytes` in `config.ts`). npm -documents no limit; the budget stays below the largest archive it has accepted -from Executor. Most of each archive is the executable, workerd and the bundled -Git, which keeps only what the runtime runs (`keptGit` in `runtime-bundle.ts`). +The npm launcher is under `.local/releases//wrapper/`. Its +package.json records the SHA-256 of each platform archive found under +`.local/releases/--/`, or under the directory +`release:wrapper` is given; publishing requires all five. The runtimes are never +published to npm. Through 2.0.0-beta.36 each was published as a version of the +`executor` package, such as `2.0.0-beta.36-win32-x64`, which semver ranks above +the launcher, so a range update such as `bun update -g` replaced the launcher +with a runtime. Those versions are deprecated so package managers skip them. +These build commands do not publish anything. +The app compiler and framework are prepared for workerd at build time, so the +executable downloads nothing more. Most of each archive is the executable, +workerd and the bundled Git, which keeps only what the runtime runs (`keptGit` +in `runtime-bundle.ts`). External source maps stay in the separate CI artifact for release diagnostics. Stock Bun lets any local process run its own code inside a compiled executable, diff --git a/scripts/releases/config.ts b/scripts/releases/config.ts index a41d494b14..4dbcb51d18 100644 --- a/scripts/releases/config.ts +++ b/scripts/releases/config.ts @@ -13,15 +13,6 @@ export { type ReleaseChannel, } from "@executor-js/utils/release-version"; -/** - * The compressed archive budget, checked before npm receives any upload. npm documents no size - * limit. It refused a 298.9 MB archive with HTTP 413, and others report refusals from about - * 185 MB; the largest Executor archive it has accepted is 187,186,037 bytes. The budget stays - * below that. - */ -export const npmArchiveBudgetMiB = 178; -export const npmArchiveBudgetBytes = npmArchiveBudgetMiB * 1024 * 1024; - /** * Native platforms supported by the packaged runtime. The full CLI suite runs against the packaged * archive on linux x64 and on Windows, which no other workflow tests; CI runs it from source on @@ -128,15 +119,14 @@ export const desktopUpdateFile = (target: Platform, channel: ReleaseChannel): st return target.arch === "x64" ? `${name}-linux.yml` : `${name}-linux-${target.arch}.yml`; }; -/** Immutable npm version for one native runtime, aliased by the launcher package. */ +/** + * The version one native runtime's package.json carries. The runtime is not published to npm: + * its archive is a release asset, which the `executor` launcher downloads. + */ export const platformVersion = (target: Platform): string => `${release.version}-${target.platform}-${target.arch}`; -/** npm alias name installed under the wrapper's node_modules. */ -export const platformPackage = (target: Platform): string => - `executor-${target.platform}-${target.arch}`; - -/** The actual platform archive filename produced by npm pack. */ +/** The platform's release archive, as npm pack names it. */ export const platformArchive = (target: Platform): string => `executor-${platformVersion(target)}.tgz`; diff --git a/scripts/releases/npm-launcher.mjs b/scripts/releases/npm-launcher.mjs index 35ae2bf4b1..6a44cfbdfe 100644 --- a/scripts/releases/npm-launcher.mjs +++ b/scripts/releases/npm-launcher.mjs @@ -1,39 +1,216 @@ #!/usr/bin/env node -// The `executor` npm package's command. It starts the Executor executable from the platform -// package npm installed as an optional dependency, passes signals on, and exits as it exits. The -// executable carries its own runtime, so this file uses only what every Node since 14 has. -import { spawn } from "child_process"; +// The `executor` npm package's command. The package holds no runtime: on first use of a version it +// downloads that release's archive for this platform from the public GitHub release, checks it +// against the SHA-256 this package's package.json records, and unpacks it into +// ~/.executor/v2/launcher/. It then starts the Executor executable there, passes signals +// on, and exits as it exits. +// +// npm, Bun, pnpm and Yarn resolve `executor` alone, so an update can only ever land on another +// launcher; the running executable lives outside node_modules, so an update never has to replace a +// file in use. The executable carries its own runtime, so this file uses only what every Node since +// 14 has. +import { spawn, spawnSync } from "child_process"; +import { createHash, randomBytes } from "crypto"; +import { + createWriteStream, + existsSync, + mkdirSync, + readdirSync, + renameSync, + rmSync, + statSync, + utimesSync, +} from "fs"; +import { request as httpRequest } from "http"; +import { request as httpsRequest } from "https"; import { createRequire } from "module"; -import { dirname, join } from "path"; +import { homedir } from "os"; +import { join } from "path"; import { fileURLToPath } from "url"; const require = createRequire(import.meta.url); const manifest = require("./package.json"); -const name = "executor-" + process.platform + "-" + process.arch; -const channel = manifest.version.indexOf("-beta.") === -1 ? "" : "@beta"; -const reinstall = "npm i -g executor" + channel + " --include=optional"; +const version = manifest.version; +const target = process.platform + "-" + process.arch; +const digest = (manifest.runtimes || {})[target]; +const channel = version.indexOf("-beta.") === -1 ? "" : "@beta"; +const executableName = process.platform === "win32" ? "executor.exe" : "executor"; +const archive = "executor-" + version + "-" + target + ".tgz"; +// The same mirror setting the install scripts and `executor upgrade` read. +const downloads = ( + process.env.EXECUTOR_DOWNLOAD_URL || + "https://github.com/UsefulSoftwareCo/executor/releases/download" +).replace(/\/+$/, ""); +const url = downloads + "/executor@" + version + "/" + archive; +const root = join(homedir(), ".executor", "v2", "launcher"); +const installed = join(root, version); +const executable = join(installed, executableName); const fail = (lines) => { for (const line of lines) console.error(line); process.exit(1); }; -if (!Object.prototype.hasOwnProperty.call(manifest.optionalDependencies || {}, name)) +if (typeof digest !== "string") fail(["Executor does not support " + process.platform + "/" + process.arch + "."]); -let root; -try { - root = dirname(require.resolve(name + "/package.json")); -} catch { - // npm skips optional dependencies with --omit=optional, or when a lockfile made on another - // platform did not list this one. - fail([ - "Executor's native runtime is missing: " + name + " is not installed.", - "Reinstall with optional dependencies: " + reinstall, - ]); -} +/** GET `address`, following redirects; resolves to the response of the final location. */ +const get = (address, redirects) => + new Promise((resolve, reject) => { + const request = (address.startsWith("http:") ? httpRequest : httpsRequest)( + address, + { headers: { "user-agent": "executor-launcher/" + version } }, + (response) => { + const status = response.statusCode || 0; + if (status >= 300 && status < 400 && response.headers.location) { + response.resume(); + if (redirects >= 10) return reject(new Error("too many redirects")); + return resolve(get(new URL(response.headers.location, address).href, redirects + 1)); + } + if (status !== 200) { + response.resume(); + return reject(new Error("HTTP " + status)); + } + resolve(response); + }, + ); + // A connection that stops sending for a minute is not coming back. + request.setTimeout(60000, () => request.destroy(new Error("the connection stalled"))); + request.on("error", reject); + request.end(); + }); + +/** Download the archive to `file`, hashing it as it arrives; resolves to its SHA-256. */ +const download = async (file) => { + const response = await get(url, 0); + const total = Number(response.headers["content-length"]) || 0; + const hash = createHash("sha256"); + const out = createWriteStream(file); + const tty = process.stderr.isTTY; + let received = 0; + let shown = -1; + const label = "Downloading Executor " + version + " for " + target; + if (!tty) console.error(label + " (first run of this version)"); + await new Promise((resolve, reject) => { + response.on("data", (chunk) => { + hash.update(chunk); + received += chunk.length; + const step = total ? Math.floor((received * 100) / total) : Math.floor(received / 1048576); + if (tty && step !== shown) { + shown = step; + const amount = (received / 1048576).toFixed(1) + " MB"; + process.stderr.write( + "\r\x1b[2K" + label + (total ? " " + step + "%" : "") + " (" + amount + ")", + ); + } + }); + response.on("error", reject); + out.on("error", reject); + out.on("finish", resolve); + response.pipe(out); + }); + if (tty) process.stderr.write("\r\x1b[2K"); + return hash.digest("hex"); +}; + +/** + * Keep this version and the most recently installed other one, which a background Executor started + * before an update may still run from, as the install scripts keep `previous`. Downloads that + * stopped part way are removed after a day. Anything in use, which Windows refuses to delete, stays + * for a later download to remove. + */ +const prune = () => { + const day = 24 * 60 * 60 * 1000; + const entries = readdirSync(root) + .filter((name) => name !== version) + .map((name) => ({ name, time: statSync(join(root, name)).mtimeMs })); + const versions = entries.filter((entry) => !entry.name.startsWith(".")); + versions.sort((left, right) => right.time - left.time); + const stale = versions + .slice(1) + .concat(entries.filter((entry) => entry.name.startsWith(".") && Date.now() - entry.time > day)); + for (const entry of stale) { + try { + rmSync(join(root, entry.name), { recursive: true, force: true }); + } catch { + // In use; a later download removes it. + } + } +}; + +const install = async () => { + mkdirSync(root, { recursive: true }); + const staging = join(root, ".download-" + process.pid + "-" + randomBytes(4).toString("hex")); + mkdirSync(staging); + // `fail` exits at once, without running `finally`. + const cleanup = () => rmSync(staging, { recursive: true, force: true }); + process.on("exit", cleanup); + try { + let actual; + try { + actual = await download(join(staging, archive)); + } catch (error) { + fail([ + "Executor could not download its runtime from " + url + ": " + error.message + ".", + "Check your connection and try again. EXECUTOR_DOWNLOAD_URL names a mirror of the releases.", + ]); + } + if (actual !== digest) + fail([ + archive + + " does not match the SHA-256 this package records (expected " + + digest + + ", got " + + actual + + ").", + "Nothing was installed. Check EXECUTOR_DOWNLOAD_URL, or reinstall with: npm i -g executor" + + channel, + ]); + // The system's tar, as the install scripts use: Windows' own bsdtar, since Git's GNU tar on + // PATH reads a drive letter as a remote host. The archive path is relative for the same reason. + const tar = + process.platform === "win32" + ? join(process.env.SystemRoot || "C:\\Windows", "System32", "tar.exe") + : "tar"; + // Nothing reaches stdout, which an MCP client starting `executor mcp` reads as the protocol. + const unpacked = spawnSync(tar, ["-xzf", archive], { + cwd: staging, + stdio: ["ignore", "ignore", "inherit"], + }); + if (unpacked.status !== 0) + fail([ + "Executor could not unpack " + + archive + + " with " + + tar + + (unpacked.error ? ": " + unpacked.error.message : "."), + ]); + try { + renameSync(join(staging, "package"), installed); + } catch (error) { + // Another `executor` command installed this version first. + if (!existsSync(executable)) + fail([ + "Executor could not install its runtime in " + installed + ": " + error.message, + "Remove " + installed + " and try again.", + ]); + } + // The version's time orders it for pruning; the archive's own times are from 1985. + const now = new Date(); + utimesSync(installed, now, now); + } finally { + process.removeListener("exit", cleanup); + cleanup(); + } + try { + prune(); + } catch { + // Pruning never stops Executor from starting. + } +}; + +if (!existsSync(executable)) await install(); -const executable = join(root, process.platform === "win32" ? "executor.exe" : "executor"); const child = spawn(executable, process.argv.slice(2), { stdio: "inherit", env: Object.assign({}, process.env, { @@ -57,7 +234,7 @@ for (const signal of signals) process.on(signal, forward); child.on("error", (error) => fail([ "Executor could not start " + executable + ": " + error.message, - "Reinstall with: " + reinstall, + "Remove " + installed + " to download it again.", ]), ); child.on("exit", (code, signal) => { diff --git a/scripts/releases/package.ts b/scripts/releases/package.ts index 8bda78acc5..215437b2bd 100644 --- a/scripts/releases/package.ts +++ b/scripts/releases/package.ts @@ -4,14 +4,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { Config, Console, Effect, FileSystem, Path, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { bundleLocalRuntime } from "./runtime-bundle.ts"; -import { - nativePlatform, - npmArchiveBudgetBytes, - npmArchiveBudgetMiB, - platformArchive, - platformVersion, - release, -} from "./config.ts"; +import { nativePlatform, platformArchive, platformVersion, release } from "./config.ts"; import { installWindowsGitHttpBackend } from "./windows-git.ts"; import { launcherAnalytics } from "./analytics.ts"; import { buildStandalone } from "./standalone.ts"; @@ -169,19 +162,7 @@ const build = Effect.gen(function* () { ) return yield* Effect.die(new Error("A published macOS build must be signed and notarized.")); yield* buildStandalone(root, output, stage, target, signing); - // Guard each native target before its offline tests or any publication can run. const bytes = Number((yield* fs.stat(path.join(output, platformArchive(target)))).size); - const budgetBytes = npmArchiveBudgetBytes; - yield* fs.writeFileString( - path.join(output, "npm-size.json"), - JSON.stringify({ bytes, budgetBytes }, null, 2), - ); - if (bytes > budgetBytes) - return yield* Effect.die( - new Error( - `npm archive exceeds the ${npmArchiveBudgetMiB} MiB release budget: ${bytes} bytes`, - ), - ); yield* Console.log(`Native artifact: ${output} (${(bytes / 1024 / 1024).toFixed(1)} MiB)`); }); NodeRuntime.runMain(Effect.scoped(build).pipe(Effect.provide(NodeServices.layer))); diff --git a/scripts/releases/publish-npm.ts b/scripts/releases/publish-npm.ts index 86c18bc7b7..3171628c74 100644 --- a/scripts/releases/publish-npm.ts +++ b/scripts/releases/publish-npm.ts @@ -1,6 +1,10 @@ /** - * Publish one complete version once. An accepted upload is polled, never uploaded again; - * a retried job skips archives whose registry integrity matches byte for byte. + * Publish the `executor` launcher for one version once. The native runtimes are not on npm: the + * launcher downloads them from the public GitHub release and checks each against the SHA-256 its + * package.json records. So this refuses a launcher that does not record every platform's archive + * in the artifact directory, or whose archives the public release does not list in its + * SHA256SUMS. An accepted upload is polled, never uploaded again; a retried job skips a launcher + * whose registry integrity matches byte for byte. */ import { createHash } from "node:crypto"; import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; @@ -16,22 +20,21 @@ import { Schema, Stream, } from "effect"; -import { FetchHttpClient, HttpClient } from "effect/http"; +import { FetchHttpClient, HttpClient, HttpClientResponse } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; -import { - npmArchiveBudgetBytes, - npmArchiveBudgetMiB, - platformArchive, - platformVersion, - platforms, - release, -} from "./config.ts"; +import { platformArchive, platforms, release } from "./config.ts"; const RegistryVersion = Schema.Struct({ version: Schema.String, dist: Schema.Struct({ integrity: Schema.String }), }); const Tags = Schema.Record(Schema.String, Schema.String); +const Launcher = Schema.fromJsonString( + Schema.Struct({ + version: Schema.String, + runtimes: Schema.Record(Schema.String, Schema.String), + }), +); NodeRuntime.runMain( Effect.scoped( @@ -54,31 +57,9 @@ NodeRuntime.runMain( } }); yield* collect(directory); - const packages = platforms.map((target) => { - const matches = files.filter((file) => path.basename(file) === platformArchive(target)); - const [file, ...extra] = matches; - if (file === undefined || extra.length !== 0) - throw new Error(`Expected exactly one ${platformArchive(target)}.`); - return { - file, - version: platformVersion(target), - tag: `${release.channel}-${target.platform}-${target.arch}`, - }; - }); - packages.push({ - file: `.local/releases/${release.version}/wrapper/executor-${release.version}.tgz`, - version: release.version, - tag: release.channel, - }); - const registry = "https://registry.npmjs.org"; - const tags = http.get(`${registry}/-/package/executor/dist-tags`).pipe( - Effect.flatMap((response) => response.json), - Effect.flatMap(Schema.decodeUnknownEffect(Tags)), - ); - const before = yield* tags; - const integrityOf = (file: string) => + const digest = (file: string, algorithm: "sha256" | "sha512") => Effect.gen(function* () { - const hash = createHash("sha512"); + const hash = createHash(algorithm); yield* fs.stream(file).pipe( Stream.runForEach((bytes) => Effect.sync(() => { @@ -86,89 +67,111 @@ NodeRuntime.runMain( }), ), ); - return `sha512-${hash.digest("base64")}`; + return hash; + }); + + const launcherDirectory = path.join(".local/releases", release.version, "wrapper"); + const file = path.join(launcherDirectory, `executor-${release.version}.tgz`); + if (!(yield* fs.exists(file))) return yield* Effect.die(new Error(`Missing ${file}`)); + const launcher = yield* fs + .readFileString(path.join(launcherDirectory, "package.json")) + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Launcher))); + if (launcher.version !== release.version) + return yield* Effect.die( + new Error(`The launcher is ${launcher.version}, not ${release.version}.`), + ); + const names = platforms.map((target) => `${target.platform}-${target.arch}`); + const unknown = Object.keys(launcher.runtimes).filter((name) => !names.includes(name)); + if (unknown.length !== 0) + return yield* Effect.die(new Error(`The launcher records unknown runtimes: ${unknown}.`)); + + // The archives the public release serves, as its SHA256SUMS lists them. + const sums = yield* http + .get(`https://github.com/${release.repository}/releases/download/${release.tag}/SHA256SUMS`) + .pipe( + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.flatMap((response) => response.text), + ); + const listed = new Map(); + for (const line of sums.split("\n")) { + const match = /^([0-9a-f]{64}) [ *](?:\.\/)?(.+)$/u.exec(line.trim()); + if (match !== null) listed.set(match[2] ?? "", match[1] ?? ""); + } + for (const target of platforms) { + const name = platformArchive(target); + const [archive, ...extra] = files.filter((candidate) => path.basename(candidate) === name); + if (archive === undefined || extra.length !== 0) + return yield* Effect.die(new Error(`Expected exactly one ${name}.`)); + const sha256 = (yield* digest(archive, "sha256")).digest("hex"); + if (launcher.runtimes[`${target.platform}-${target.arch}`] !== sha256) + return yield* Effect.die(new Error(`The launcher does not record ${name} as built.`)); + if (listed.get(name) !== sha256) + return yield* Effect.die( + new Error(`The public release ${release.tag} does not serve ${name} as built.`), + ); + } + + const registry = "https://registry.npmjs.org"; + const tags = http.get(`${registry}/-/package/executor/dist-tags`).pipe( + Effect.flatMap((response) => response.json), + Effect.flatMap(Schema.decodeUnknownEffect(Tags)), + ); + const before = yield* tags; + const integrity = `sha512-${(yield* digest(file, "sha512")).digest("base64")}`; + // A retried job resumes only over a launcher npm already holds byte for byte. + const existing = yield* http.get(`${registry}/executor/${release.version}`); + if (existing.status === 404) { + yield* existing.text; + const temporary = yield* fs.makeTempDirectoryScoped({ prefix: "executor-npm-" }); + const npmrc = path.join(temporary, "npmrc"); + // npm substitutes the explicitly supplied environment variable. The file contains no secret. + yield* fs.writeFileString(npmrc, "//registry.npmjs.org/:_authToken=${NPM_TOKEN}\n", { + mode: 0o600, }); - // A retried job resumes only over archives npm already holds byte for byte. - const pending: Array<(typeof packages)[number] & { integrity: string }> = []; - for (const pkg of packages) { - if (!(yield* fs.exists(pkg.file))) - return yield* Effect.die(new Error(`Missing ${pkg.file}`)); - if (Number((yield* fs.stat(pkg.file)).size) > npmArchiveBudgetBytes) + const code = yield* processes.exitCode( + ChildProcess.make( + "npm", + ["publish", file, "--tag", release.channel, "--access", "public", "--ignore-scripts"], + { + env: { NPM_CONFIG_USERCONFIG: npmrc, NPM_TOKEN: Redacted.value(token) }, + extendEnv: true, + stdout: "inherit", + stderr: "inherit", + }, + ), + ); + if (code !== 0) return yield* Effect.die( new Error( - `npm archive exceeds the ${npmArchiveBudgetMiB} MiB release budget: ${pkg.file}`, + `npm publish ${release.version} failed. Inspect the registry and do not blindly retry accepted uploads.`, ), ); - const integrity = yield* integrityOf(pkg.file); - const response = yield* http.get(`${registry}/executor/${pkg.version}`); - if (response.status === 404) { - yield* response.text; - pending.push({ ...pkg, integrity }); - continue; - } + } else { const published = - response.status === 200 - ? yield* response.json.pipe(Effect.flatMap(Schema.decodeUnknownEffect(RegistryVersion))) + existing.status === 200 + ? yield* existing.json.pipe(Effect.flatMap(Schema.decodeUnknownEffect(RegistryVersion))) : undefined; if (published?.dist.integrity !== integrity) return yield* Effect.die( new Error( - `Cannot publish ${pkg.version}: the registry holds a different archive (status ${response.status}). Inspect existing publication before retrying.`, + `Cannot publish ${release.version}: the registry holds a different archive (status ${existing.status}). Inspect existing publication before retrying.`, ), ); - yield* Console.log(`Already published ${pkg.version} with matching integrity`); + yield* Console.log(`Already published ${release.version} with matching integrity`); } - const temporary = yield* fs.makeTempDirectoryScoped({ prefix: "executor-npm-" }); - const npmrc = path.join(temporary, "npmrc"); - // npm substitutes the explicitly supplied environment variable. The file contains no secret. - yield* fs.writeFileString(npmrc, "//registry.npmjs.org/:_authToken=${NPM_TOKEN}\n", { - mode: 0o600, - }); - const upload = (pkg: (typeof pending)[number]) => - Effect.gen(function* () { - const code = yield* processes.exitCode( - ChildProcess.make( - "npm", - ["publish", pkg.file, "--tag", pkg.tag, "--access", "public", "--ignore-scripts"], - { - env: { NPM_CONFIG_USERCONFIG: npmrc, NPM_TOKEN: Redacted.value(token) }, - extendEnv: true, - stdout: "inherit", - stderr: "inherit", - }, - ), - ); - if (code !== 0) - return yield* Effect.die( - new Error( - `npm publish ${pkg.version} failed. Inspect the registry and do not blindly retry accepted uploads.`, - ), - ); - }); - const verify = (pkg: (typeof pending)[number]) => - http.get(`${registry}/executor/${pkg.version}`).pipe( - Effect.flatMap((response) => response.json), - Effect.flatMap(Schema.decodeUnknownEffect(RegistryVersion)), - Effect.flatMap((published) => - published.version === pkg.version && published.dist.integrity === pkg.integrity - ? Effect.void - : Effect.fail(new Error(`Registry integrity does not match ${pkg.version}`)), - ), - Effect.timeout(15_000), - // npm can take over 15 minutes to expose a large accepted archive. - Effect.retry({ schedule: Schedule.spaced(15_000), times: 160 }), - Effect.andThen(Console.log(`Verified public npm archive ${pkg.version}`)), - ); - // npm exposes each accepted runtime after minutes, so all runtimes upload before any is - // awaited. The launcher names them as optional dependencies, so it uploads only once - // every runtime is public. - const runtimes = pending.filter((pkg) => pkg.version !== release.version); - const launcher = pending.filter((pkg) => pkg.version === release.version); - yield* Effect.forEach(runtimes, upload, { discard: true }); - yield* Effect.forEach(runtimes, verify, { concurrency: "unbounded", discard: true }); - yield* Effect.forEach(launcher, (pkg) => Effect.andThen(upload(pkg), verify(pkg)), { - discard: true, - }); + yield* http.get(`${registry}/executor/${release.version}`).pipe( + Effect.flatMap((response) => response.json), + Effect.flatMap(Schema.decodeUnknownEffect(RegistryVersion)), + Effect.flatMap((published) => + published.version === release.version && published.dist.integrity === integrity + ? Effect.void + : Effect.fail(new Error(`Registry integrity does not match ${release.version}`)), + ), + Effect.timeout(15_000), + // npm can take minutes to expose an accepted version. + Effect.retry({ schedule: Schedule.spaced(15_000), times: 160 }), + ); + yield* Console.log(`Verified public npm archive ${release.version}`); const after = yield* tags; if ( after[release.channel] !== release.version || diff --git a/scripts/releases/standalone.ts b/scripts/releases/standalone.ts index 9e6c76ce7c..57d1d2ab16 100644 --- a/scripts/releases/standalone.ts +++ b/scripts/releases/standalone.ts @@ -1,8 +1,9 @@ /** * Turn the packaged runtime into the release install: one `executor` executable that carries the * runtime's code, and beside it the files it runs or reads from disk. Those are workerd, Git, the - * native bindings, PGlite, TypeScript, the dashboard and Motel. npm publishes this directory as the - * platform package, and the same archive is the download the installers and Homebrew use. + * native bindings, PGlite, TypeScript, the dashboard and Motel. npm pack archives this directory as + * the platform's release asset: the download the `executor` npm launcher, the install scripts and + * Homebrew use. * * Layout, relative to the executable: * - `runtime/node_modules/`: packages loaded from disk (see `installedPackages`) and resources @@ -55,7 +56,7 @@ const bunRuntime = Effect.gen(function* () { /** * Build the release install from `stage`, the packaged runtime, into `/standalone`, and - * pack it as the platform's npm archive in `output`. Returns the install directory. + * pack it as the platform's release archive in `output`. Returns the install directory. */ export const buildStandalone = ( root: string, @@ -179,14 +180,12 @@ export const buildStandalone = ( { name: "executor", version: platformVersion(target), + // A release asset that the executor npm package downloads, never published to npm. + private: true, description: "Executor for one platform, started by the executor package", license: "MIT", homepage: release.cloudOrigin, repository: { type: "git", url: `https://github.com/${release.repository}.git` }, - publishConfig: { - access: "public", - tag: `${release.channel}-${target.platform}-${target.arch}`, - }, os: [target.platform], cpu: [target.arch], files: [ diff --git a/scripts/releases/wrapper.ts b/scripts/releases/wrapper.ts index 3115a3db3b..673b3169a0 100644 --- a/scripts/releases/wrapper.ts +++ b/scripts/releases/wrapper.ts @@ -1,13 +1,17 @@ /** - * Build the `executor` npm package: a launcher that starts the executable from the platform package - * it names as an exact optional dependency. Nothing is published. + * Build the `executor` npm package: a launcher that downloads its platform's release archive on + * first use and starts the executable in it. The package records each archive's SHA-256, read from + * `/--/`, where the directory is the first argument (default + * `.local/releases`). A publishing run supplies every platform's archive; a test build may have + * only its own. Nothing is published. */ +import { createHash } from "node:crypto"; import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { Console, Effect, FileSystem, Path } from "effect"; +import { Console, Effect, FileSystem, Path, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import manifest from "../../apps/cli/package.json" with { type: "json" }; -import { platforms, platformPackage, platformVersion, release } from "./config.ts"; +import { platformArchive, platforms, release } from "./config.ts"; import { npmCommand } from "./npm.ts"; const build = Effect.gen(function* () { @@ -16,6 +20,33 @@ const build = Effect.gen(function* () { const processes = yield* ChildProcessSpawner.ChildProcessSpawner; const root = yield* path.fromFileUrl(new URL("../../", import.meta.url)); const directory = path.join(root, ".local/releases", release.version, "wrapper"); + const archives = path.resolve(root, process.argv[2] ?? ".local/releases"); + const sha256 = (file: string) => + Effect.gen(function* () { + const hash = createHash("sha256"); + yield* fs.stream(file).pipe( + Stream.runForEach((bytes) => + Effect.sync(() => { + hash.update(bytes); + }), + ), + ); + return hash.digest("hex"); + }); + const runtimes: Record = {}; + for (const target of platforms) { + // `release:cli` and the release workflow's artifacts both lay out --/. + const file = path.join( + archives, + `${release.version}-${target.platform}-${target.arch}`, + platformArchive(target), + ); + if (yield* fs.exists(file)) runtimes[`${target.platform}-${target.arch}`] = yield* sha256(file); + } + if (Object.keys(runtimes).length === 0) + return yield* Effect.die( + new Error(`No release archive for ${release.version} in ${archives}.`), + ); if (yield* fs.exists(directory)) yield* fs.remove(directory, { recursive: true }); yield* fs.makeDirectory(directory, { recursive: true }); yield* fs.writeFileString( @@ -27,18 +58,15 @@ const build = Effect.gen(function* () { bin: { executor: "bin.mjs" }, files: ["bin.mjs", "bin/executor", "bin/package.json", "README.md", "LICENSE"], publishConfig: { access: "public", tag: release.channel }, - optionalDependencies: Object.fromEntries( - platforms.map((target) => [ - platformPackage(target), - `npm:executor@${platformVersion(target)}`, - ]), - ), + // The SHA-256 of each platform's release archive, which the launcher checks a download + // against. npm's integrity covers this file, so it covers the runtime too. + runtimes, }, null, 2, ), ); - // The launcher reads its platform packages and channel from this package.json. + // The launcher reads its version, runtimes and channel from this package.json. yield* fs.copyFile( path.join(root, "scripts/releases/npm-launcher.mjs"), path.join(directory, "bin.mjs"), @@ -74,7 +102,7 @@ const build = Effect.gen(function* () { ); if (code !== 0) return yield* Effect.die(new Error(`npm pack failed (${code})`)); yield* Console.log( - `Launcher archive: ${path.join(directory, `executor-${release.version}.tgz`)}`, + `Launcher archive: ${path.join(directory, `executor-${release.version}.tgz`)} (runtimes: ${Object.keys(runtimes).join(", ")})`, ); });