From ccd1f85f9e3538bacc2f5b37ba777b4d3850f24f Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:55:26 -0400 Subject: [PATCH 001/127] Give each connection its own page and list connections as cards The sidebar's Connect becomes Connections. The list shows the full-access URL and each scoped connection as a card with a Connect button. Clicking a scoped card opens one page that replaces the old detail view and editor: the name is edited in place, apps are turned on and scoped right on their rows, and Save appears only after a change. Copy MCP URL, Rename and Revoke sit in a menu. The shared app row no longer repeats an app's only account next to its subtitle, and can show a host link to connect an account when an app has none. Co-Authored-By: Claude Opus 5.5 --- .../local/server/src/implementation/server.ts | 2 + apps/local/web/src/implementation/app.tsx | 8 +- .../src/implementation/pages/connection.tsx | 617 ++++++++++++++++++ .../src/implementation/pages/connections.tsx | 227 +++++++ .../web/src/implementation/routeTree.gen.ts | 43 ++ .../routes/_dashboard.connections.tsx | 8 + .../_dashboard.connections_.$connectionId.tsx | 13 + .../dashboard/connection-app-picker.tsx | 40 +- .../dashboard/scoped-connections.tsx | 4 +- 9 files changed, 939 insertions(+), 23 deletions(-) create mode 100644 apps/local/web/src/implementation/pages/connection.tsx create mode 100644 apps/local/web/src/implementation/pages/connections.tsx create mode 100644 apps/local/web/src/implementation/routes/_dashboard.connections.tsx create mode 100644 apps/local/web/src/implementation/routes/_dashboard.connections_.$connectionId.tsx diff --git a/apps/local/server/src/implementation/server.ts b/apps/local/server/src/implementation/server.ts index 904019e782..cb9b0c3a31 100644 --- a/apps/local/server/src/implementation/server.ts +++ b/apps/local/server/src/implementation/server.ts @@ -418,6 +418,8 @@ export const localApi = ( HttpRouter.add("GET", "/apps/:app/open", web.document), HttpRouter.add("GET", "/apps/:app/delete", web.document), HttpRouter.add("GET", "/connect", web.document), + HttpRouter.add("GET", "/connections", web.document), + HttpRouter.add("GET", "/connections/:connection", web.document), HttpRouter.add("GET", "/approvals", web.document), HttpRouter.add("GET", "/approvals/:run", web.document), HttpRouter.add("GET", "/account-connect/:connection", web.document), diff --git a/apps/local/web/src/implementation/app.tsx b/apps/local/web/src/implementation/app.tsx index fe5f4fae84..ed677cc8ef 100644 --- a/apps/local/web/src/implementation/app.tsx +++ b/apps/local/web/src/implementation/app.tsx @@ -9,7 +9,7 @@ import { UserCircleIcon, LaptopIcon, PackageIcon, - Plug01Icon, + ConnectIcon, Shield01Icon, } from "@hugeicons/core-free-icons"; import { overviewAtom } from "../contracts/api.ts"; @@ -145,14 +145,14 @@ function Dashboard() { Apps{Option.isSome(data) && {data.value.apps.length}} - - Connect + + Connections + `${count.toLocaleString("en-US")} ${count === 1 ? one : many}`; + +const choicesFor = (inventory: Inventory): ReadonlyMap => { + const profiles = new Map(); + for (const profile of inventory.profiles) { + const list = profiles.get(profile.app); + if (list === undefined) profiles.set(profile.app, [profile]); + else list.push(profile); + } + return new Map( + inventory.apps.map((app) => [ + app.id, + appChoices(app, profiles.get(app.id) ?? [], inventory.accounts), + ]), + ); +}; + +/** A stored connection as an editable draft, keeping removed apps and profiles visible. */ +const draftFrom = ( + connection: ConnectionView, + choices: ReadonlyMap, +): ConnectionDraft => ({ + id: connection.id, + name: connection.name, + apps: connection.policy.apps.map((item): ConnectionApp => { + const choice = choices.get(item.app); + return { + id: item.app, + name: choice?.app.name ?? "Unavailable app", + tools: item.tools, + ...(item.events === undefined ? {} : { events: item.events }), + targets: item.runsAs.map((target): ConnectionTarget => { + if (target.kind === "app") return target; + const option = choice?.options.find( + (candidate) => candidate.target.kind === "profile" && candidate.target.id === target.id, + ); + return option?.target ?? { kind: "profile", id: target.id, label: "Profile unavailable" }; + }), + }; + }), +}); + +export function ConnectionPage({ connectionId }: { readonly connectionId: string }) { + const inventory = useQuery(overviewAtom); + const connections = useQuery(mcpConnectionsAtom); + if (Option.isNone(inventory.data) || Option.isNone(connections.data)) + return ( + + + + ); + const choices = choicesFor(inventory.data.value); + if (connectionId === "new") + return ( + + ); + const stored = connections.data.value.find((item) => item.id === connectionId); + if (stored === undefined) + return ( + + +

Connection unavailable

+

+ It was revoked or no longer exists. +

+ + ); + return ( + + ); +} + +function BackLink() { + return ( + + ); +} + +function ConnectionForm({ + choices, + accounts, + stored, + initial, +}: { + readonly choices: ReadonlyMap; + readonly accounts: Inventory["accounts"]; + /** Undefined while creating. */ + readonly stored: ConnectionView | undefined; + readonly initial: ConnectionDraft; +}) { + const navigate = useNavigate(); + const submit = useAtomSet(saveMcpConnectionAtom, { mode: "promiseExit" }); + const saving = useAtomValue(saveMcpConnectionAtom); + const [draft, setDraft] = useState(initial); + const [expanded, setExpanded] = useState(); + const [connecting, setConnecting] = useState(false); + const [revoking, setRevoking] = useState(false); + const [copied, setCopied] = useState(false); + const [search, setSearch] = useState(""); + const [needsSetupOnly, setNeedsSetupOnly] = useState(false); + const query = useDeferredValue(search.trim().toLowerCase()); + const nameInput = useRef(null); + const renaming = useRef(false); + // Included apps stay on top in a fixed order so rows never jump while you toggle them. + const [pinned] = useState(() => new Set(initial.apps.map((app) => app.id))); + const ordered = useMemo( + () => + [...choices.values()].toSorted( + (a, b) => + Number(pinned.has(b.app.id)) - Number(pinned.has(a.app.id)) || + a.app.name.localeCompare(b.app.name), + ), + [choices, pinned], + ); + const selections = new Map(draft.apps.map((app) => [app.id, app])); + const unavailable = draft.apps.filter((app) => !choices.has(app.id)); + const attention = new Set( + draft.apps.flatMap((selection) => { + const choice = choices.get(selection.id); + return choice !== undefined && selectionIssue(choice, selection) !== undefined + ? [selection.id] + : []; + }), + ); + const blocked = attention.size + unavailable.length; + const visible = ordered.filter( + (choice) => + (!needsSetupOnly || attention.has(choice.app.id)) && + (query === "" || choice.app.name.toLowerCase().includes(query)), + ); + const included = visible.filter((choice) => pinned.has(choice.app.id)); + const others = visible.filter((choice) => !pinned.has(choice.app.id)); + const { count, sentinel } = useIncrementalList(others.length, `${query}:${needsSetupOnly}`); + const addable = visible.filter((choice) => !selections.has(choice.app.id)); + const unassigned = draft.apps.filter( + (app) => app.targets.length === 0 && (choices.get(app.id)?.options.length ?? 0) > 0, + ); + const input = connectionInput(draft); + const creating = stored === undefined; + const dirty = creating || JSON.stringify(input) !== JSON.stringify(connectionInput(initial)); + const valid = + input !== undefined && input.name.length > 0 && draft.apps.length > 0 && blocked === 0; + const setSelection = (id: AppId, next: ConnectionApp | undefined) => + setDraft((previous) => ({ + ...previous, + apps: + next === undefined + ? previous.apps.filter((app) => app.id !== id) + : previous.apps.some((app) => app.id === id) + ? previous.apps.map((app) => (app.id === id ? next : app)) + : [...previous.apps, next], + })); + const renderRow = (choice: AppChoices) => ( + setExpanded(open ? choice.app.id : undefined)} + onChange={(next) => setSelection(choice.app.id, next)} + renderConnectAccount={(app) => ( + + )} + renderTools={({ app, profile, names, onChange }) => ( + + )} + /> + ); + return ( + +
{ + event.preventDefault(); + if (!valid || saving.waiting) return; + const saved = await submit({ existing: !creating, input }); + if (Exit.isSuccess(saved) && creating) + void navigate({ + to: "/connections/$connectionId", + params: { connectionId: saved.value.id }, + }); + }} + > + +
+
+ + setDraft((previous) => ({ ...previous, name: event.target.value })) + } + className="-ml-2 w-full max-w-120 rounded-md border border-transparent bg-transparent px-2 py-0.5 text-[22px] font-semibold tracking-[-0.035em] outline-none placeholder:text-muted-foreground/60 hover:border-border focus-visible:border-ring focus-visible:ring-2 focus-visible:ring-ring/30" + /> +

+ {creating + ? "Turn on the apps this agent needs. It can't see anything else." + : "Agents using this connection see only the apps turned on below. Changes apply right away."} +

+
+ {stored !== undefined && ( +
+ + !open && setCopied(false)}> + + + + { + // Rename moves focus to the title instead of back to the menu button. + if (!renaming.current) return; + renaming.current = false; + event.preventDefault(); + nameInput.current?.select(); + }} + > + { + event.preventDefault(); + void navigator.clipboard.writeText(stored.url).then(() => setCopied(true)); + }} + > + + {copied ? "Copied" : "Copy MCP URL"} + + { + renaming.current = true; + }} + > + + Rename + + + setRevoking(true)} + > + + Revoke connection + + + +
+ )} +
+ + {unavailable.length > 0 && ( +
+ + {plural(unavailable.length, "app")} in this connection{" "} + {unavailable.length === 1 ? "no longer exists" : "no longer exist"}:{" "} + {unavailable.map((app) => app.name).join(", ")} + + +
+ )} + +
+ + {query !== "" && addable.length > 1 && ( + + )} + {unassigned.length > 0 && ( + + setDraft((previous) => ({ + ...previous, + apps: previous.apps.map((app) => { + const choice = choices.get(app.id); + const option = + choice === undefined ? undefined : accountOption(choice, account); + return app.targets.length === 0 && option !== undefined + ? { ...app, targets: [option.target] } + : app; + }), + })) + } + /> + )} +
+ {needsSetupOnly && ( +

+ Showing apps that need setup.{" "} + +

+ )} + + {included.length > 0 && ( + pinned.has(app.id)).length} + hint={undefined} + > + {included.map(renderRow)} + + )} + {others.length > 0 && ( + 0 ? "Other apps" : "Apps"} + count={undefined} + hint={ + pinned.size > 0 && !others.some((choice) => selections.has(choice.app.id)) + ? "Turn one on to give this agent access." + : undefined + } + > + {others.slice(0, count).map(renderRow)} + + )} + {sentinel} + {visible.length === 0 && ( +

+ {query !== "" ? "No apps match your search." : "Every app that's on is ready."} +

+ )} + +
+

+ {draft.apps.length === 0 ? ( + "No apps turned on" + ) : blocked > 0 ? ( + + ) : creating ? ( + `${plural(draft.apps.length, "app")} turned on` + ) : ( + "Unsaved changes" + )} +

+
+ {creating ? ( + + ) : ( + + )} + +
+ {AsyncResult.isFailure(saving) && ( +
+ +
+ )} +
+ + {stored !== undefined && ( + <> + app.name), + } + : undefined + } + onClose={() => setConnecting(false)} + /> + setRevoking(false)} + onRevoked={() => void navigate({ to: "/connections" })} + /> + + )} +
+ ); +} + +function AppSection({ + title, + count, + hint, + children, +}: { + readonly title: string; + readonly count: number | undefined; + readonly hint: string | undefined; + readonly children: ReactNode; +}) { + return ( +
+
+

{title}

+ {count !== undefined && ( + {count} + )} + {hint && {hint}} +
+
{children}
+
+ ); +} + +function RevokeDialog({ + connection, + onClose, + onRevoked, +}: { + readonly connection: ConnectionView | undefined; + readonly onClose: () => void; + readonly onRevoked: () => void; +}) { + const submit = useAtomSet(revokeMcpConnectionAtom, { mode: "promiseExit" }); + const state = useAtomValue(revokeMcpConnectionAtom); + return ( + !open && !state.waiting && onClose()} + > + + + Revoke {connection?.name}? + + Agents using this connection lose access immediately. Its URL stops working. + + + {AsyncResult.isFailure(state) && } + + + + + + + ); +} diff --git a/apps/local/web/src/implementation/pages/connections.tsx b/apps/local/web/src/implementation/pages/connections.tsx new file mode 100644 index 0000000000..38e930bd10 --- /dev/null +++ b/apps/local/web/src/implementation/pages/connections.tsx @@ -0,0 +1,227 @@ +/** + * Connections page prototype (EXE-2). Real connections, apps and MCP URLs only. + * Each scoped card opens its own page, where access is edited in place. + */ +import { useState } from "react"; +import { Link } from "@tanstack/react-router"; +import { Option } from "effect"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { Add01Icon, ConnectIcon } from "@hugeicons/core-free-icons"; +import type { App, AppId } from "@executor-js/sdk"; +import type { ConnectionView } from "@executor-js/mcp-auth/connections"; +import { Button } from "@executor-js/ui/components/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@executor-js/ui/components/dialog"; +import { PageFrame } from "@executor-js/ui/dashboard/page"; +import { ProviderIcon } from "@executor-js/ui/dashboard/common"; +import { McpInstallInstructions } from "@executor-js/ui/dashboard/connect"; +import { useQuery } from "@executor-js/ui/dashboard/context"; +import { providerDisplayUrl } from "@executor-js/ui/contracts/dashboard"; +import { overviewAtom } from "../../contracts/api.ts"; +import { mcpConnectionsAtom } from "../../contracts/mcp-connections.ts"; +import { mcpInstallationAtom } from "../../contracts/mcp.ts"; +import { LoadingRows } from "../components/common.tsx"; + +interface ConnectionApp { + readonly app: App | undefined; + readonly name: string; +} + +/** The full-access endpoint, then each scoped connection. */ +interface Connection { + readonly id: string; + readonly name: string; + readonly url: string; + readonly full: boolean; + readonly apps: readonly ConnectionApp[]; +} + +const connectionsFrom = ( + endpoint: string, + connections: readonly ConnectionView[], + apps: readonly App[], +): readonly Connection[] => { + const byId = new Map(apps.map((app) => [app.id, app])); + return [ + { + id: "default", + name: "Full access", + url: endpoint, + full: true, + apps: apps.map((app) => ({ app, name: app.name })), + }, + ...connections.map((connection) => ({ + id: connection.id, + name: connection.name, + url: connection.url, + full: false, + apps: connection.policy.apps.map((item) => { + const app = byId.get(item.app); + return { app, name: app?.name ?? "Unavailable app" }; + }), + })), + ]; +}; + +export function ConnectionsPage() { + const inventory = useQuery(overviewAtom); + const connections = useQuery(mcpConnectionsAtom); + const installation = useQuery(mcpInstallationAtom); + const [connecting, setConnecting] = useState(); + if ( + Option.isNone(inventory.data) || + Option.isNone(connections.data) || + Option.isNone(installation.data) + ) + return ( + + + + ); + const items = connectionsFrom( + installation.data.value.endpoint, + connections.data.value, + inventory.data.value.apps, + ); + return ( + +
+
+

Connections

+

+ Each connection gives an agent its own MCP URL that reaches only the apps you pick. +

+
+ +
+
+ {items.map((item) => ( + + ))} +
+ app.name), + } + } + onClose={() => setConnecting(undefined)} + /> + + ); +} + +/** What the Connect dialog needs; `apps` is undefined for the full-access URL. */ +export interface ConnectTarget { + readonly name: string; + readonly url: string; + readonly apps: readonly string[] | undefined; +} + +export function ConnectDialog({ + target, + onClose, +}: { + readonly target: ConnectTarget | undefined; + readonly onClose: () => void; +}) { + return ( + !open && onClose()}> + + + Connect {target?.name} + + {target?.apps === undefined + ? "This URL reaches every app. Make a new connection to limit what an agent sees." + : `This URL only reaches ${target.apps.join(", ")}.`} + + + {target && ( + + )} + + + ); +} + +function AppIcon({ item }: { readonly item: ConnectionApp }) { + const provider = + item.app === undefined + ? undefined + : Object.values(item.app.requirements.accounts)[0]?.definition; + return ; +} + +/** Scoped cards open their page; the stretched name link keeps Connect a separate button. */ +function ConnectionCard({ + connection, + onConnect, +}: { + readonly connection: Connection; + readonly onConnect: (connection: Connection) => void; +}) { + const shown = connection.apps.slice(0, 4); + return ( +
+ + {connection.full ? ( + {connection.name} + ) : ( + + {connection.name} + + )} + {connection.full && ( + + Default + + )} + +
    + {shown.map((item) => ( +
  • + + {item.name} +
    • + ))} + {connection.apps.length > shown.length && ( +
  • + and {connection.apps.length - shown.length} more +
    • + )} +
+
+ +
+
+ ); +} diff --git a/apps/local/web/src/implementation/routeTree.gen.ts b/apps/local/web/src/implementation/routeTree.gen.ts index badaee9bc6..c49f9a1f48 100644 --- a/apps/local/web/src/implementation/routeTree.gen.ts +++ b/apps/local/web/src/implementation/routeTree.gen.ts @@ -13,12 +13,14 @@ import { Route as DashboardRouteImport } from './routes/_dashboard' import { Route as AppAuthRouteImport } from './routes/app-auth' import { Route as DashboardInventoryRouteImport } from './routes/_dashboard._inventory' import { Route as DashboardConnectRouteImport } from './routes/_dashboard.connect' +import { Route as DashboardConnectionsRouteImport } from './routes/_dashboard.connections' import { Route as AccountConnectConnectionIdRouteImport } from './routes/account-connect.$connectionId' import { Route as McpAuthorizeRouteImport } from './routes/mcp.authorize' import { Route as DashboardInventoryIndexRouteImport } from './routes/_dashboard._inventory.index' import { Route as DashboardInventorySplatRouteImport } from './routes/_dashboard._inventory.$' import { Route as DashboardApprovalsIndexRouteImport } from './routes/_dashboard.approvals.index' import { Route as DashboardApprovalsRunIdRouteImport } from './routes/_dashboard.approvals.$runId' +import { Route as DashboardConnectionsConnectionIdRouteImport } from './routes/_dashboard.connections_.$connectionId' import { Route as McpApproveRequestIdRouteImport } from './routes/mcp.approve.$requestId' import { Route as DashboardInventoryAccountsIndexRouteImport } from './routes/_dashboard._inventory.accounts.index' import { Route as DashboardInventoryAppsIndexRouteImport } from './routes/_dashboard._inventory.apps.index' @@ -49,6 +51,11 @@ const DashboardConnectRoute = DashboardConnectRouteImport.update({ path: '/connect', getParentRoute: () => DashboardRoute, } as any) +const DashboardConnectionsRoute = DashboardConnectionsRouteImport.update({ + id: '/connections', + path: '/connections', + getParentRoute: () => DashboardRoute, +} as any) const AccountConnectConnectionIdRoute = AccountConnectConnectionIdRouteImport.update({ id: '/account-connect/$connectionId', @@ -80,6 +87,12 @@ const DashboardApprovalsRunIdRoute = DashboardApprovalsRunIdRouteImport.update({ path: '/approvals/$runId', getParentRoute: () => DashboardRoute, } as any) +const DashboardConnectionsConnectionIdRoute = + DashboardConnectionsConnectionIdRouteImport.update({ + id: '/connections_/$connectionId', + path: '/connections/$connectionId', + getParentRoute: () => DashboardRoute, + } as any) const McpApproveRequestIdRoute = McpApproveRequestIdRouteImport.update({ id: '/mcp/approve/$requestId', path: '/mcp/approve/$requestId', @@ -150,10 +163,12 @@ export interface FileRoutesByFullPath { '/': typeof DashboardInventoryIndexRoute '/app-auth': typeof AppAuthRoute '/connect': typeof DashboardConnectRoute + '/connections': typeof DashboardConnectionsRoute '/account-connect/$connectionId': typeof AccountConnectConnectionIdRoute '/mcp/authorize': typeof McpAuthorizeRoute '/$': typeof DashboardInventorySplatRoute '/approvals/$runId': typeof DashboardApprovalsRunIdRoute + '/connections/$connectionId': typeof DashboardConnectionsConnectionIdRoute '/mcp/approve/$requestId': typeof McpApproveRequestIdRoute '/approvals/': typeof DashboardApprovalsIndexRoute '/apps/$appId': typeof DashboardInventoryAppsAppIdRoute @@ -171,10 +186,12 @@ export interface FileRoutesByTo { '/': typeof DashboardInventoryIndexRoute '/app-auth': typeof AppAuthRoute '/connect': typeof DashboardConnectRoute + '/connections': typeof DashboardConnectionsRoute '/account-connect/$connectionId': typeof AccountConnectConnectionIdRoute '/mcp/authorize': typeof McpAuthorizeRoute '/$': typeof DashboardInventorySplatRoute '/approvals/$runId': typeof DashboardApprovalsRunIdRoute + '/connections/$connectionId': typeof DashboardConnectionsConnectionIdRoute '/mcp/approve/$requestId': typeof McpApproveRequestIdRoute '/approvals': typeof DashboardApprovalsIndexRoute '/apps/$appId': typeof DashboardInventoryAppsAppIdRoute @@ -194,10 +211,12 @@ export interface FileRoutesById { '/app-auth': typeof AppAuthRoute '/_dashboard/_inventory': typeof DashboardInventoryRouteWithChildren '/_dashboard/connect': typeof DashboardConnectRoute + '/_dashboard/connections': typeof DashboardConnectionsRoute '/account-connect/$connectionId': typeof AccountConnectConnectionIdRoute '/mcp/authorize': typeof McpAuthorizeRoute '/_dashboard/_inventory/$': typeof DashboardInventorySplatRoute '/_dashboard/approvals/$runId': typeof DashboardApprovalsRunIdRoute + '/_dashboard/connections_/$connectionId': typeof DashboardConnectionsConnectionIdRoute '/mcp/approve/$requestId': typeof McpApproveRequestIdRoute '/_dashboard/_inventory/': typeof DashboardInventoryIndexRoute '/_dashboard/approvals/': typeof DashboardApprovalsIndexRoute @@ -218,10 +237,12 @@ export interface FileRouteTypes { | '/' | '/app-auth' | '/connect' + | '/connections' | '/account-connect/$connectionId' | '/mcp/authorize' | '/$' | '/approvals/$runId' + | '/connections/$connectionId' | '/mcp/approve/$requestId' | '/approvals/' | '/apps/$appId' @@ -239,10 +260,12 @@ export interface FileRouteTypes { | '/' | '/app-auth' | '/connect' + | '/connections' | '/account-connect/$connectionId' | '/mcp/authorize' | '/$' | '/approvals/$runId' + | '/connections/$connectionId' | '/mcp/approve/$requestId' | '/approvals' | '/apps/$appId' @@ -261,10 +284,12 @@ export interface FileRouteTypes { | '/app-auth' | '/_dashboard/_inventory' | '/_dashboard/connect' + | '/_dashboard/connections' | '/account-connect/$connectionId' | '/mcp/authorize' | '/_dashboard/_inventory/$' | '/_dashboard/approvals/$runId' + | '/_dashboard/connections_/$connectionId' | '/mcp/approve/$requestId' | '/_dashboard/_inventory/' | '/_dashboard/approvals/' @@ -318,6 +343,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof DashboardConnectRouteImport parentRoute: typeof DashboardRoute } + '/_dashboard/connections': { + id: '/_dashboard/connections' + path: '/connections' + fullPath: '/connections' + preLoaderRoute: typeof DashboardConnectionsRouteImport + parentRoute: typeof DashboardRoute + } '/account-connect/$connectionId': { id: '/account-connect/$connectionId' path: '/account-connect/$connectionId' @@ -360,6 +392,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof DashboardApprovalsRunIdRouteImport parentRoute: typeof DashboardRoute } + '/_dashboard/connections_/$connectionId': { + id: '/_dashboard/connections_/$connectionId' + path: '/connections/$connectionId' + fullPath: '/connections/$connectionId' + preLoaderRoute: typeof DashboardConnectionsConnectionIdRouteImport + parentRoute: typeof DashboardRoute + } '/mcp/approve/$requestId': { id: '/mcp/approve/$requestId' path: '/mcp/approve/$requestId' @@ -476,7 +515,9 @@ const DashboardInventoryRouteWithChildren = interface DashboardRouteChildren { DashboardInventoryRoute: typeof DashboardInventoryRouteWithChildren DashboardConnectRoute: typeof DashboardConnectRoute + DashboardConnectionsRoute: typeof DashboardConnectionsRoute DashboardApprovalsRunIdRoute: typeof DashboardApprovalsRunIdRoute + DashboardConnectionsConnectionIdRoute: typeof DashboardConnectionsConnectionIdRoute DashboardApprovalsIndexRoute: typeof DashboardApprovalsIndexRoute DashboardWebhooksAppIdSubscriptionIdRoute: typeof DashboardWebhooksAppIdSubscriptionIdRoute } @@ -484,7 +525,9 @@ interface DashboardRouteChildren { const DashboardRouteChildren: DashboardRouteChildren = { DashboardInventoryRoute: DashboardInventoryRouteWithChildren, DashboardConnectRoute: DashboardConnectRoute, + DashboardConnectionsRoute: DashboardConnectionsRoute, DashboardApprovalsRunIdRoute: DashboardApprovalsRunIdRoute, + DashboardConnectionsConnectionIdRoute: DashboardConnectionsConnectionIdRoute, DashboardApprovalsIndexRoute: DashboardApprovalsIndexRoute, DashboardWebhooksAppIdSubscriptionIdRoute: DashboardWebhooksAppIdSubscriptionIdRoute, diff --git a/apps/local/web/src/implementation/routes/_dashboard.connections.tsx b/apps/local/web/src/implementation/routes/_dashboard.connections.tsx new file mode 100644 index 0000000000..33d7ed8b69 --- /dev/null +++ b/apps/local/web/src/implementation/routes/_dashboard.connections.tsx @@ -0,0 +1,8 @@ +import { createFileRoute } from "@tanstack/react-router"; +import { ConnectionsPage } from "../pages/connections.tsx"; + +/** Generated-tree route for /_dashboard/connections. */ +export const Route = createFileRoute("/_dashboard/connections")({ + staticData: { section: "connect" }, + component: ConnectionsPage, +}); diff --git a/apps/local/web/src/implementation/routes/_dashboard.connections_.$connectionId.tsx b/apps/local/web/src/implementation/routes/_dashboard.connections_.$connectionId.tsx new file mode 100644 index 0000000000..6953f97fca --- /dev/null +++ b/apps/local/web/src/implementation/routes/_dashboard.connections_.$connectionId.tsx @@ -0,0 +1,13 @@ +import { createFileRoute } from "@tanstack/react-router"; +import { ConnectionPage } from "../pages/connection.tsx"; + +/** Generated-tree route for /_dashboard/connections_/$connectionId. */ +export const Route = createFileRoute("/_dashboard/connections_/$connectionId")({ + staticData: { section: "connect" }, + component: ConnectionRoute, +}); + +function ConnectionRoute() { + const { connectionId } = Route.useParams(); + return ; +} diff --git a/packages/ui/src/implementation/dashboard/connection-app-picker.tsx b/packages/ui/src/implementation/dashboard/connection-app-picker.tsx index 305086c324..796edb1c52 100644 --- a/packages/ui/src/implementation/dashboard/connection-app-picker.tsx +++ b/packages/ui/src/implementation/dashboard/connection-app-picker.tsx @@ -149,6 +149,7 @@ export function ConnectionAppRow({ onExpandedChange, onChange, renderTools, + renderConnectAccount, }: { readonly choices: AppChoices; readonly selection: ConnectionApp | undefined; @@ -156,6 +157,8 @@ export function ConnectionAppRow({ readonly onExpandedChange: (expanded: boolean) => void; readonly onChange: (selection: ConnectionApp | undefined) => void; readonly renderTools: (props: ConnectionToolPickerProps) => ReactNode; + /** A host link to connect an account, shown when the app has none yet. */ + readonly renderConnectAccount?: (app: App) => ReactNode; }) { const id = useId(); const { app } = choices; @@ -193,17 +196,27 @@ export function ConnectionAppRow({ issue === undefined ? "text-muted-foreground" : "text-amber-600 dark:text-amber-400" }`} > - {issue ?? - (!choices.needsAccount - ? "No account needed" - : choices.options.length === 0 - ? "No connected accounts" - : only !== undefined - ? connectionTargetLabel(only.target) - : `${choices.options.length} ways to run`)} + {renderConnectAccount !== undefined && + choices.needsAccount && + choices.options.length === 0 + ? "No connected accounts" + : (issue ?? + (!choices.needsAccount + ? "No account needed" + : choices.options.length === 0 + ? "No connected accounts" + : only !== undefined + ? connectionTargetLabel(only.target) + : `${choices.options.length} ways to run`))} + {renderConnectAccount !== undefined && + choices.needsAccount && + choices.options.length === 0 && ( + // Outside the label so following the link never toggles the app. + {renderConnectAccount(app)} + )} {selection !== undefined && (
{choices.needsAccount && ( @@ -375,15 +388,8 @@ function AccountsPicker({ }) { const selected = new Set(selection.targets.map(connectionTargetKey)); const only = choices.options.length === 1 ? choices.options[0] : undefined; - if (only !== undefined && selected.has(connectionTargetKey(only.target))) - return ( - - {connectionTargetLabel(only.target)} - - ); + // The row's subtitle already names the only way to run it. + if (only !== undefined && selected.has(connectionTargetKey(only.target))) return null; const label = selection.targets.length === 0 ? "Runs as…" diff --git a/packages/ui/src/implementation/dashboard/scoped-connections.tsx b/packages/ui/src/implementation/dashboard/scoped-connections.tsx index 52d4b433c4..95d5ac0d3d 100644 --- a/packages/ui/src/implementation/dashboard/scoped-connections.tsx +++ b/packages/ui/src/implementation/dashboard/scoped-connections.tsx @@ -420,7 +420,7 @@ function ConnectionList({ ); } -function SearchInput({ +export function SearchInput({ value, onChange, placeholder, @@ -491,7 +491,7 @@ function Tabs({ } /** Bulk-including apps leaves each waiting for an account; assign one account to all that accept it. */ -function AssignAccount({ +export function AssignAccount({ apps, byId, accounts, From b5ffc28358f1aeb6792b9d8f5e73277381810ce4 Mon Sep 17 00:00:00 2001 From: Leo <108278866+leoisadev1@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:20:55 -0400 Subject: [PATCH 002/127] Fill app logo tiles edge to edge Logos rendered at 17px inside a bordered 34px box. They now fill a rounded white tile with a hairline edge, like the integrations catalog, fetched at a size that stays sharp in the largest tile. The package fallback keeps its bordered box. Pages that shrank the logo image inside the tile no longer do. Co-Authored-By: Claude Opus 5.5 --- .../src/implementation/pages/connections.tsx | 2 +- .../implementation/dashboard/app-accounts.tsx | 2 +- .../implementation/dashboard/app-detail.tsx | 2 +- .../src/implementation/dashboard/common.tsx | 19 ++++++++++++------- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/apps/local/web/src/implementation/pages/connections.tsx b/apps/local/web/src/implementation/pages/connections.tsx index 38e930bd10..64f6c45908 100644 --- a/apps/local/web/src/implementation/pages/connections.tsx +++ b/apps/local/web/src/implementation/pages/connections.tsx @@ -204,7 +204,7 @@ function ConnectionCard({ {shown.map((item) => (
  • {item.name} diff --git a/packages/ui/src/implementation/dashboard/app-accounts.tsx b/packages/ui/src/implementation/dashboard/app-accounts.tsx index 228f48c159..b10ae4329a 100644 --- a/packages/ui/src/implementation/dashboard/app-accounts.tsx +++ b/packages/ui/src/implementation/dashboard/app-accounts.tsx @@ -404,7 +404,7 @@ export function AppAccounts({ } className="min-w-0 space-y-2.5" > -
    +
    -
    +
    ([]); const normalized = name.toLowerCase().trim(); - const size = large ? 24 : 17; + // Fetched large enough to stay sharp in the biggest tile on a 2x screen. + const fetchSize = 64; const domain = url ?? domains.get(normalized) ?? (/^[a-z\d-]+(?:\.[a-z\d-]+)+$/i.test(normalized) ? normalized : null); - const source = normalized === "executor" ? "/favicon.png" : faviconUrl(domain, size); + const source = normalized === "executor" ? "/favicon.png" : faviconUrl(domain, fetchSize); const icon = source !== null && !failed.includes(source) ? source : null; return ( _svg]:w-4.25 [&_>_svg]:h-4.25 [&_>_svg]:object-contain max-[740px]:[.catalog-row_>_&]:row-[1_/_3]", - large && - "provider-icon-large w-11 h-11 rounded-[9px] [&_img]:w-6 [&_img]:h-6 [&_>_svg]:w-6 [&_>_svg]:h-6", + "provider-icon relative inline-flex size-8.5 shrink-0 items-center justify-center overflow-hidden rounded-[8px] max-[740px]:[.catalog-row_>_&]:row-[1_/_3]", + large && "provider-icon-large size-11 rounded-[10px]", + // A logo fills its tile edge to edge; the hairline keeps white logos visible on light pages. + icon + ? "bg-white after:pointer-events-none after:absolute after:inset-0 after:rounded-[inherit] after:ring-1 after:ring-foreground/10 after:ring-inset [&_img]:size-full [&_img]:object-cover" + : "border border-border bg-background [&_>_svg]:size-4.25", + !icon && large && "[&_>_svg]:size-6", )} aria-hidden > @@ -42,8 +47,8 @@ export function ProviderIcon({ setFailed((current) => (current.includes(icon) ? current : [...current, icon])) From 1d77e427008ac950f3575d6c51a89f6879b4dfb2 Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Sat, 10 Oct 2026 00:58:11 -0500 Subject: [PATCH 003/127] fix(self-host): stop tracing idle scheduler and provisioning polls Self-host checks for due scheduled runs and for queued provisioning jobs every second, and traced every check: a schedule.dispatch span with its storage and SQL spans, plus a root sql.execute span. An instance with no traffic exported about 800 spans a minute into the bundled Motel store, which reaches its 1 GiB bound within hours and then evicts on every export, writing to disk constantly. A check that finds nothing now records nothing. Each scheduled operation a check finds runs in its own schedule.dispatch trace, so the background work under it keeps its dispatcher root, and each provisioning job is traced as before. Local runs the same scheduler and stops tracing its idle checks too. Co-Authored-By: Claude Opus 5.5 --- .changeset/untraced-idle-polls.md | 11 +++++++++++ .../server/src/implementation/provisioning.ts | 2 ++ packages/sdk/src/implementation/schedule-worker.ts | 13 ++++++++++--- 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 .changeset/untraced-idle-polls.md diff --git a/.changeset/untraced-idle-polls.md b/.changeset/untraced-idle-polls.md new file mode 100644 index 0000000000..8a03940249 --- /dev/null +++ b/.changeset/untraced-idle-polls.md @@ -0,0 +1,11 @@ +--- +"@executor-js/sdk": patch +"@executor-js/hosted-server": patch +--- + +The scheduler's check for due runs, every second on self-host and local, and +self-host's check for queued provisioning jobs, every second, no longer record +traces when they find nothing to do. Each scheduled operation a check finds is +still traced, in its own `schedule.dispatch` trace, and so is each provisioning +job. An idle self-host instance no longer fills the bundled Motel store with +polling spans. diff --git a/apps/hosted/server/src/implementation/provisioning.ts b/apps/hosted/server/src/implementation/provisioning.ts index e2e94b8345..377d9d6e81 100644 --- a/apps/hosted/server/src/implementation/provisioning.ts +++ b/apps/hosted/server/src/implementation/provisioning.ts @@ -210,6 +210,8 @@ export const drainProvisioning = (services: ProvisioningServices) => Effect.flatMap( Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ id: Schema.String }))), ), + // Self-host polls every second. The poll records nothing; the jobs it finds are traced. + Effect.withTracerEnabled(false), ); for (const job of jobs) yield* provision(job.id, services).pipe( diff --git a/packages/sdk/src/implementation/schedule-worker.ts b/packages/sdk/src/implementation/schedule-worker.ts index d768a15c93..ead6a165ec 100644 --- a/packages/sdk/src/implementation/schedule-worker.ts +++ b/packages/sdk/src/implementation/schedule-worker.ts @@ -23,11 +23,18 @@ export const startScheduleWorker = ( runner: config.runner, maxCandidates: config.concurrency, authorize, - execute: (operation) => pool.withPermitsIfAvailable(1)(operation).pipe(Effect.asVoid), + // Each operation a pass finds is one trace, rooted in a dispatch as a Cloud + // coordinator's pass is. + execute: (operation) => + pool + .withPermitsIfAvailable(1)( + operation.pipe(Effect.withSpan("schedule.dispatch"), Effect.withTracerEnabled(true)), + ) + .pipe(Effect.asVoid), }) .pipe( - // Each pass is one trace, as a Cloud coordinator's dispatch is. - Effect.withSpan("schedule.dispatch"), + // A pass runs every second; one that finds nothing to run records nothing. + Effect.withTracerEnabled(false), Effect.catch(() => Effect.logError("Scheduled dispatch failed")), Effect.forkIn(scope), Effect.asVoid, From 1af069e540da94df00f19a5dec9223027c9ee020 Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Sat, 10 Oct 2026 01:17:47 -0500 Subject: [PATCH 004/127] fix(self-host): stop tracing idle event delivery polls Self-host and local look for due event deliveries every two seconds, and traced every pass: an events.dispatch span with its delivery, subscription and retention statements, about 15 spans each. On an instance with no traffic that is about 450 spans a minute in the bundled Motel store, the largest idle source since the event worker arrived. A pass that finds nothing to deliver now records nothing. Each delivery attempt and each subscription a pass expires is still traced, in its own trace. Cloud's coordinator calls the same delivery code with tracing on and is unchanged. Co-Authored-By: Claude Opus 5.5 --- .changeset/untraced-event-poll.md | 8 ++++++++ packages/sdk/src/implementation/event-worker.ts | 3 ++- packages/sdk/src/implementation/events.ts | 12 ++++++++++-- 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 .changeset/untraced-event-poll.md diff --git a/.changeset/untraced-event-poll.md b/.changeset/untraced-event-poll.md new file mode 100644 index 0000000000..10ada67374 --- /dev/null +++ b/.changeset/untraced-event-poll.md @@ -0,0 +1,8 @@ +--- +"@executor-js/sdk": patch +--- + +Self-host and local check for due event deliveries every two seconds. That +check no longer records traces when it finds nothing to deliver; each delivery +attempt and each subscription it expires is still traced. An idle self-host +instance no longer fills the bundled Motel store with polling spans. diff --git a/packages/sdk/src/implementation/event-worker.ts b/packages/sdk/src/implementation/event-worker.ts index 12fb1d95a0..e4009f441a 100644 --- a/packages/sdk/src/implementation/event-worker.ts +++ b/packages/sdk/src/implementation/event-worker.ts @@ -12,7 +12,8 @@ export const deliverEvents = (executor: Executor) => yield* Effect.flatten(ScheduleHostReady); yield* executor.events.deliver({ maxDeliveries: batch }).pipe( Effect.repeat({ while: (more) => more }), - Effect.withSpan("events.dispatch"), + // A pass runs every two seconds; one that finds nothing to deliver records nothing. + Effect.withTracerEnabled(false), Effect.catch(() => Effect.logError("Event delivery failed")), Effect.repeat(Schedule.spaced("2 seconds")), ); diff --git a/packages/sdk/src/implementation/events.ts b/packages/sdk/src/implementation/events.ts index fde0f19d89..e3361d5e1e 100644 --- a/packages/sdk/src/implementation/events.ts +++ b/packages/sdk/src/implementation/events.ts @@ -808,7 +808,11 @@ export const makeEvents = (input: { limit: 100, }), ); - for (const row of lapsed) yield* stopWith(EventSubscriptionId.make(row.id), "expired"); + // A host may poll without tracing; the work a pass finds is traced all the same. + for (const row of lapsed) + yield* stopWith(EventSubscriptionId.make(row.id), "expired").pipe( + Effect.withTracerEnabled(true), + ); const due = yield* query(() => db.findMany("eventDeliveries", { where: (b) => @@ -825,7 +829,11 @@ export const makeEvents = (input: { due, (row) => claim(row.id).pipe( - Effect.flatMap((claimed) => (claimed === null ? Effect.void : attempt(claimed))), + Effect.flatMap((claimed) => + claimed === null + ? Effect.void + : attempt(claimed).pipe(Effect.withTracerEnabled(true)), + ), Effect.catch(() => Effect.logWarning("Event delivery attempt failed")), ), { concurrency: 8, discard: true }, From 59c5fbc42c11ab04d10d06b52404160001a95fe2 Mon Sep 17 00:00:00 2001 From: Davies Ayo <29122859+daviesayo@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:18:29 +1100 Subject: [PATCH 005/127] fix(cloud): let the schedule coordinator hibernate between passes The schedule coordinator's database window stayed open for two minutes after each pass, while the retired coordinator's heartbeat and the recovery alarm woke it at least once a minute. The open sockets and the pending idle timer kept the object from ever becoming eligible for hibernation, so it billed duration for every second of the day. An object database now takes the time its window lingers after the last call. The schedules owner passes zero, so its window closes as soon as a pass ends. MCP sessions keep the two-minute window. --- apps/hosted/cloud/src/infrastructure/object-database.ts | 8 +++++--- apps/hosted/cloud/src/infrastructure/schedules.ts | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/hosted/cloud/src/infrastructure/object-database.ts b/apps/hosted/cloud/src/infrastructure/object-database.ts index 5aecf443a4..a3965f363c 100644 --- a/apps/hosted/cloud/src/infrastructure/object-database.ts +++ b/apps/hosted/cloud/src/infrastructure/object-database.ts @@ -103,7 +103,8 @@ const applicationName = (owner: string) => `executor ${owner}`.slice(0, 63); * window's connections; other calls use the rest and never join the transaction. A connection * the server drops is replaced on its next use, by the pool itself. Opening a connection gets a * second attempt when the first fails or times out ({@link cloudDatabasePool}). When the last - * call ends, the window closes after {@link idleWindow}; the next call opens a new one. A caller + * call ends, the window closes after the owner's linger, {@link idleWindow} unless the owner + * gives another; the next call opens a new one. A caller * that gives up a connection retires the window instead: later calls open a new one, and the * retired window closes as soon as its last call ends, closing that connection with it. A call * that never ends therefore keeps its retired window open, and the object's windows together can @@ -112,7 +113,7 @@ const applicationName = (owner: string) => `executor ${owner}`.slice(0, 63); */ export const cloudObjectDatabase = Effect.gen(function* () { const connection = yield* cloudDirectDatabaseConnection; - return Effect.fn(function* (owner: string) { + return Effect.fn(function* (owner: string, linger: Duration.Duration = idleWindow) { // Owned by the object for its whole life; only windows are ever closed. const root = Scope.makeUnsafe(); const lock = yield* Semaphore.make(1); @@ -193,7 +194,8 @@ export const cloudObjectDatabase = Effect.gen(function* () { if (leased.leases > 0) return; // A retired window closes once its last caller is done. if (window !== leased) return yield* Scope.close(leased.scope, Exit.void); - leased.idle = yield* Effect.sleep(idleWindow).pipe( + if (Duration.isZero(linger)) return yield* close(leased); + leased.idle = yield* Effect.sleep(linger).pipe( Effect.andThen(lock.withPermits(1)(close(leased))), Effect.forkIn(root), ); diff --git a/apps/hosted/cloud/src/infrastructure/schedules.ts b/apps/hosted/cloud/src/infrastructure/schedules.ts index 10391a3353..d5322b723b 100644 --- a/apps/hosted/cloud/src/infrastructure/schedules.ts +++ b/apps/hosted/cloud/src/infrastructure/schedules.ts @@ -12,7 +12,7 @@ import { scheduleRecoveryMilliseconds } from "../contracts/schedules.ts"; import * as Cloudflare from "alchemy/Cloudflare"; import { RuntimeContext } from "alchemy"; import { makeExecutionMemo } from "alchemy/Runtime/ExecutionMemo"; -import { Config, Clock, Effect, Exit, Layer, Schema, Semaphore } from "effect"; +import { Config, Clock, Duration, Effect, Exit, Layer, Schema, Semaphore } from "effect"; import { HostedExecutor, ScheduledAuthority, ScheduleWakeup } from "@executor-js/hosted-server"; import { defaultScheduleWorkerOptions } from "@executor-js/sdk/scheduling"; import { cloudProduct } from "./product.ts"; @@ -63,7 +63,7 @@ const makePlacedScheduleCoordinator = Effect.gen(function* () { const state = yield* Cloudflare.DurableObjectState; const lifetime = yield* previewLifetime; // Alarms, wakes and the dispatched runs share the coordinator's held connections. - const database = yield* objectDatabase("schedules"); + const database = yield* objectDatabase("schedules", Duration.zero); const pool = yield* Semaphore.make(concurrency); const lifecycle = yield* Semaphore.make(1); const alarms = yield* Semaphore.make(1); From ffb55ee7bbb7f165b649c6ba4cd60942157da2e1 Mon Sep 17 00:00:00 2001 From: Abhishek B R Date: Sat, 17 Oct 2026 15:38:31 +0530 Subject: [PATCH 006/127] Keep MCP tools listed when a server refuses the optional event stream The Streamable HTTP client opens a standalone GET event stream after initialization. Some servers answer it with 401 even though every POST succeeds, and transportFetch failed the whole session on that response, so discovery reported the app's credentials as rejected. Hand the refusal back to the SDK instead, which treats it as no stream, as it does without Executor. POSTs and the legacy SSE GET are unchanged. Fixes #2306 --- e2e/test-plan.ts | 10 ++ e2e/tests/mcp-event-stream.spec.ts | 154 ++++++++++++++++++++++++ packages/apps/src/implementation/mcp.ts | 9 +- 3 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 e2e/tests/mcp-event-stream.spec.ts diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index 613bc8213f..4c8a4585ba 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -2015,6 +2015,16 @@ export const scenarios = plan({ local: na("Exercises the hosted app deploy and tool call APIs."), }, }, + mcpRefusedEventStream: { + fixtures: "actors", + file: "mcp-event-stream.spec.ts", + title: "Remote MCP tools stay available when the server refuses the optional event stream", + targets: { + "self-host": scheduled, + cloud: na("Uses a loopback upstream to exercise the shared Worker MCP transport."), + local: na("Exercises the hosted app deploy and tool call APIs."), + }, + }, templateAccounts: { fixtures: "actors", file: "template-accounts.spec.ts", diff --git a/e2e/tests/mcp-event-stream.spec.ts b/e2e/tests/mcp-event-stream.spec.ts new file mode 100644 index 0000000000..95c8ac6c98 --- /dev/null +++ b/e2e/tests/mcp-event-stream.spec.ts @@ -0,0 +1,154 @@ +/** A remote MCP server that refuses the optional event stream still serves its tools. */ +import { expect, layer } from "@effect/vitest"; +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; +import { Deferred, Effect, Layer, Ref, Schema } from "effect"; +import { HttpRouter, HttpServer, HttpServerRequest, HttpServerResponse } from "effect/http"; +import { createServer } from "node:http"; +import { randomUUID } from "node:crypto"; +import { scenarios } from "../test-plan.ts"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { App } from "../support/contracts.ts"; +import { Evidence } from "../support/evidence.ts"; +import { mcpSdkVersion, withApps } from "../support/apps-release.ts"; + +const JsonRpcRequest = Schema.Struct({ + id: Schema.optional(Schema.Union([Schema.String, Schema.Number])), + method: Schema.String, + params: Schema.optional(Schema.Struct({ protocolVersion: Schema.optional(Schema.String) })), +}); + +/** + * An MCP server that answers every POST but refuses the standalone GET event stream with a 401 + * and a JSON-RPC error, as some hosted servers do instead of the 405 the spec asks for. It holds + * `tools/list` until that GET has arrived, so the refusal reaches the client during discovery. + */ +const refusedStreamMcpServer = Effect.gen(function* () { + const requests = yield* Ref.make>([]); + const streamRequested = yield* Deferred.make(); + const answer = (id: string | number, result: object) => + HttpServerResponse.json({ jsonrpc: "2.0", id, result }); + const handler = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + if (request.method === "GET") { + yield* Ref.update(requests, (seen) => [...seen, "GET 401"]); + yield* Deferred.succeed(streamRequested, undefined); + return yield* HttpServerResponse.json( + { + jsonrpc: "2.0", + error: { code: -32600, message: "Authorization required" }, + id: null, + }, + { + status: 401, + headers: { + "www-authenticate": `Bearer resource_metadata="http://127.0.0.1/.well-known/oauth-protected-resource"`, + }, + }, + ); + } + if (request.method !== "POST") return HttpServerResponse.empty({ status: 405 }); + const message = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(JsonRpcRequest))( + yield* request.text, + ); + yield* Ref.update(requests, (seen) => [...seen, `POST ${message.method}`]); + if (message.id === undefined) return HttpServerResponse.empty({ status: 202 }); + switch (message.method) { + case "initialize": + return yield* answer(message.id, { + protocolVersion: message.params?.protocolVersion, + capabilities: { tools: {} }, + serverInfo: { name: "Refused stream fixture", version: "1.0.0" }, + }); + case "tools/list": + yield* Deferred.await(streamRequested).pipe(Effect.timeout("5 seconds"), Effect.ignore); + return yield* answer(message.id, { + tools: [ + { + name: "ping", + description: "Answer pong", + inputSchema: { type: "object" }, + annotations: { readOnlyHint: true }, + }, + ], + }); + case "tools/call": + return yield* answer(message.id, { + content: [{ type: "text", text: "pong" }], + structuredContent: { reply: "pong" }, + }); + default: + return yield* HttpServerResponse.json({ + jsonrpc: "2.0", + id: message.id, + error: { code: -32601, message: "Method not found" }, + }); + } + }); + const services = yield* Layer.build( + HttpRouter.serve(HttpRouter.add("*", "/mcp", handler), { + disableLogger: true, + disableListenLog: true, + }).pipe(Layer.provideMerge(NodeHttpServer.layer(createServer, { host: "127.0.0.1", port: 0 }))), + ); + const server = yield* HttpServer.HttpServer.pipe(Effect.provideContext(services)); + if (!("port" in server.address)) return yield* Effect.die("Fixture must listen on TCP"); + return { url: `http://127.0.0.1:${server.address.port}/mcp`, requests: Ref.get(requests) }; +}); + +const appFiles = (url: string) => [ + { + path: "package.json", + content: JSON.stringify({ + dependencies: withApps({ "@modelcontextprotocol/sdk": mcpSdkVersion }), + }), + }, + { + path: "index.ts", + content: `import { defineApp, router } from "apps"; +import { mcpRouter } from "apps/mcp"; +export default defineApp({}, async (ctx) => ({ tools: router({ + remote: await mcpRouter({ url: ${JSON.stringify(url)}, signal: ctx.signal }), +}) }));`, + }, +]; + +layer(HostedLive, { excludeTestServices: true })("MCP event stream", (it) => { + it.effect(scenarios.mcpRefusedEventStream.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors, + evidence = yield* Evidence; + const server = yield* refusedStreamMcpServer; + const prefix = `/api/organizations/${actors.organization.id}`; + const deployed = yield* api.request(actors.owner, "POST", `${prefix}/apps/deploy`, { + name: `MCP event stream ${randomUUID().slice(0, 8)}`, + files: appFiles(server.url), + }); + yield* evidence.json("deploy.json", deployed); + expect(deployed.status, JSON.stringify(deployed.body)).toBe(200); + const app = yield* body(App, deployed); + const path = `${prefix}/apps/${app.id}`; + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", path).pipe(Effect.orDie), + ); + const called = yield* api.request(actors.owner, "POST", `${path}/tools/call`, { + tool: "remote.ping", + kind: "query", + input: {}, + }); + yield* evidence.json("call.json", called); + expect(called.status, JSON.stringify(called.body)).toBe(200); + expect(called.body).toMatchObject({ structuredContent: { reply: "pong" } }); + // The server did refuse the stream during the session; the tools were listed regardless. + const requests = yield* server.requests; + yield* evidence.json("upstream-requests.json", requests); + expect(requests).toContain("GET 401"); + expect(requests).toContain("POST tools/list"); + }), + ), + ); +}); diff --git a/packages/apps/src/implementation/mcp.ts b/packages/apps/src/implementation/mcp.ts index 0d7b072f19..b94284c914 100644 --- a/packages/apps/src/implementation/mcp.ts +++ b/packages/apps/src/implementation/mcp.ts @@ -154,6 +154,7 @@ const transportFetch = telemetry: Effect.Success, rejected: Deferred.Deferred, answered: Ref.Ref, + kind: "http" | "sse", ): FetchLike => (url, init) => Effect.runPromiseWith(telemetry.context)( @@ -197,7 +198,11 @@ const transportFetch = ); const { body, upstream } = yield* errorBody(response); const provider = httpProviderError(response.status, response.headers); - if (provider !== undefined) { + // The Streamable HTTP client's GET only opens the optional server-initiated stream. A + // server can refuse it and still answer every POST, and the client carries on without the + // stream, so its refusal does not fail the session. The SSE transport's GET is the session. + const optionalStream = kind === "http" && request.method === "GET"; + if (provider !== undefined && !optionalStream) { // EventSource replaces rejected fetch errors. Retain our safe failure // within this session so SSE cannot erase its status or reason. const refused = providerErrorDetail(provider, { upstream }); @@ -293,7 +298,7 @@ function withClient( const pending = new Set>(); const { client, transport } = yield* Effect.acquireRelease( Effect.sync(() => { - const fetch = transportFetch(connection, telemetry, rejected, answered); + const fetch = transportFetch(connection, telemetry, rejected, answered, kind); const client = new Client( { name: "executor-apps", version: "0.1.0" }, { From 6d05df3bdd3e4aac36a8f2af2a847d0e3e7ee6eb Mon Sep 17 00:00:00 2001 From: The-AarushiSingh <175547726+The-AarushiSingh@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:47:15 +0530 Subject: [PATCH 007/127] fix(self-host): add Connect SSO button on Security page Password-created owners had no UI path to link an SSO identity. The linking endpoint already existed. This adds a Connect SSO button that starts the same authenticated flow. --- .../components/change-password.tsx | 51 +++++++++++++++++++ .../routes/account.security.tsx | 3 +- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/apps/hosted/self-host/web/src/implementation/components/change-password.tsx b/apps/hosted/self-host/web/src/implementation/components/change-password.tsx index dd2582c6eb..ebf9e8818a 100644 --- a/apps/hosted/self-host/web/src/implementation/components/change-password.tsx +++ b/apps/hosted/self-host/web/src/implementation/components/change-password.tsx @@ -125,3 +125,54 @@ export function ChangePassword() { ); } + +/** Starts the existing authenticated SSO linking flow. */ +export function ConnectSso() { + const [error, setError] = useState(); + return ( + + + +

    Single sign-on

    +     + + Connect a Google or other OIDC account so you can sign in without your password. + +     + + {error && ( +

    + {error} +

    + )} +     + +
    + +
    +     +     + ); +} diff --git a/apps/hosted/self-host/web/src/implementation/routes/account.security.tsx b/apps/hosted/self-host/web/src/implementation/routes/account.security.tsx index cb576bd582..20ed5c7976 100644 --- a/apps/hosted/self-host/web/src/implementation/routes/account.security.tsx +++ b/apps/hosted/self-host/web/src/implementation/routes/account.security.tsx @@ -2,7 +2,7 @@ import { createFileRoute } from "@tanstack/react-router"; import { parseAccountSearch } from "@executor-js/hosted-web/contracts/navigation"; import { SecurityPending } from "@executor-js/hosted-web/account"; import { SecurityPage } from "@executor-js/hosted-web/pages/security"; -import { ChangePassword, ChangePasswordPending } from "../components/change-password.tsx"; +import { ChangePassword, ChangePasswordPending, ConnectSso } from "../components/change-password.tsx"; /** Self-host signs in with a password (or SSO); the password is changed here. */ export const Route = createFileRoute("/account/security")({ @@ -15,6 +15,7 @@ export const Route = createFileRoute("/account/security")({ component: () => ( + ), }); From 6e21918fb449077c7e0ddc8b1f104abf4faafd3b Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 23 Oct 2026 08:38:38 +0000 Subject: [PATCH 008/127] Export 50ee24e --- .changeset/openapi-v1-documents.md | 16 ++ .../src/implementation/v1-import/items.ts | 38 +++- .../src/implementation/v1-import/openapi.ts | 119 +++++----- .../src/implementation/v1-import/source.ts | 63 ++++++ bun.lock | 2 +- e2e/support/openapi-import.ts | 160 +++++++++++++ e2e/test-plan.ts | 11 + e2e/tests/cloud-v1-import.spec.ts | 167 +++++++++++++- e2e/tests/live-openapi-import.spec.ts | 121 ++++++++++ .../skills/app-authoring/integrations.md | 9 +- .../executor-skill-digests.gen.ts | 2 +- packages/apps/package.json | 2 +- .../src/implementation/openapi-compile.ts | 211 +++++++++++++----- .../src/implementation/openapi-document.ts | 205 ++++++++++++----- .../apps/src/implementation/openapi-names.ts | 6 +- .../src/implementation/openapi-request.ts | 31 ++- packages/apps/src/openapi.ts | 32 ++- 17 files changed, 992 insertions(+), 203 deletions(-) create mode 100644 .changeset/openapi-v1-documents.md diff --git a/.changeset/openapi-v1-documents.md b/.changeset/openapi-v1-documents.md new file mode 100644 index 0000000000..f39e2c3168 --- /dev/null +++ b/.changeset/openapi-v1-documents.md @@ -0,0 +1,16 @@ +--- +"apps": patch +--- + +`liveOpenapiRouter` imports the OpenAPI documents Executor v1 imported. A schema +reference may reach any schema in the document: a component whose name is not +URI-encoded, a property inside a component, or a schema under `paths` or another +component. An operation that answers with an event stream or another type, such +as JSON, is a tool that asks for the other type. A path may carry a fixed query, +sent before the operation's own parameters, or a fragment, which is not sent; it +does not change the tool's name. An empty field such as `operationId:` counts as +absent, a parameter in a location OpenAPI 3 does not have is not an input, and a +request body without media types sends no body. + +`openapiOperationNames` lists every operation a document declares with the name +the router gives it, reading the document as the router does. diff --git a/apps/hosted/cloud/src/implementation/v1-import/items.ts b/apps/hosted/cloud/src/implementation/v1-import/items.ts index b4d2abe46a..9ec533a649 100644 --- a/apps/hosted/cloud/src/implementation/v1-import/items.ts +++ b/apps/hosted/cloud/src/implementation/v1-import/items.ts @@ -49,7 +49,7 @@ import { type OwnerId, type OAuthSetupFailed, } from "@executor-js/sdk/core"; -import { planOperationNames } from "apps/openapi"; +import { OpenapiCompileError, openapiOperationNames } from "apps/openapi"; import { Effect, Option, Redacted, Schema } from "effect"; import type { SqlClient } from "effect/sql"; import { V1StaleCategory } from "../../contracts/v1-import-run.ts"; @@ -74,7 +74,6 @@ import { import { openapiHealth, v1IntrospectionQuery } from "./health.ts"; import { credentialOrigins, - documentOperations, importedDocument, operationBaseUrls, parseSpec, @@ -616,7 +615,7 @@ export const compiledTools = ( /** The v1 reads source generation needs. */ export type GenerationSource = Pick< V1Source, - "tools" | "openApiOperations" | "openApiBinding" | "blob" | "blobMissing" + "tools" | "openApiOperations" | "openApiBinding" | "blob" | "blobMissing" | "specification" >; /** @@ -795,8 +794,21 @@ export const generate = ( case "openapi": { const stored = yield* context.source.blob(tenant, "openapi", `spec/${decoded.specHash}`); if (Option.isNone(stored)) return yield* fail(context.source.blobMissing); - const parsed = parseSpec(stored.value); - if (!parsed.ok) return yield* failStale("unreadable_spec", parsed.reason); + const read = parseSpec(stored.value); + if (read.ok === false) return yield* failStale("unreadable_spec", read.reason); + // v1 stored some specifications as their URL and fetched the document once; so does this. + // v1 kept what it fetched, so a document that cannot be fetched or read now still blocks. + const parsed = + read.ok === "url" + ? parseSpec( + yield* context.source + .specification(read.url) + .pipe(Effect.catchTag("SpecificationUnavailable", (error) => fail(error.reason))), + ) + : read; + if (parsed.ok === "url") + return yield* fail("v1's stored specification is a URL whose document is another URL."); + if (!parsed.ok) return yield* fail(parsed.reason); const { document } = parsed; const operations = yield* context.source.openApiOperations(tenant, integration.slug); const tools = yield* context.source.tools(tenant, subjects, integration.slug); @@ -830,10 +842,20 @@ export const generate = ( return yield* fail( `Its operations call ${insecure.join(", ")} over plain HTTP, and an imported app sends an account's credentials to more than one host only over HTTPS.`, ); - // v2's names for every operation, as liveOpenapiRouter plans them, mapped to v1's. + // The name liveOpenapiRouter gives each operation, as it reads the document, mapped to v1's. + const declared = yield* Effect.tryPromise({ + try: () => openapiOperationNames(prepared.document), + catch: (error) => error, + }).pipe( + Effect.catch((error) => + error instanceof OpenapiCompileError ? fail(error.reason) : Effect.die(error), + ), + ); const names: Record = {}; - for (const operation of planOperationNames(documentOperations(prepared))) - if (operation.v1 !== undefined) names[operation.name] = operation.v1; + for (const operation of declared) { + const v1 = prepared.named.get(`${operation.method.toLowerCase()} ${operation.path}`); + if (v1 !== undefined) names[operation.name] = v1; + } const exposed = new Set(Object.values(names)); // Every tool v1 bound or last listed, except those of the TRACE operations it drops and // of the operations v1 never reached. diff --git a/apps/hosted/cloud/src/implementation/v1-import/openapi.ts b/apps/hosted/cloud/src/implementation/v1-import/openapi.ts index 3304c6c1ca..597ab47557 100644 --- a/apps/hosted/cloud/src/implementation/v1-import/openapi.ts +++ b/apps/hosted/cloud/src/implementation/v1-import/openapi.ts @@ -16,11 +16,22 @@ const isObject = (value: unknown): value is Json => const methods = ["get", "put", "post", "delete", "patch", "head", "options", "trace"] as const; -/** A stored specification as an object, or why it is not one, without any of its text. */ +/** + * A stored specification as an object, the URL v1 stored in its place, or why it is neither, + * without any of its text. + */ export type ParsedSpec = | { readonly ok: true; readonly document: Json } + | { readonly ok: "url"; readonly url: string } | { readonly ok: false; readonly reason: string }; +/** A specification v1 stored only as its URL, which v1 fetched when the integration was added. */ +const specificationUrl = (value: unknown) => { + if (typeof value !== "string") return undefined; + const url = value.trim(); + return /^https?:\/\//iu.test(url) && URL.canParse(url) ? url : undefined; +}; + const kindOf = (value: unknown) => value === null ? "null" : Array.isArray(value) ? "a list" : `a ${typeof value}`; @@ -64,9 +75,12 @@ export const parseSpec = (bytes: Uint8Array): ParsedSpec => { // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: the YAML parser throws on input it cannot read try { const yaml = loadYaml(text, { schema: JSON_SCHEMA, json: true }); + const url = specificationUrl(yaml); return isObject(yaml) ? { ok: true, document: yaml } - : unreadable(`it is ${kindOf(yaml)}, not an object`); + : url !== undefined + ? { ok: "url", url } + : unreadable(`it is ${kindOf(yaml)}, not an object`); } catch (error) { if (!(error instanceof YAMLException)) return unreadable( @@ -276,14 +290,22 @@ export const plainHttpHosts = (origins: readonly string[]) => return url.protocol === "https:" || isLoopbackHostname(url.hostname) ? [] : [url.host]; }); -/** Every `#/components/...` reference reachable from a value. */ +/** + * The component each `#/components/
    /` reference reachable from a value names, as + * `[section, name]` in JSON. A reference's tokens are read as the compiler reads them: percent + * decoded, then unescaped, so `#/components/schemas/Paper%20Input` names `Paper Input`. + */ const references = (value: unknown, found: Set) => { if (Array.isArray(value)) for (const item of value) references(item, found); else if (isObject(value)) for (const [key, item] of Object.entries(value)) - if (key === "$ref" && typeof item === "string" && item.startsWith("#/components/")) - found.add(item); - else references(item, found); + if (key === "$ref" && typeof item === "string" && item.startsWith("#/components/")) { + const [section, name] = item + .slice("#/components/".length) + .split("/") + .map((token) => unescape(safeDecode(token))); + if (section !== undefined && name !== undefined) found.add(JSON.stringify([section, name])); + } else references(item, found); }; const unescape = (token: string) => token.replaceAll("~1", "/").replaceAll("~0", "~"); @@ -296,9 +318,11 @@ const pruneComponents = (document: Json) => { references(rest, reached); const queue = [...reached]; for (let next = queue.pop(); next !== undefined; next = queue.pop()) { - const [section, name] = next.slice("#/components/".length).split("/").map(unescape); - const sectionValue = section === undefined ? undefined : components[section]; - const target = isObject(sectionValue) && name !== undefined ? sectionValue[name] : undefined; + const [section, name] = Schema.decodeUnknownSync(Schema.Tuple([Schema.String, Schema.String]))( + JSON.parse(next), + ); + const sectionValue = components[section]; + const target = isObject(sectionValue) ? sectionValue[name] : undefined; const more = new Set(); references(target, more); for (const reference of more) @@ -311,7 +335,7 @@ const pruneComponents = (document: Json) => { for (const [section, entries] of Object.entries(components)) { if (section === "securitySchemes" || !isObject(entries)) continue; const names = Object.keys(entries).filter((name) => - reached.has(`#/components/${section}/${name.replaceAll("~", "~0").replaceAll("/", "~1")}`), + reached.has(JSON.stringify([section, name])), ); if (names.length > 0) kept[section] = Object.fromEntries(names.map((name) => [name, entries[name]])); @@ -319,6 +343,37 @@ const pruneComponents = (document: Json) => { return { ...document, components: kept }; }; +/** + * The object a local reference reaches, following references it reaches in turn, or the value + * itself when it is no reference. An operation can be a reference, which the compiler follows: the + * cut keeps the operation it reaches, so its security is removed like any other's. A reference + * that does not resolve, or a cycle, reaches nothing. + */ +const localTarget = (document: Json, value: unknown): unknown => { + const seen = new Set(); + let current = value; + while (isObject(current) && typeof current.$ref === "string") { + const ref = current.$ref; + if (seen.has(ref) || !ref.startsWith("#/")) return undefined; + seen.add(ref); + let node: unknown = document; + for (const token of ref.slice(2).split("/").map(unescape)) { + const key = safeDecode(token); + node = isObject(node) && Object.hasOwn(node, key) ? node[key] : undefined; + } + current = node; + } + return current; +}; +const safeDecode = (token: string) => { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: decodeURIComponent throws on a malformed escape, which is then read literally + try { + return decodeURIComponent(token); + } catch { + return token; + } +}; + /** * The document cut to v1's operations, and v1's tool name for each kept operation by method and * document path. An operation matches by method and path template (v1's resolved template when @@ -342,7 +397,7 @@ export const importedDocument = ( for (const item of Object.values(source)) if (isObject(item)) for (const method of methods) { - const operation = item[method]; + const operation = localTarget(document, item[method]); if (isObject(operation) && typeof operation["x-executor-toolPath"] === "string") claimed.add(operation["x-executor-toolPath"]); } @@ -352,7 +407,7 @@ export const importedDocument = ( for (const [key, value] of Object.entries(item)) if (!(methods as readonly string[]).includes(key)) shared[key] = value; for (const method of methods) { - const operation = item[method]; + const operation = localTarget(document, item[method]); if (!isObject(operation)) continue; const template = typeof operation["x-executor-pathTemplate"] === "string" @@ -390,43 +445,3 @@ export const importedDocument = ( const { security: _security, webhooks: _webhooks, ...rest } = document; return { document: pruneComponents({ ...rest, paths }), named, traced: [...traced].sort() }; }; - -const OperationNaming = Schema.Struct({ - operationId: Schema.optional(Schema.String), - tags: Schema.optional(Schema.Array(Schema.Unknown)), -}); - -/** - * Each operation of the imported document as v2's name planner reads it (operationId, first - * non-empty tag, method and path, in document order), with v1's name for it. - */ -export const documentOperations = (prepared: { - readonly document: Json; - readonly named: ReadonlyMap; -}) => { - const paths = isObject(prepared.document.paths) ? prepared.document.paths : {}; - return Object.entries(paths).flatMap(([path, item]) => - isObject(item) - ? methods.flatMap((method) => { - const operation = item[method]; - if (!isObject(operation)) return []; - const naming = Option.getOrUndefined( - Schema.decodeUnknownOption(OperationNaming)(operation), - ); - const tag = naming?.tags?.find( - (candidate): candidate is string => - typeof candidate === "string" && candidate.trim() !== "", - ); - return [ - { - operationId: naming?.operationId, - tag, - method: method.toUpperCase(), - path, - v1: prepared.named.get(`${method} ${path}`), - }, - ]; - }) - : [], - ); -}; diff --git a/apps/hosted/cloud/src/implementation/v1-import/source.ts b/apps/hosted/cloud/src/implementation/v1-import/source.ts index 746dc8d244..fd5ec95c2a 100644 --- a/apps/hosted/cloud/src/implementation/v1-import/source.ts +++ b/apps/hosted/cloud/src/implementation/v1-import/source.ts @@ -14,6 +14,7 @@ import { Schedule, Schema, Scope, + Stream, } from "effect"; import { HttpClient, HttpClientRequest } from "effect/http"; import { SqlClient } from "effect/sql"; @@ -426,6 +427,67 @@ const readOnly = (request: HttpClientRequest.HttpClientRequest) => ? Effect.succeed(request) : Effect.die(new Error(`The v1 import reads v1 only; it refused a ${request.method} request.`)); +/** The largest specification the import fetches, as large as an OpenAPI app's router downloads. */ +const maxSpecificationBytes = 40_000_000; + +/** + * Why the document at a specification URL could not be fetched. It never names the URL, which can + * carry a key in its user, host, port, path or query. + */ +export class SpecificationUnavailable extends Schema.TaggedError()( + "SpecificationUnavailable", + { reason: Schema.String }, +) {} + +/** + * The document at the URL v1 stored as an integration's specification, fetched once, as v1 fetched + * it when the integration was added: a GET without the integration's credentials, within 60 + * seconds, of at most `maxSpecificationBytes`. Redirects are followed, as v1 followed them. + */ +export const fetchSpecification = (http: HttpClient.HttpClient, url: string) => + Effect.gen(function* () { + const unavailable = (why: string) => + new SpecificationUnavailable({ + reason: `v1 stored its specification as a URL, and ${why}.`, + }); + const response = yield* http + .execute( + HttpClientRequest.get(url).pipe( + HttpClientRequest.setHeader( + "accept", + "application/json, application/yaml, text/yaml, */*", + ), + ), + ) + .pipe(Effect.mapError(() => unavailable("fetching it failed"))); + if (response.status < 200 || response.status >= 300) + return yield* unavailable(`fetching it answered HTTP ${response.status}`); + const chunks = yield* response.stream.pipe( + Stream.mapError(() => unavailable("reading its document failed")), + Stream.limitBytes(maxSpecificationBytes, () => + Stream.fail(unavailable(`its document is larger than ${maxSpecificationBytes} bytes`)), + ), + Stream.runCollect, + ); + const bytes = new Uint8Array(chunks.reduce((size, chunk) => size + chunk.length, 0)); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + return bytes; + }).pipe( + Effect.timeoutOrElse({ + duration: "60 seconds", + orElse: () => + Effect.fail( + new SpecificationUnavailable({ + reason: "v1 stored its specification as a URL, and fetching it took over 60 seconds.", + }), + ), + }), + ); + /** * v1's state for one step: WorkOS, Vault, Postgres, blobs and billing. The scope is the step's: it * holds the one read-only Postgres client every read of the step shares, opened on the first, and @@ -654,6 +716,7 @@ export const makeV1Source = (settings: V1SourceSettings) => blob: (tenant: string, plugin: string, key: string) => blobs.get(blobKey(tenant, plugin, key)), blobMissing: blobs.missing, + specification: (url: string) => fetchSpecification(http, url), /** * A secret value from v1's Vault. Undefined when the object is missing (WorkOS answers 404, * or 400 for a name it rejects). An outage or rate limit (5xx, 429 or no answer) is retried diff --git a/bun.lock b/bun.lock index 127fd0b4e0..811a337f40 100644 --- a/bun.lock +++ b/bun.lock @@ -486,7 +486,7 @@ }, "packages/apps": { "name": "apps", - "version": "0.0.1-beta.100", + "version": "0.0.1-beta.101", "dependencies": { "@cfworker/json-schema": "4.1.1", "@csstools/css-tokenizer": "4.0.2", diff --git a/e2e/support/openapi-import.ts b/e2e/support/openapi-import.ts index 1d7ed5bf0b..7b2f4d1fca 100644 --- a/e2e/support/openapi-import.ts +++ b/e2e/support/openapi-import.ts @@ -289,6 +289,166 @@ export const credentialEchoUpstream = Effect.gen(function* () { return { origin: `http://127.0.0.1:${server.address.port}` }; }); +/** An API that answers every request with its method, path with query, Accept header and body. */ +export const requestEchoUpstream = Effect.gen(function* () { + const services = yield* Layer.build( + HttpRouter.serve( + HttpRouter.add( + "*", + "/*", + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + return yield* HttpServerResponse.json({ + method: request.method, + url: request.url, + accept: request.headers.accept ?? null, + body: yield* request.text, + }); + }), + ), + { disableLogger: true, disableListenLog: true }, + ).pipe(Layer.provideMerge(NodeHttpServer.layer(createServer, { host: "127.0.0.1", port: 0 }))), + ); + const server = yield* HttpServer.HttpServer.pipe(Effect.provideContext(services)); + if (!("port" in server.address)) return yield* Effect.die("Fixture must listen on TCP"); + return { origin: `http://127.0.0.1:${server.address.port}` }; +}); + +/** + * Shapes Executor v1 imported that the OpenAPI specification forbids or leaves unclear: schema + * references to a component whose name is not URI-encoded, into another component's properties, + * into an operation under `paths` and into a Parameter Object; a request body whose schema is + * another operation's; an operation that can answer with an event stream or JSON; paths that + * carry a fixed query or a fragment; an operationId left empty; a parameter in `body`, which + * OpenAPI 3 does not have; and a request body without media types. + */ +export const v1Document = (origin: string) => { + const json = (schema: unknown) => ({ "application/json": { schema } }); + const ok = { "200": { description: "OK", content: json({ type: "object" }) } }; + const page = "Page`1[[Item, Lib, Version=1.0.0]]"; + return { + openapi: "3.0.3", + info: { title: "v1 shapes", version: "1" }, + servers: [{ url: origin }], + components: { + schemas: { + [page]: { + type: "object", + properties: { items: { type: "array", items: { $ref: "#/components/schemas/Item" } } }, + }, + Item: { + type: "object", + properties: { + id: { type: "string" }, + owner: { + type: "object", + properties: { name: { type: "string" } }, + required: ["name"], + additionalProperties: false, + }, + }, + required: ["id"], + }, + }, + parameters: { + Limit: { name: "limit", in: "query", schema: { type: "integer", minimum: 1 } }, + Legacy: { name: "payload", in: "body", schema: { type: "object" } }, + }, + responses: { + Created: { + description: "Created", + content: json({ type: "object", properties: { created: { type: "boolean" } } }), + }, + }, + }, + paths: { + "/items": { + post: { + operationId: "createItem", + requestBody: { + required: true, + content: json({ + type: "object", + properties: { owner: { $ref: "#/components/schemas/Item/properties/owner" } }, + required: ["owner"], + additionalProperties: false, + }), + }, + responses: { + "200": { description: "OK", content: json({ $ref: `#/components/schemas/${page}` }) }, + }, + }, + }, + "/items/copy": { + post: { + operationId: "copyItem", + parameters: [ + { name: "max", in: "query", schema: { $ref: "#/components/parameters/Limit/schema" } }, + ], + requestBody: { + required: true, + content: json({ + $ref: "#/paths/~1items/post/requestBody/content/application~1json/schema", + }), + }, + responses: { "200": { $ref: "#/components/responses/Created" } }, + }, + }, + "/items/stream": { + get: { + operationId: "streamItems", + responses: { + "200": { + description: "Events, or the items at once", + content: { + "text/event-stream": { schema: { type: "string" } }, + ...json({ type: "object" }), + }, + }, + }, + }, + }, + // Only an event stream, which a tool still cannot return. + "/items/events": { + get: { + operationId: "watchItems", + responses: { + "200": { description: "Events", content: { "text/event-stream": {} } }, + }, + }, + }, + "/search?type=items": { + get: { + operationId: "searchItems", + parameters: [{ name: "q", in: "query", schema: { type: "string" } }], + responses: ok, + }, + }, + "/items/{id}?view=full": { + get: { + parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], + responses: ok, + }, + }, + // Two paths told apart only by their fragments, as some documents name one path's hosts. + "/docs#latest": { get: { operationId: "readDocs", responses: ok } }, + "/docs#archive": { get: { operationId: "readDocs", responses: ok } }, + "/legacy": { + post: { + operationId: null, + tags: ["legacy"], + parameters: [ + { $ref: "#/components/parameters/Legacy" }, + { name: "q", in: "query", schema: { type: "string" } }, + ], + requestBody: { content: {} }, + responses: ok, + }, + }, + }, + }; +}; + /** Deploy an app with the given `index.ts`; it is deleted when the scenario ends. */ export const deployPublicApp = (index: string) => Effect.gen(function* () { diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index ac840a0577..c1a928d0c5 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -891,6 +891,17 @@ export const scenarios = plan({ local: na("Hosted version API scenario"), }, }, + liveOpenapiV1Documents: { + fixtures: "actors", + file: "live-openapi-import.spec.ts", + title: + "Live OpenAPI imports the documents Executor v1 imported: loose schema references, event streams beside JSON, paths with a query or fragment, and fields v1 ignored", + targets: { + "self-host": scheduled, + cloud: na("Loopback upstream fixture"), + local: na("Hosted version API scenario"), + }, + }, liveOpenapiNaming: { fixtures: "actors", file: "live-openapi-import.spec.ts", diff --git a/e2e/tests/cloud-v1-import.spec.ts b/e2e/tests/cloud-v1-import.spec.ts index 42452f896f..ee16aefabd 100644 --- a/e2e/tests/cloud-v1-import.spec.ts +++ b/e2e/tests/cloud-v1-import.spec.ts @@ -152,11 +152,22 @@ const compiledV1Names = [ * A synthetic API with one safe read, \`GET /v1/me\`, recording each request's path, query and key. * The disabled key gets Google's answer for an API that is not enabled in the client's project. * The redirected key is sent on to \`/landing\` over plain HTTP, which answers every request as the - * user and records what reached it. + * user and records what reached it. It also serves the specification v1 stored only as its URL, + * counting each time it is fetched. */ const checkedApi = Effect.gen(function* () { const received = yield* Ref.make>([]); const landed = yield* Ref.make>([]); + const specFetches = yield* Ref.make(0); + let origin = ""; + const linked = HttpRouter.add( + "GET", + "/openapi.json", + Effect.gen(function* () { + yield* Ref.update(specFetches, (count) => count + 1); + return yield* HttpServerResponse.json(linkedDocument(origin)); + }), + ); const me = HttpRouter.add( "GET", "/v1/me", @@ -199,20 +210,83 @@ const checkedApi = Effect.gen(function* () { }), ); const services = yield* Layer.build( - HttpRouter.serve(Layer.mergeAll(me, landing), { + HttpRouter.serve(Layer.mergeAll(me, landing, linked), { disableLogger: true, disableListenLog: true, }).pipe(Layer.provideMerge(NodeHttpServer.layer(createServer, { host: "127.0.0.1", port: 0 }))), ); const server = yield* HttpServer.HttpServer.pipe(Effect.provideContext(services)); if (!("port" in server.address)) return yield* Effect.die("Fixture must listen on TCP"); + origin = `http://127.0.0.1:${server.address.port}`; return { - origin: `http://127.0.0.1:${server.address.port}`, + origin, received: Ref.get(received), landed: Ref.get(landed), + specFetches: Ref.get(specFetches), }; }); +/** The document behind the specification v1 stored only as its URL. */ +const linkedDocument = (origin: string) => ({ + openapi: "3.0.3", + info: { title: "Linked API", version: "1" }, + servers: [{ url: `${origin}/v1` }], + paths: { + "/me": { + get: { + operationId: "readMe", + tags: ["profile"], + responses: { "200": { description: "ok" } }, + }, + }, + }, +}); + +/** + * A document whose operations are references, as some generators write them: each names an + * Operation Object under `components`, with its own operationId, tags and a security requirement + * whose scheme the document declares. v1 named such an operation from its method and path. + * One answers with a component whose name holds a space. + */ +const referencedText = JSON.stringify({ + openapi: "3.0.3", + info: { title: "Referenced API", version: "1" }, + servers: [{ url: "https://api.example.com" }], + security: [], + components: { + securitySchemes: { key: { type: "apiKey", in: "header", name: "X-Key" } }, + pathItems: { + listWidgets: { + operationId: "listWidgets", + tags: ["widgets"], + security: [{ key: [] }], + responses: { "200": { description: "ok" } }, + }, + getWidget: { + operationId: "getWidget", + tags: ["widgets"], + parameters: [{ name: "id", in: "path", required: true, schema: { type: "string" } }], + responses: { + "200": { + description: "ok", + // A component whose name has a space, referenced with it percent-encoded. + content: { + "application/json": { schema: { $ref: "#/components/schemas/Widget%20Page" } }, + }, + }, + }, + }, + }, + schemas: { "Widget Page": { type: "object", properties: { id: { type: "string" } } } }, + }, + paths: { + "/v1/widgets": { get: { $ref: "#/components/pathItems/listWidgets" } }, + "/v1/widgets/{id}": { get: { $ref: "#/components/pathItems/getWidget" } }, + }, +}); +/** v1's names for the Referenced API's operations, by method and path. */ +const referencedV1Names = { "widgets.list": "/v1/widgets", "widgets.read": "/v1/widgets/{id}" }; + /** * A workspace whose OpenAPI integration declares v1's account check (an identity read with a pinned * argument) and an empty base URL, a public MCP server that sends a credential in a fixed header, @@ -278,6 +352,12 @@ const seedChecks = (organization: string, user: string, slug: string, origin: st }); const mergedHash = createHash("sha256").update(mergedText).digest("hex"); const merged = `merged_${randomUUID().slice(0, 6)}`; + const referencedHash = createHash("sha256").update(referencedText).digest("hex"); + const referenced = `referenced_${randomUUID().slice(0, 6)}`; + // v1 kept a specification given as a URL as that text, and fetched the document from it. + const linkedText = `${origin}/openapi.json`; + const linkedHash = createHash("sha256").update(linkedText).digest("hex"); + const linked = `linked_${randomUUID().slice(0, 6)}`; const document = { openapi: "3.0.3", info: { title: "Checked API", version: "1" }, @@ -309,6 +389,8 @@ const seedChecks = (organization: string, user: string, slug: string, origin: st [compiled, "compiled"], [unmatched, "unmatched"], [merged, "merged"], + [referenced, "referenced"], + [linked, "linked"], ] as const) yield* emulators.v1VaultObject( vaultName(userScope, tokenItem(integration, connection)), @@ -433,11 +515,17 @@ const seedChecks = (organization: string, user: string, slug: string, origin: st organization, `${api}.*.*.me.unlisted`, ), - row( - "insert into blob (namespace, key, value) values ($1, $2, $3)", - `o:${organization}/openapi`, - `spec/${compiledHash}`, - compiledText, + ...[ + [compiledHash, compiledText], + [referencedHash, referencedText], + [linkedHash, linkedText], + ].map(([key, text]) => + row( + "insert into blob (namespace, key, value) values ($1, $2, $3)", + `o:${organization}/openapi`, + `spec/${key}`, + text ?? "", + ), ), row( "insert into blob (namespace, key, value) values ($1, $2, $3)", @@ -460,13 +548,17 @@ const seedChecks = (organization: string, user: string, slug: string, origin: st }, }), ), - // No operation rows: v1 compiled the document and found each listed tool by its name. + // No operation rows for the first two: v1 compiled the document and found each listed tool + // by its name. The Merged API's row is above; the others bind each tool to its operation by + // method and path. ...[ { integration: compiled, name: "Compiled API", connection: "compiled", + specHash: compiledHash, tools: compiledV1Names, + operations: {}, }, // The organization revised it before v1 last listed its tools, so the tool v2 would not // provide is one v1 still lists, which blocks. @@ -474,24 +566,55 @@ const seedChecks = (organization: string, user: string, slug: string, origin: st integration: unmatched, name: "Unmatched API", connection: "unmatched", + specHash: compiledHash, tools: ["items.itemsList", "items.archive"], revised: { at: 1_000, listed: 2_000 }, + operations: {}, }, { integration: merged, name: "Merged API", connection: "merged", - tools: ["calendar.channels.stop", "drive.channels.stop"], specHash: mergedHash, + tools: ["calendar.channels.stop", "drive.channels.stop"], + operations: {}, + }, + { + integration: referenced, + name: "Referenced API", + connection: "referenced", + specHash: referencedHash, + tools: Object.keys(referencedV1Names), + operations: referencedV1Names, }, - ].flatMap(({ integration, name, connection, tools, specHash, revised }) => [ + { + integration: linked, + name: "Linked API", + connection: "linked", + specHash: linkedHash, + tools: ["profile.read"], + operations: { "profile.read": "/me" }, + }, + ].flatMap(({ integration, name, connection, specHash, tools, operations, revised }) => [ + ...Object.entries(operations).map(([tool, path]) => + row( + "insert into plugin_storage (tenant, plugin_id, collection, key, data) values ($1, 'openapi', 'operation', $2, $3::json)", + organization, + `op.${integration}.${tool}`, + JSON.stringify({ + integration, + toolName: tool, + binding: { method: "get", pathTemplate: path, parameters: [] }, + }), + ), + ), row( "insert into integration (tenant, slug, plugin_id, name, config, config_revised_at) values ($1, $2, 'openapi', $3, $4::json, nullif($5, '')::bigint)", organization, integration, name, JSON.stringify({ - specHash: specHash ?? compiledHash, + specHash, authenticationTemplate: [ { slug: "apikey-0", @@ -3094,6 +3217,10 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import", (it) => { expect(results.get("Merged API")?.status, results.get("Merged API")?.reason).toBe( "imported", ); + for (const name of ["Referenced API", "Linked API"]) + expect(results.get(name)?.status, results.get(name)?.reason).toBe("imported"); + // v1 stored the Linked API's specification as its URL; the import fetched it once. + expect(yield* upstream.specFetches).toBe(1); // A tool v1 listed that the document does not name fails the integration by name. expect(results.get("Unmatched API")).toMatchObject({ status: "failed", @@ -3143,6 +3270,22 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import", (it) => { expect((yield* decode(Tools, compiledTools)).items.map((tool) => tool.name).sort()).toEqual( compiledV1Names, ); + // Operations that are references, and the operations of a specification v1 stored as its + // URL, keep v1's names. + for (const [name, expected] of [ + ["Referenced API", Object.keys(referencedV1Names)], + ["Linked API", ["profile.read"]], + ] as const) { + const listed = yield* call( + `List the ${name}'s tools`, + "GET", + `/api/organizations/${target.id}/apps/${appId(name)}/tools`, + ); + expect(listed.status, JSON.stringify(listed.body)).toBe(200); + expect((yield* decode(Tools, listed)).items.map((tool) => tool.name).sort()).toEqual( + expected, + ); + } const source = yield* call( "Read the MCP app's deployed source", "GET", diff --git a/e2e/tests/live-openapi-import.spec.ts b/e2e/tests/live-openapi-import.spec.ts index 914b792b9b..bb7009f1f5 100644 --- a/e2e/tests/live-openapi-import.spec.ts +++ b/e2e/tests/live-openapi-import.spec.ts @@ -13,7 +13,9 @@ import { malformedSwaggerDocument, openapi32Upstream, otherHostDocument, + requestEchoUpstream, unsupportedAuthDocument, + v1Document, } from "../support/openapi-import.ts"; import { scenarios } from "../test-plan.ts"; @@ -637,6 +639,125 @@ export default defineApp({}, async ({ cache, fetch, signal }) => ({ ), ); + it.effect(scenarios.liveOpenapiV1Documents.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const upstream = yield* requestEchoUpstream; + const app = yield* deployPublicApp(`import { defineApp, object, query, router } from "apps"; +import { liveOpenapiRouter, openapiToolNames } from "apps/openapi"; +const options = { + source: { document: ${JSON.stringify(v1Document(upstream.origin))} }, + allowedOrigin: ${JSON.stringify(upstream.origin)}, + securitySchemes: {}, methods: {}, oauth: [], +}; +export default defineApp({}, async ({ cache, fetch, signal }) => ({ + tools: router({ + api: liveOpenapiRouter({ ...options, cache, fetch, signal }), + names: query({ description: "Tool names", input: object({}) }, () => + openapiToolNames({ ...options, signal }), + ), + }), +})); +`); + // Every operation v1 imported is a tool. Only the one that answers with nothing but an + // event stream is not: a tool cannot return a stream. + const listing = yield* app.call("names", "query", {}); + expect(listing.status, JSON.stringify(listing.body)).toBe(200); + const { tools, skipped } = yield* body(OpenapiToolNames, listing); + expect(skipped).toEqual([]); + expect(tools.map((tool) => [tool.name, tool.method, tool.path])).toEqual([ + ["items.createItem", "POST", "/items"], + ["items.copyItem", "POST", "/items/copy"], + ["items.streamItems", "GET", "/items/stream"], + ["search.searchItems", "GET", "/search?type=items"], + ["items.getItems", "GET", "/items/{id}?view=full"], + // Their names differ only in the stable hash of the whole path. + ["docs.readDocsGet00jgt2p1", "GET", "/docs#latest"], + ["docs.readDocsGet00ru2ttw", "GET", "/docs#archive"], + ["legacy.postLegacy", "POST", "/legacy"], + ]); + + const Echo = Schema.Struct({ + method: Schema.String, + url: Schema.String, + accept: Schema.NullOr(Schema.String), + body: Schema.String, + }); + const call = (tool: string, kind: "query" | "mutation", input: unknown) => + app.call(`api.${tool}`, kind, input).pipe( + Effect.tap((response) => + Effect.sync(() => expect(response.status, JSON.stringify(response.body)).toBe(200)), + ), + Effect.flatMap((response) => body(Echo, response)), + ); + const rejected = (tool: string, kind: "query" | "mutation", input: unknown) => + app + .call(`api.${tool}`, kind, input) + .pipe( + Effect.tap((response) => + Effect.sync(() => + expect(response.status, `${tool}: ${JSON.stringify(response.body)}`).not.toBe( + 200, + ), + ), + ), + ); + + // A reference into another component's properties validates the input as its target. + expect( + yield* call("items.createItem", "mutation", { body: { owner: { name: "Ada" } } }), + ).toMatchObject({ method: "POST", url: "/items", body: '{"owner":{"name":"Ada"}}' }); + yield* rejected("items.createItem", "mutation", { body: { owner: {} } }); + // So do a request body that reuses another operation's schema and a parameter schema + // that points into a Parameter Object. + expect( + yield* call("items.copyItem", "mutation", { + query: { max: 2 }, + body: { owner: { name: "Ada" } }, + }), + ).toMatchObject({ method: "POST", url: "/items/copy?max=2" }); + yield* rejected("items.copyItem", "mutation", { + query: { max: 0 }, + body: { owner: { name: "Ada" } }, + }); + yield* rejected("items.copyItem", "mutation", { body: { owner: { title: "Ada" } } }); + // The response refers to a component whose name is not URI-encoded; its schema is read. + const described = yield* app.tool("api.items.createItem"); + expect(described.status, JSON.stringify(described.body)).toBe(200); + expect(JSON.stringify(described.body)).toContain('"required":["id"]'); + + // The operation that can also answer with JSON asks for JSON only. + expect(yield* call("items.streamItems", "query", {})).toEqual({ + method: "GET", + url: "/items/stream", + accept: "application/json", + body: "", + }); + // A path's fixed query is sent with the operation's own; its fragment is not sent. + expect(yield* call("search.searchItems", "query", { query: { q: "lamp" } })).toMatchObject({ + method: "GET", + url: "/search?type=items&q=lamp", + }); + expect(yield* call("items.getItems", "query", { path: { id: "a1" } })).toMatchObject({ + method: "GET", + url: "/items/a1?view=full", + }); + for (const docs of ["docs.readDocsGet00jgt2p1", "docs.readDocsGet00ru2ttw"]) + expect(yield* call(docs, "query", {})).toMatchObject({ method: "GET", url: "/docs" }); + // An empty operationId is no operationId. The parameter in `body` and the request body + // without media types are not inputs, as v1 dropped them. + expect(yield* call("legacy.postLegacy", "mutation", { query: { q: "old" } })).toEqual({ + method: "POST", + url: "/legacy?q=old", + accept: "application/json", + body: "", + }); + yield* rejected("legacy.postLegacy", "mutation", { body: {} }); + }), + ), + ); + it.effect(scenarios.liveOpenapiNaming.title, (context) => withHostedCase( context, diff --git a/packages/app-templates/executor/skills/app-authoring/integrations.md b/packages/app-templates/executor/skills/app-authoring/integrations.md index 111535afc8..9f47005ffe 100644 --- a/packages/app-templates/executor/skills/app-authoring/integrations.md +++ b/packages/app-templates/executor/skills/app-authoring/integrations.md @@ -298,7 +298,10 @@ operationId `listProjects` tagged `projects` becomes becomes `accounts.connect`. Both parts are camelCased from their words: tag `Team Members` with operationId `list_team_members` becomes `teamMembers.listTeamMembers`, and `GetUserByID` becomes `getUserById`. -Without an operationId the name comes from the method and path. Names that +Without an operationId the name comes from the method and path. A fixed query +or fragment in a declared path, as in `/search?type=items`, is not part of the +name; the query is sent before the operation's parameters and the fragment is +not sent. Names that collide, whether or not they come from operationIds, are told apart in this order: first the path's version segment, as in `users.v2.listUsers`; then the path segments that differ, so `GET /builds` and `GET /builds/{build_num}` become @@ -321,7 +324,9 @@ account: call `liveOpenapiRouter` without `accountRouter` and with normalized metadata. Use `contentType` to choose an alternate declared request media type. Binary request bodies and multipart binary fields take base64 strings. Binary responses return `{ base64, contentType }`; text and JSON -sequences (NDJSON, JSON Lines, `json-seq`) return text. Success responses have +sequences (NDJSON, JSON Lines, `json-seq`) return text. An operation that can +answer with an event stream or another type asks for the other type; one that +answers only with an event stream is not a tool. Success responses have a 16 MiB / 30-second read bound. Live SSE requires an authored subscription. diff --git a/packages/app-templates/src/implementation/executor-skill-digests.gen.ts b/packages/app-templates/src/implementation/executor-skill-digests.gen.ts index 9a37471fcd..611f774c7e 100644 --- a/packages/app-templates/src/implementation/executor-skill-digests.gen.ts +++ b/packages/app-templates/src/implementation/executor-skill-digests.gen.ts @@ -6,7 +6,7 @@ export const executorSkillDigests: Readonly> = { "app-authoring/accounts.md": "142fd6cc82e6537a6e47b0c58e7930c8218952e57bf9ed24836081270c225778", "app-authoring/deploy.md": "dab905832c653eb6a22d6bfd64e4ec3b8cb994288972f61f29fe23c945d75822", "app-authoring/events.md": "8f7465e35c6c94b06978b89680181df60c741225de3ba8158f22f76ee7e91a5a", - "app-authoring/integrations.md": "9484086b1cb68aa9a9a1a0eb4ca20654c76702eda82675b35f46861bedc30a3a", + "app-authoring/integrations.md": "cb0ac98ad26d840e7ec18e79756a9592acfc65045aa67caf28b18c102ca01690", "app-authoring/starter.md": "382bd769d7b10e9c09eaa2367755f23153a31a484aaca9c449ab4aac7d61e224", "app-authoring/storage.md": "d2c2288c8a6dafa0a88f09e7e91409d9afc08158205ea6d25064d0dee67a33df", "app-authoring/tools.md": "460eed5aaf4f84071945f891b955765a39d5372773cb6fff4f4e5d0d99e68a14", diff --git a/packages/apps/package.json b/packages/apps/package.json index c55a93d36d..3a3845deae 100644 --- a/packages/apps/package.json +++ b/packages/apps/package.json @@ -1,6 +1,6 @@ { "name": "apps", - "version": "0.0.1-beta.100", + "version": "0.0.1-beta.101", "private": true, "description": "TypeScript apps with typed tools, accounts, storage, and UI.", "type": "module", diff --git a/packages/apps/src/implementation/openapi-compile.ts b/packages/apps/src/implementation/openapi-compile.ts index 59bbdbff2f..8541f22e13 100644 --- a/packages/apps/src/implementation/openapi-compile.ts +++ b/packages/apps/src/implementation/openapi-compile.ts @@ -43,11 +43,20 @@ function fail(code: TemplateError["code"], reason: string): never { throw new TemplateError({ code, reason }); } const record = (value: unknown): JsonObject => documentObject(value); +/** + * An Operation Object without its null fields. YAML writes a field left empty, such as + * `operationId:`, as null; it means the field is absent, as Executor v1 read it. + */ +const withoutNulls = (operation: JsonObject): JsonObject => + Object.values(operation).includes(null) + ? Object.fromEntries(Object.entries(operation).filter(([, value]) => value !== null)) + : operation; /** Naming reads only these fields of each declared Operation Object. */ const OperationNaming = Schema.Struct({ operationId: Schema.optionalKey(Schema.String), - tags: Schema.optionalKey(Schema.Array(Schema.String)), + tags: Schema.optionalKey(Schema.Array(Schema.Unknown)), }); +const isParameterLocation = Schema.is(Parameter.fields.in); const identifier = (name: string) => name.replace(/[^a-zA-Z0-9_]/g, "_"); /** Parse an absolute URL from the definition. Failures never echo it: a URL can hold credentials. */ function parseUrl(value: string, base?: string): URL { @@ -190,6 +199,45 @@ function errorContent(document: OpenApiDocument, response: JsonObject): Json | u } } +const isEventStream = (type: string) => + type.split(";")[0]?.trim().toLowerCase() === "text/event-stream"; +/** Whether a response, resolved or not, offers an event stream. */ +const offersEventStream = (document: OpenApiDocument, response: unknown) => { + const content = errorContent(document, record(response)); + return content !== undefined && Object.keys(record(content)).some(isEventStream); +}; +/** + * An operation that can answer with an event stream or with another media type, such as JSON, + * without the event stream in its responses, so it asks for and returns the other type, as + * Executor v1 called it. An operation whose successes offer only an event stream keeps it. + */ +function withoutStreamAlternatives(document: OpenApiDocument, operation: JsonObject): JsonObject { + if (operation.responses === undefined) return operation; + const responses = Object.entries(record(operation.responses)); + if (!responses.some(([, response]) => offersEventStream(document, response))) return operation; + const alternative = responses.some(([status, response]) => { + if (!/^2(?:[0-9]{2}|XX)$/i.test(status)) return false; + const content = errorContent(document, record(response)); + return ( + content !== undefined && Object.keys(record(content)).some((type) => !isEventStream(type)) + ); + }); + if (!alternative) return operation; + return { + ...operation, + responses: Object.fromEntries( + responses.map(([status, response]) => { + if (!offersEventStream(document, response)) return [status, response]; + const resolved = document.resolve(record(response)); + const content = Object.entries(record(resolved.content)).filter( + ([type]) => !isEventStream(type), + ); + return [status, { ...resolved, content: Object.fromEntries(content) }]; + }), + ), + }; +} + /** Keep tagged errors, including response/component refs and anyOf alternatives. * Public text comes from a declared string message or the schema's static description. * `scripts/check-api-errors.ts` applies this to Executor's own API documents. @@ -315,6 +363,79 @@ function prefixParameters(prefix: string): readonly string[] { return names; } +/** + * Every operation a definition declares, in definition order, with the tool name it has. Names are + * planned over every declared operation, so whether the compiler supports one operation never + * renames another, and a left-out operation keeps the name it would have. + */ +function declaredOperations(document: OpenApiDocument) { + const { spec } = document; + // OpenAPI 3.2 adds QUERY and `additionalOperations`, keyed by the exact method to send. + const version32 = spec.openapi.startsWith("3.2."); + const candidates = Object.entries(spec.paths).flatMap(([path, source]) => { + const item = document.resolve(source); + const itemAt = declaredAt(document, source, `#/paths/${pointerToken(path)}`); + const additional = + version32 && item.additionalOperations !== undefined + ? Object.entries(record(item.additionalOperations)).map(([method, value]) => ({ + method, + value, + at: `${itemAt}/additionalOperations/${pointerToken(method)}`, + })) + : []; + const fixed = (version32 ? fixedMethods32 : fixedMethods) + .filter((method) => item[method.toLowerCase()] !== undefined) + .map((method): { method: string; value: Json | undefined; at: string } => ({ + method, + value: item[method.toLowerCase()], + at: `${itemAt}/${method.toLowerCase()}`, + })); + return [...fixed, ...additional].map((operation) => ({ ...operation, path, item, itemAt })); + }); + return planOperationNames( + candidates.map((candidate) => { + const resolved = resolvedOrUndefined(document, candidate.value); + const naming = Schema.decodeUnknownOption(OperationNaming)( + resolved === undefined ? undefined : withoutNulls(resolved), + ).pipe(Option.getOrUndefined); + return { + ...candidate, + operationId: naming?.operationId, + tag: naming?.tags?.find( + (tag): tag is string => typeof tag === "string" && tag.trim() !== "", + ), + }; + }), + ); +} + +/** + * The tool name, method and declared path of every operation a definition declares, in definition + * order, whether or not the compiler imports it: the names `compileOpenApiDocument` gives them. + * The document is consumed, as compilation consumes it. + */ +export const compiledOperationNames = (document: JsonObject) => + // oxlint-disable-next-line executor/authored-code-through-adapter -- this module's own parser + Effect.tryPromise({ + try: async () => + declaredOperations(await openApiDocument(document)).map(({ name, method, path }) => ({ + name, + method, + path, + })), + catch: (error) => + error instanceof TemplateError + ? Exit.fail(error) + : Schema.isSchemaError(error) + ? Exit.fail( + new TemplateError({ + code: "invalid_document", + reason: "The definition's structure does not match OpenAPI.", + }), + ) + : Exit.die(error), + }).pipe(Effect.catch((exit) => exit)); + /** Compile a revision of a live OpenAPI source into tool declarations for the app runtime. * The document is consumed: compilation upgrades it in place instead of copying it. It must be a * tree, with no object reached twice, as parsed JSON is. @@ -461,43 +582,8 @@ export const compileOpenApiDocument = ( if ("error" in known) throw known.error; return known.address; }; - // OpenAPI 3.2 adds QUERY and `additionalOperations`, keyed by the exact method to send. - const version32 = spec.openapi.startsWith("3.2."); - const candidates = Object.entries(spec.paths).flatMap(([path, source]) => { - const item = document.resolve(source); - const itemAt = declaredAt(document, source, `#/paths/${pointerToken(path)}`); - const additional = - version32 && item.additionalOperations !== undefined - ? Object.entries(record(item.additionalOperations)).map(([method, value]) => ({ - method, - value, - at: `${itemAt}/additionalOperations/${pointerToken(method)}`, - })) - : []; - const fixed = (version32 ? fixedMethods32 : fixedMethods) - .filter((method) => item[method.toLowerCase()] !== undefined) - .map((method): { method: string; value: Json | undefined; at: string } => ({ - method, - value: item[method.toLowerCase()], - at: `${itemAt}/${method.toLowerCase()}`, - })); - return [...fixed, ...additional].map((operation) => ({ ...operation, path, item, itemAt })); - }); - if (!candidates.length) fail("no_operations", "This API does not contain any operations."); - // Names are planned over every declared operation, so whether this compiler supports one - // operation never renames another, and a left-out operation keeps the name it would have. - const named = planOperationNames( - candidates.map((candidate) => { - const naming = Schema.decodeUnknownOption(OperationNaming)( - resolvedOrUndefined(document, candidate.value), - ).pipe(Option.getOrUndefined); - return { - ...candidate, - operationId: naming?.operationId, - tag: naming?.tags?.find((tag) => tag.trim() !== ""), - }; - }), - ); + const named = declaredOperations(document); + if (!named.length) fail("no_operations", "This API does not contain any operations."); // Left-out operations are reported in definition order, whatever stage left them out. const order = new Map(named.map(({ name }, index) => [name, index])); const inDefinitionOrder = (operations: readonly OpenapiSkippedOperation[]) => @@ -505,15 +591,17 @@ export const compileOpenApiDocument = ( for (const [index, { name, path, item, itemAt, method, value, at }] of named.entries()) { if (index % 100 === 99) await yieldToRuntime(); try { - if (!path.startsWith("/") || path.includes("?") || path.includes("#")) - fail("operation_path", "The path must start with / and contain no ? or #."); + if (!path.startsWith("/")) fail("operation_path", "The path must start with /."); if (!isImportedMethod(method)) fail( "operation_method", `Executor does not send the HTTP method ${method}. It imports ${fixedMethods32.join(", ")} operations.`, ); const operationMethods: GeneratedSecrets[] = []; - const resolved = document.resolve(record(value)); + const resolved = withoutStreamAlternatives( + document, + withoutNulls(document.resolve(record(value))), + ); const operation = decodeDefinition(Operation, resolved, at, "operation"); // Tags only label the tool; a malformed list is dropped rather than rejecting it. const tags = Option.getOrUndefined( @@ -563,12 +651,11 @@ export const compileOpenApiDocument = ( ]; const parameters = new Map(); for (const { parameter, at: parameterAt } of declared) { - const p = decodeDefinition( - Parameter, - document.resolve(record(parameter)), - parameterAt, - "parameter", - ); + const declaration = document.resolve(record(parameter)); + // A parameter in a place OpenAPI 3 does not have, such as Swagger 2.0's `body`, is not + // sent, as Executor v1 left it out. + if (!isParameterLocation(declaration.in)) continue; + const p = decodeDefinition(Parameter, declaration, parameterAt, "parameter"); if (p.in === "querystring") { const [media, ...others] = Object.keys(p.content ?? {}); const kind = @@ -632,14 +719,18 @@ export const compileOpenApiDocument = ( } const bodyVariants: Array<(api: ApiSchema) => JsonObject> = []; let retainedBody: GeneratedOperation["request"]["requestBody"]; - if (operation.requestBody) { - const bodyAt = declaredAt(document, resolved.requestBody, `${at}/requestBody`); - const request = decodeDefinition( - RequestBody, - document.resolve(operation.requestBody), - bodyAt, - "request body", - ); + const bodyAt = declaredAt(document, resolved.requestBody, `${at}/requestBody`); + const request = + operation.requestBody === undefined + ? undefined + : decodeDefinition( + RequestBody, + document.resolve(operation.requestBody), + bodyAt, + "request body", + ); + // A request body that declares no media types has nothing to send, as Executor v1 read it. + if (request !== undefined && Object.keys(request.content).length > 0) { const rawContent = record(document.resolve(record(resolved.requestBody)).content); const retainedContent: Record = {}; for (const [contentType, content] of Object.entries(request.content)) { @@ -696,8 +787,6 @@ export const compileOpenApiDocument = ( })); } retainedBody = { ...request, content: retainedContent }; - if (!bodyVariants.length) - fail("request_body", "The request body declares no media types."); properties.body = () => ({}); properties.contentType = () => ({ type: "string", @@ -757,10 +846,10 @@ export const compileOpenApiDocument = ( fail("input_schema", "Its input schema is not valid JSON."); } const outputSchema = responseSchema(document, resolved, at, method); - const streaming = Object.values(operation.responses ?? {}).some((response) => { - const content = errorContent(document, response); - return content !== undefined && Object.hasOwn(record(content), "text/event-stream"); - }); + // Only an operation without another success type is left with an event stream. + const streaming = Object.values(operation.responses ?? {}).some((response) => + offersEventStream(document, response), + ); built.push({ methods: operationMethods, declared: { name, method, path }, diff --git a/packages/apps/src/implementation/openapi-document.ts b/packages/apps/src/implementation/openapi-document.ts index 48272d0be9..aade53dcb6 100644 --- a/packages/apps/src/implementation/openapi-document.ts +++ b/packages/apps/src/implementation/openapi-document.ts @@ -98,54 +98,82 @@ const holdsReference = (value: unknown): boolean => (typeof value.$ref === "string" || Object.values(value).some(holdsReference)); /** - * Reject a schema keyword whose value is not a schema but holds a reference, such as an array - * `items` of `$ref`s, which JSON Schema 2020-12 and OpenAPI 3.0 both forbid. Converters pass such a - * value through without reading it, so its reference would stay unresolved and fail validation. - * Such a value without a reference passes through unchanged, as validators accept it. The failure - * names the keyword's JSON Pointer, starting from `at`. + * Check a schema's keywords and write each reference in it as `canonical` names it. A schema keyword + * whose value is not a schema but holds a reference, such as an array `items` of `$ref`s, which + * JSON Schema 2020-12 and OpenAPI 3.0 both forbid, is rejected: converters pass such a value + * through without reading it, so its reference would stay unresolved and fail validation. Such a + * value without a reference passes through unchanged, as validators accept it. The failure names + * the keyword's JSON Pointer, starting from `at`. Only the objects on the way to a rewritten + * reference are copied; a schema without one is returned as it is. */ -const checkSchemaKeywords = (schema: JsonObject, at: string) => { +const prepareSchema = ( + schema: JsonObject, + at: string, + canonical: (ref: string) => string, +): JsonObject => { const path: string[] = []; /** Visit a schema, or reject a value that is not one when it holds a reference. */ - const child = (value: Schema.Json, problem: () => string) => { + const child = (value: Schema.Json, problem: () => string): Schema.Json => { if (isSchemaValue(value)) return visit(value); if (holdsReference(value)) fail( "schema_keyword", `The schema at ${[at, ...path].join("/")} is invalid: ${problem()}, so the reference inside it cannot be resolved.`, ); + return value; }; - const visit = (node: Schema.Json) => { - if (!isObject(node)) return; + const visit = (node: Schema.Json): Schema.Json => { + if (!isObject(node)) return node; + let copy: Record | undefined; + const set = (key: string, value: Schema.Json) => { + if (value !== node[key]) (copy ??= { ...node })[key] = value; + }; for (const [key, value] of Object.entries(node)) { path.push(pointerToken(key)); - if (schemaKeywords.has(key)) - child(value, () => `"${key}" must be a schema, not ${jsonKind(value)}`); - else if (schemaArrayKeywords.has(key) || schemaMapKeywords.has(key)) { - const entries = schemaArrayKeywords.has(key) - ? Array.isArray(value) - ? [...value.entries()] - : undefined - : isObject(value) - ? Object.entries(value) - : undefined; - if (entries === undefined) + if (key === "$ref" && typeof value === "string") set(key, canonical(value)); + else if (schemaKeywords.has(key)) + set( + key, + child(value, () => `"${key}" must be a schema, not ${jsonKind(value)}`), + ); + else if (schemaArrayKeywords.has(key) && Array.isArray(value)) { + let items: Schema.Json[] | undefined; + for (const [index, item] of value.entries()) { + path.push(String(index)); + const next = child( + item, + () => `each "${key}" entry must be a schema, not ${jsonKind(item)}`, + ); + if (next !== item) (items ??= [...value])[index] = next; + path.pop(); + } + if (items !== undefined) set(key, items); + } else if (schemaMapKeywords.has(key) && isObject(value)) { + let entries: Record | undefined; + for (const [name, item] of Object.entries(value)) { + path.push(pointerToken(name)); + const next = child( + item, + () => `each "${key}" entry must be a schema, not ${jsonKind(item)}`, + ); + if (next !== item) (entries ??= { ...value })[name] = next; + path.pop(); + } + if (entries !== undefined) set(key, entries); + } else if (schemaArrayKeywords.has(key) || schemaMapKeywords.has(key)) + set( + key, child(value, () => schemaArrayKeywords.has(key) ? `"${key}" must be an array of schemas, not ${jsonKind(value)}` : `"${key}" must map names to schemas, not ${jsonKind(value)}`, - ); - else - for (const [name, item] of entries) { - path.push(pointerToken(String(name))); - child(item, () => `each "${key}" entry must be a schema, not ${jsonKind(item)}`); - path.pop(); - } - } + ), + ); path.pop(); } + return copy ?? node; }; - visit(schema); + return record(visit(schema)); }; /** Fields that map names to Path Item Objects, whose `$ref` siblings extend the referenced item. */ const pathItemMaps = new Set(["paths", "webhooks", "pathItems"]); @@ -369,6 +397,59 @@ export async function openApiDocument(input: JsonObject) { const definitions = new Map(); // Component names each definition references directly, for transitive reach checks. const references = new Map>(); + // Schemas a reference reaches somewhere other than a whole component, by definition name. + const pointed = new Map(); + const componentSection = (path: readonly string[]) => + path[0] === "components" && path[1] === "schemas" && path[2] !== undefined; + /** + * The definition name of the schema a local reference reaches: a component's own name, or, for + * any other place in the definition, such as a component's property, a Parameter Object's schema + * or an operation's request body schema, the JSON Pointer the reference holds. + * Documents often leave characters such as `[` or spaces unencoded in the fragment, which is then + * read as a plain JSON Pointer. + */ + const definitionName = ( + ref: string, + ): { readonly name: string } | { readonly problem: "outside" | "missing" | "conflict" } => { + const path = localPath(ref); + if (path === undefined || path.length === 0) return { problem: "outside" }; + const component = componentSection(path) ? path[2] : undefined; + if (component !== undefined && !Object.hasOwn(components, component)) + return { problem: "missing" }; + if (component !== undefined && path.length === 3) return { name: component }; + // A JSON Pointer, which a URI fragment would prefix with `#`: the validator does not encode + // that character in definition names. + const name = path.map((token) => `/${JsonPointer.escapeToken(token)}`).join(""); + if (pointed.has(name)) return { name }; + if (Object.hasOwn(components, name)) return { problem: "conflict" }; + let node: Schema.Json | undefined = component === undefined ? restored : components[component]; + for (const token of component === undefined ? path : path.slice(3)) { + node = isObject(node) + ? Object.hasOwn(node, token) + ? node[token] + : undefined + : Array.isArray(node) && /^(?:0|[1-9][0-9]*)$/.test(token) + ? node[Number(token)] + : undefined; + if (node === undefined) return { problem: "missing" }; + } + if (!isSchemaValue(node)) return { problem: "missing" }; + pointed.set(name, node); + return { name }; + }; + const conflictReason = (ref: string) => + `The reference ${JSON.stringify(ref)} reaches a schema whose definition name a component schema already has.`; + /** The API schema a definition name stands for. Boolean schemas read as the objects they mean. */ + const definitionSchema = (name: string): JsonObject | undefined => { + const schema = Object.hasOwn(components, name) ? components[name] : pointed.get(name); + return schema === undefined + ? undefined + : schema === true + ? {} + : schema === false + ? { not: {} } + : record(schema); + }; return { spec: parsed, @@ -380,19 +461,16 @@ export async function openApiDocument(input: JsonObject) { origin(value: unknown): string | undefined { return isObject(value) ? origins.get(value) : undefined; }, - /** Inspect a component schema without expanding its children. OpenAPI object refs must already be resolved by Swagger. */ + /** + * Inspect the schema a reference reaches without expanding its children: a component, or any + * other schema in the definition. OpenAPI object references are already resolved. + */ resolve(value: JsonObject): JsonObject { const visited = new Set(); while (value.$ref !== undefined) { const ref = value.$ref; - const path = typeof ref === "string" ? JsonPointer.parseUriFragment(ref) : undefined; - if ( - typeof ref !== "string" || - path?.length !== 3 || - path[0] !== "components" || - path[1] !== "schemas" || - path[2] === undefined - ) + const found = typeof ref === "string" ? definitionName(ref) : { problem: "outside" }; + if (typeof ref !== "string" || ("problem" in found && found.problem === "outside")) fail( "external_reference", `The reference ${JSON.stringify(ref)} does not resolve within the definition. Only local references to objects the definition declares are supported.`, @@ -400,12 +478,15 @@ export async function openApiDocument(input: JsonObject) { if (visited.has(ref)) fail("circular_reference", `The schema ${ref} is a circular alias of itself.`); visited.add(ref); - if (!Object.hasOwn(components, path[2])) + if ("problem" in found && found.problem === "conflict") + fail("schema_reference", conflictReason(ref)); + const schema = "name" in found ? definitionSchema(found.name) : undefined; + if (schema === undefined) fail( "missing_component", `The reference ${ref} names a schema the definition does not declare.`, ); - value = record(components[path[2]]); + value = schema; } return value; }, @@ -420,28 +501,32 @@ export async function openApiDocument(input: JsonObject) { const direct = new Set(); const local: Record = {}; const visit = (input: JsonObject, direct: Set, at: string): JsonObject => { - checkSchemaKeywords(input, at); + // Every reference is written as the component reference the converter rewrites to + // `#/$defs/`, whatever schema in the definition it reaches. + const prepared = prepareSchema(input, at, (ref) => { + const found = definitionName(ref); + if ("name" in found) + return JsonPointer.formatUriFragment(["components", "schemas", found.name]); + return fail( + found.problem === "missing" ? "missing_component" : "schema_reference", + found.problem === "missing" + ? `The schema at ${at} references ${ref}, which the definition does not declare.` + : found.problem === "conflict" + ? conflictReason(ref) + : `The schema at ${at} references ${JSON.stringify(ref)}, which is outside the definition. Only references within it are supported.`, + ); + }); let document: ReturnType; try { - document = convert(input, { + document = convert(prepared, { onReference(ref) { const path = JsonPointer.parseUriFragment(ref); - const name = path?.[2]; - if ( - path?.length !== 3 || - path[0] !== "components" || - path[1] !== "schemas" || - name === undefined - ) - fail( + const name = path?.length === 3 && componentSection(path) ? path[2] : undefined; + const component = name === undefined ? undefined : definitionSchema(name); + if (name === undefined || component === undefined) + return fail( "schema_reference", - `The schema at ${at} references ${JSON.stringify(ref)}. Only references to #/components/schemas/ are supported.`, - ); - const component = Object.hasOwn(components, name) ? components[name] : undefined; - if (component === undefined) - fail( - "missing_component", - `The schema at ${at} references ${ref}, which the definition does not declare.`, + `The schema at ${at} references ${JSON.stringify(ref)}, which does not resolve.`, ); direct.add(name); if (!definitions.has(name)) { @@ -451,7 +536,11 @@ export async function openApiDocument(input: JsonObject) { const names = new Set(); definitions.set( name, - visit(component, names, `#/components/schemas/${pointerToken(name)}`), + visit( + component, + names, + pointed.has(name) ? `#${name}` : `#/components/schemas/${pointerToken(name)}`, + ), ); references.set(name, names); } diff --git a/packages/apps/src/implementation/openapi-names.ts b/packages/apps/src/implementation/openapi-names.ts index a047d068f1..d9358f9487 100644 --- a/packages/apps/src/implementation/openapi-names.ts +++ b/packages/apps/src/implementation/openapi-names.ts @@ -33,8 +33,12 @@ const pascal = (value: string) => { }; const version = /^v\d+(?:[._-]\d+)?$/i; +/** + * The path's segments. A fixed query or fragment, as in `/search?type=items`, names nothing; only + * the last round, the hash, reads the whole path, so paths that differ only there stay distinct. + */ const segments = (path: string) => - path + (path.split(/[?#]/)[0] ?? path) .split("/") .map((segment) => segment.trim()) .filter((segment) => segment.length > 0); diff --git a/packages/apps/src/implementation/openapi-request.ts b/packages/apps/src/implementation/openapi-request.ts index 97ac1e6faa..a97c0ec48d 100644 --- a/packages/apps/src/implementation/openapi-request.ts +++ b/packages/apps/src/implementation/openapi-request.ts @@ -162,8 +162,21 @@ function reservedPathValue(value: unknown): string { * A prefix check alone permits an empty item ID to reach a collection endpoint. * Reserved resource names may span segments; ordinary values must stay in one segment. */ +/** + * A declared path's route and its fixed query. A path Executor v1 imported can carry a query, such + * as `/search?type=items`, which is sent before the operation's own parameters, and a fragment, + * which is never sent. + */ +function declaredPath(path: string) { + const [declared = path] = path.split("#"); + const at = declared.indexOf("?"); + return at === -1 + ? { route: declared, query: "" } + : { route: declared.slice(0, at), query: declared.slice(at + 1) }; +} + function operationPath(op: OpenapiOperation): RegExp { - const template = op.baseUrl.replace(/\/$/, "") + op.path; + const template = op.baseUrl.replace(/\/$/, "") + declaredPath(op.path).route; // Keep real template parameters distinct from literal percent-encoded braces. let marker = "executorPathParameter"; while (template.includes(marker)) marker += "_"; @@ -278,13 +291,21 @@ export function createRequest(config: { if (authorized === undefined) throw new Error("The selected account cannot call this tool"); const parameters: Record = {}; - // Reserved path values are expanded before Swagger sees the operation. - let path = op.path; + // Reserved path values are expanded before Swagger sees the operation's route. + const declared = declaredPath(op.path); + let path = declared.route; + // A path parameter the fixed query names is filled in there too, as Executor v1 did. + let fixedQuery = declared.query; // Swagger has no `querystring` serializer, so the whole query string is added here. const querystrings: string[] = []; const swaggerParameters = op.request.parameters.filter((p) => { const group = args[p.in === "header" ? "headers" : p.in]; const value = group === undefined ? undefined : object(group)[p.name]; + if (p.in === "path" && value !== undefined && fixedQuery.includes(`{${p.name}}`)) + fixedQuery = fixedQuery.replaceAll( + `{${p.name}}`, + encodeURIComponent(scalar(value)), + ); if (p.in === "path" && p.allowReserved === true && p.content === undefined) { if (value === undefined) throw new Error("A required path parameter is missing"); path = path.replaceAll(`{${p.name}}`, reservedPathValue(value)); @@ -352,8 +373,8 @@ export function createRequest(config: { if (pathname.split("/").some((segment) => /^(?:\.|%2e){1,2}$/i.test(segment))) throw new Error("This path value would change the request path"); const url = new URL(prepared.url); - if (querystrings.length > 0) - url.search = [url.search.slice(1), ...querystrings] + if (fixedQuery !== "" || querystrings.length > 0) + url.search = [fixedQuery, url.search.slice(1), ...querystrings] .filter((part) => part !== "") .join("&"); const origin = new URL(op.baseUrl).origin; diff --git a/packages/apps/src/openapi.ts b/packages/apps/src/openapi.ts index 50043a54c7..80fd3dfe85 100644 --- a/packages/apps/src/openapi.ts +++ b/packages/apps/src/openapi.ts @@ -1,8 +1,10 @@ /** OpenAPI helpers over normalized operation metadata; no optional dependency is needed. */ import { protocolRouter, type OperationKinds } from "./implementation/protocol-operations.ts"; -import { Effect } from "effect"; +import { Effect, Schema } from "effect"; import type { OpenapiToolsOptions } from "./contracts/openapi.ts"; +import { JsonObject } from "./contracts/schema.ts"; import { openapiToolsEffect } from "./implementation/openapi.ts"; +import { compiledOperationNames } from "./implementation/openapi-compile.ts"; export { OpenapiError, isOpenapiTextMedia, @@ -34,6 +36,34 @@ export * from "./contracts/openapi-compile.ts"; * operationId, first tag, method and path alone. It reads no document and compiles nothing. */ export { planOperationNames, type OperationNameInput } from "./implementation/openapi-names.ts"; + +/** One operation a document declares, as `openapiOperationNames` lists it. */ +export interface OpenapiDeclaredOperation { + /** The tool name `liveOpenapiRouter` gives the operation. */ + readonly name: string; + /** The HTTP method, such as `GET`. */ + readonly method: string; + /** The path as the document declares it. */ + readonly path: string; +} + +/** + * Every operation a document declares, in document order, with the name `liveOpenapiRouter` gives + * it, whether or not the router can import it. Unlike `planOperationNames`, it reads the document + * as the router does, references included. It rejects with `OpenapiCompileError` when the + * document is not one the router reads. The document is not modified. + */ +export const openapiOperationNames = ( + document: Readonly>, +): Promise => + Effect.runPromise( + Effect.suspend(() => + compiledOperationNames( + // A copy the reader can own, as a tree. + Schema.decodeUnknownSync(JsonObject)(JSON.parse(JSON.stringify(document))), + ), + ), + ); export { liveOpenapiRouter, openapiToolNames, From 1b67b8af4526ae15f9a726e5e832e3df2f503834 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 23 Oct 2026 08:50:26 +0000 Subject: [PATCH 009/127] Export 833987e --- e2e/tests/provider-errors.spec.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/e2e/tests/provider-errors.spec.ts b/e2e/tests/provider-errors.spec.ts index 81d18b1d5a..11d21385f7 100644 --- a/e2e/tests/provider-errors.spec.ts +++ b/e2e/tests/provider-errors.spec.ts @@ -388,8 +388,13 @@ export default defineApp({ user: { accounts: { service: provider.many() } } }, a .then(() => page.getByRole("button", { name: "Run tool", exact: true }).waitFor()), ); const input = JSON.stringify({ accountId: affected, input: {} }); + // The accounts share one schema, so the runner opens on the form; the call is entered as JSON. yield* browser.use("Enter tool input", (page) => - page.getByLabel("Input", { exact: true }).fill(input), + page + .getByRole("tablist", { name: "Input format", exact: true }) + .getByRole("tab", { name: "JSON", exact: true }) + .click() + .then(() => page.getByLabel("Input", { exact: true }).fill(input)), ); yield* upstream.configure({ status: 401, phase: "call" }); yield* browser.use("Run the failing tool", (page) => From f69c667936e4c02c75f9d7cc79fb750c230839b9 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:10:09 +0000 Subject: [PATCH 010/127] Export d89244c --- .changeset/openapi-parameter-descriptions.md | 8 +++ bun.lock | 2 +- e2e/tests/app-routers.spec.ts | 53 ++++++++++++++++++- packages/apps/package.json | 2 +- .../src/implementation/openapi-compile.ts | 29 ++++++++-- 5 files changed, 87 insertions(+), 7 deletions(-) create mode 100644 .changeset/openapi-parameter-descriptions.md diff --git a/.changeset/openapi-parameter-descriptions.md b/.changeset/openapi-parameter-descriptions.md new file mode 100644 index 0000000000..8c7704d6bf --- /dev/null +++ b/.changeset/openapi-parameter-descriptions.md @@ -0,0 +1,8 @@ +--- +"apps": patch +--- + +OpenAPI tools keep each parameter's `description` and `deprecated` in their +input schema, so `describe` shows agents a path, query, header or cookie +parameter's documentation beside it, as it already does for body properties and +MCP tool properties. Search's one-line input types are unchanged. diff --git a/bun.lock b/bun.lock index b9c5750d6a..afc8616d7c 100644 --- a/bun.lock +++ b/bun.lock @@ -486,7 +486,7 @@ }, "packages/apps": { "name": "apps", - "version": "0.0.1-beta.102", + "version": "0.0.1-beta.103", "dependencies": { "@cfworker/json-schema": "4.1.1", "@csstools/css-tokenizer": "4.0.2", diff --git a/e2e/tests/app-routers.spec.ts b/e2e/tests/app-routers.spec.ts index 618cc7c746..0a91277e0a 100644 --- a/e2e/tests/app-routers.spec.ts +++ b/e2e/tests/app-routers.spec.ts @@ -24,6 +24,10 @@ const Wire = Schema.Struct({ const instructions = "Search the docs before answering. Cite the page you used."; +/** Property documentation a server or document gives, which only describe shows agents. */ +const startDoc = "Start time in unix milliseconds, not an ISO 8601 string."; +const statusDoc = "One of: draft, open, paid."; + /** An MCP server that describes itself during initialization. One tool name contains a dot. */ const docsServer = Effect.gen(function* () { const routes = Layer.mergeAll( @@ -54,7 +58,14 @@ const docsServer = Effect.gen(function* () { tools: ["lookup", "search.pages"].map((name) => ({ name, description: `Synthetic ${name}`, - inputSchema: { type: "object", properties: {}, additionalProperties: false }, + inputSchema: { + type: "object", + properties: + name === "lookup" + ? { start: { type: ["integer", "string"], description: startDoc } } + : {}, + additionalProperties: false, + }, // One tool declares the shape of its structured content; the other declares none. ...(name === "lookup" ? { @@ -102,10 +113,20 @@ const billing = (origin: string) => ({ get: { operationId: "listInvoices", tags: ["invoices"], + // OpenAPI documents a parameter on the parameter, beside its schema. + parameters: [ + { $ref: "#/components/parameters/Status" }, + { name: "page", in: "query", deprecated: true, schema: { type: "integer" } }, + ], responses: { "200": { description: "Invoices" } }, }, }, }, + components: { + parameters: { + Status: { name: "status", in: "query", description: statusDoc, schema: { type: "string" } }, + }, + }, }); /** Static routers, an MCP server and an OpenAPI document under keys, and a server that is down. */ @@ -470,6 +491,36 @@ return { items: described.items, lookup: await docs.lookup({}), pages: await doc "const name: unknown = value.structuredContent?.tool;", ), ).toEqual([]); + // Describe documents each property with the text its server or document gives, attached + // to the property it describes. Search's one-line input type stays compact without it. + const documented = yield* execute( + "Describe documented properties", + `const found = await tools.search({ namespace: ${JSON.stringify(app.slug)}, limit: 50 }); +const paths = found.items.map((item) => item.path).filter((path) => /\\.(docs\\.lookup|listInvoices)$/.test(path)); +const described = await tools.search.describe({ paths }); +return { found: found.items.filter((item) => paths.includes(item.path)), items: described.items };`, + ); + const properties = yield* Schema.decodeUnknownEffect( + Schema.Struct({ + found: Schema.Array(Schema.Struct({ path: Schema.String, input: Schema.String })), + items: Described.fields.items, + }), + )(documented.execution.value); + const describedAs = (path: string) => + properties.items.find((item) => item.path.endsWith(path))?.signature ?? ""; + expect(describedAs(".docs.lookup")).toMatch( + new RegExp(`/\\*\\* ${startDoc} \\*/\\s*start\\?: number \\| string`), + ); + expect(describedAs(".listInvoices")).toMatch( + new RegExp(`/\\*\\* ${statusDoc} \\*/\\s*status\\?: string`), + ); + expect(describedAs(".listInvoices")).toMatch( + /\/\*\* @deprecated @integer \*\/\s*page\?: number/, + ); + expect(properties.found.map((item) => item.input).sort()).toEqual([ + "{ query?: { status?: string, page?: number } }", + "{ start?: number | string }", + ]); const failed = yield* execute( "Call a tool in the router whose server is down", `return await tools[${JSON.stringify(app.slug)}].offline.anything({});`, diff --git a/packages/apps/package.json b/packages/apps/package.json index 75f0e0b986..5a4523803e 100644 --- a/packages/apps/package.json +++ b/packages/apps/package.json @@ -1,6 +1,6 @@ { "name": "apps", - "version": "0.0.1-beta.102", + "version": "0.0.1-beta.103", "private": true, "description": "TypeScript apps with typed tools, accounts, storage, and UI.", "type": "module", diff --git a/packages/apps/src/implementation/openapi-compile.ts b/packages/apps/src/implementation/openapi-compile.ts index 8541f22e13..1f6a8c6eb4 100644 --- a/packages/apps/src/implementation/openapi-compile.ts +++ b/packages/apps/src/implementation/openapi-compile.ts @@ -128,6 +128,17 @@ const binaryInput: JsonObject = { description: "File bytes encoded as base64.", contentEncoding: "base64", }; +/** + * A Parameter Object's documentation, which OpenAPI keeps beside its schema rather than in it. + * It describes this use of the schema, so it replaces the schema's own on the input property; + * requests never read it. Documentation alone never fails an operation that is otherwise valid. + */ +const parameterDocumentation = (parameter: JsonObject): JsonObject => ({ + ...(typeof parameter.description === "string" && parameter.description.trim() !== "" + ? { description: parameter.description } + : {}), + ...(parameter.deprecated === true ? { deprecated: true } : {}), +}); /** * Where a Responses, Request Body or Parameter entry is declared: the object a reference resolved * to, or the entry's own place under its operation. @@ -649,7 +660,10 @@ export const compileOpenApiDocument = ( }), ), ]; - const parameters = new Map(); + const parameters = new Map< + string, + { parameter: Parameter; documentation: JsonObject; at: string | undefined } + >(); for (const { parameter, at: parameterAt } of declared) { const declaration = document.resolve(record(parameter)); // A parameter in a place OpenAPI 3 does not have, such as Swagger 2.0's `body`, is not @@ -666,7 +680,11 @@ export const compileOpenApiDocument = ( `The querystring parameter at ${parameterAt} must declare one form, JSON or text media type in content.`, ); } - parameters.set(`${p.in}:${p.name}`, { parameter: p, at: parameterAt }); + parameters.set(`${p.in}:${p.name}`, { + parameter: p, + documentation: parameterDocumentation(declaration), + at: parameterAt, + }); } // A path prefix parameter the operation declares without placing it keeps its schema. for (const name of prefix) { @@ -679,6 +697,7 @@ export const compileOpenApiDocument = ( if (!parameters.has(key)) parameters.set(key, { parameter: { name, in: "path", required: true, schema: { type: "string" } }, + documentation: {}, at: undefined, }); } @@ -687,7 +706,7 @@ export const compileOpenApiDocument = ( string, { properties: Record Json>; required: string[] } >(); - for (const { parameter: p, at: parameterAt } of parameters.values()) { + for (const { parameter: p, documentation, at: parameterAt } of parameters.values()) { const key = p.in === "header" ? "headers" : p.in; const group = groups.get(key) ?? { properties: {}, required: [] }; const [mediaType, content] = @@ -700,7 +719,9 @@ export const compileOpenApiDocument = ( : mediaSchemaAt(document, `${parameterAt}`, mediaType, content); // A parameter that only the path prefix declares is Executor's own Draft 2020-12. group.properties[p.name] = - parameterAt === undefined ? () => schema : (api) => api(schema, schemaAt); + parameterAt === undefined + ? () => schema + : (api) => ({ ...api(schema, schemaAt), ...documentation }); if (p.required || p.in === "path") group.required.push(p.name); groups.set(key, group); } From fe1afa85026a90c4cb775347cdd6ef509865001e Mon Sep 17 00:00:00 2001 From: The-AarushiSingh <175547726+The-AarushiSingh@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:39:05 +0530 Subject: [PATCH 011/127] Reject control characters and invisible-only app names A NUL in an app name returned a 500 storage error, and names made only of invisible characters created an app that listed as blank. AppName now refuses control characters and line separators and requires a visible character, and quick-add uses the same schema. Stored names are read with NonEmptyString, so existing apps are unaffected. Co-authored-by: Cursor --- e2e/test-plan.ts | 10 ++++ e2e/tests/app-name-validation.spec.ts | 63 +++++++++++++++++++++++ packages/catalog/src/contracts/imports.ts | 4 +- packages/sdk/src/contracts/apps.ts | 10 +++- 4 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 e2e/tests/app-name-validation.spec.ts diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index caed05bc63..532b1a39c6 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -3524,6 +3524,16 @@ export const scenarios = plan({ local: na("Exercises the shared importer and OAuth setup through hosted APIs."), }, }, + appNameValidation: { + fixtures: "actors", + file: "app-name-validation.spec.ts", + title: "App names with control or invisible-only characters are refused with a bad request", + targets: { + "self-host": scheduled, + cloud: na("Cloud egress refuses loopback MCP hosts, so no Cloud scenario imports one."), + local: na("Exercises the shared importer and app rename through hosted APIs."), + }, + }, importApprovals: { fixtures: "actors", file: "import-approvals.spec.ts", diff --git a/e2e/tests/app-name-validation.spec.ts b/e2e/tests/app-name-validation.spec.ts new file mode 100644 index 0000000000..bb201be7ca --- /dev/null +++ b/e2e/tests/app-name-validation.spec.ts @@ -0,0 +1,63 @@ +/** App names that storage cannot hold or that would list as blank are refused, not stored or failed. */ +import { expect, layer } from "@effect/vitest"; +import { Effect, Schema } from "effect"; +import { randomUUID } from "node:crypto"; +import { Actors } from "../support/actors.ts"; +import { Api, body } from "../support/api.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { oauthSetupIssuer } from "../support/oauth-setup-issuer.ts"; +import { scenarios } from "../test-plan.ts"; + +const App = Schema.Struct({ id: Schema.String, name: Schema.String }); + +/** A NUL cannot be stored, a newline breaks the one-line name, and invisible text lists as blank. */ +const refused = ["Zero\u0000Width", "Two\nlines", "Tabbed\tname", "\u200b\u200b", "\u200d"]; + +layer(HostedLive, { excludeTestServices: true })("App name validation", (it) => { + it.effect(scenarios.appNameValidation.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors; + const issuer = yield* oauthSetupIssuer; + yield* issuer.configure({ serveMcp: true }); + const prefix = `/api/organizations/${actors.organization.id}`; + const suffix = randomUUID().slice(0, 8); + const url = `${issuer.origin}/mcp`; + + const imported = yield* api.request(actors.owner, "POST", `${prefix}/apps/import`, { + source: { kind: "mcp", name: `Named ${suffix}`, url }, + }); + expect(imported.status, JSON.stringify(imported.body)).toBe(200); + const app = yield* body(App, imported); + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", `${prefix}/apps/${app.id}`).pipe(Effect.orDie), + ); + const appPath = `${prefix}/apps/${app.id}`; + + for (const name of refused) { + const label = JSON.stringify(name); + // Quick add and rename both refuse the name as a bad request, never a storage error. + const add = yield* api.request(actors.owner, "POST", `${prefix}/apps/import`, { + source: { kind: "mcp", name, url }, + }); + expect(add.status, `import ${label}: ${JSON.stringify(add.body)}`).toBe(400); + const rename = yield* api.request(actors.owner, "PATCH", `${appPath}/name`, { name }); + expect(rename.status, `rename ${label}: ${JSON.stringify(rename.body)}`).toBe(400); + } + const unchanged = yield* body(App, yield* api.request(actors.owner, "GET", appPath)); + expect(unchanged.name).toBe(`Named ${suffix}`); + + // Names in other scripts, with accents or joined emoji, stay valid. + const kept = `Café 日本語 👨‍👩‍👧 ${suffix}`; + const renamed = yield* api.request(actors.owner, "PATCH", `${appPath}/name`, { + name: kept, + }); + expect(renamed.status, JSON.stringify(renamed.body)).toBe(200); + const read = yield* body(App, yield* api.request(actors.owner, "GET", appPath)); + expect(read.name).toBe(kept); + }), + ), + ); +}); diff --git a/packages/catalog/src/contracts/imports.ts b/packages/catalog/src/contracts/imports.ts index 15db664282..f7e4cfbb62 100644 --- a/packages/catalog/src/contracts/imports.ts +++ b/packages/catalog/src/contracts/imports.ts @@ -1,6 +1,6 @@ /** Custom imports retain public configuration; credentials are connected after deploying. */ import { Schema } from "effect"; -import { HttpUrl } from "@executor-js/sdk"; +import { AppName, HttpUrl } from "@executor-js/sdk"; /** Credential-free URL syntax; the product decides which network destinations are allowed. */ export const ImportUrl = HttpUrl.check( @@ -18,7 +18,7 @@ export const ImportUrl = HttpUrl.check( */ export const CustomAppInput = Schema.Struct({ kind: Schema.Literal("mcp"), - name: Schema.NonEmptyString, + name: AppName, url: ImportUrl, }); export type CustomAppInput = typeof CustomAppInput.Type; diff --git a/packages/sdk/src/contracts/apps.ts b/packages/sdk/src/contracts/apps.ts index 4be92feab6..d90f175e22 100644 --- a/packages/sdk/src/contracts/apps.ts +++ b/packages/sdk/src/contracts/apps.ts @@ -149,11 +149,17 @@ export const sameAccounts = (left: AppAccounts, right: AppAccounts) => { /** AppAccounts in a GET query: JSON text, so an empty requirement survives as []. */ export const AppAccountsQuery = Schema.optional(Schema.fromJsonString(AppAccounts)); -/** Display names are nonblank and bounded; they do not replace stable app identities. */ +/** + * Display names are one bounded line with a visible character; they do not replace stable app + * identities. Control characters are refused because storage cannot hold NUL, and names made only + * of invisible characters would list as blank. Names already stored are read with `NonEmptyString`, + * so this only narrows what new writes accept. + */ export const AppName = Schema.String.check( Schema.isMinLength(1), Schema.isMaxLength(120), - Schema.isPattern(/\S/u), + Schema.isPattern(/^[^\p{Cc}\p{Zl}\p{Zp}]*$/u), + Schema.isPattern(/[^\s\p{Cc}\p{Cf}]/u), ); /** Informational origin captured when a copy is made. It never grants access or drives updates. */ From 5baa59d0c3a8e8e5f8f817f68d2b9c3847565eaa Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:22:21 +0000 Subject: [PATCH 012/127] Export 0a0c0510 --- .../cloud/web/src/implementation/analytics-replay.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/hosted/cloud/web/src/implementation/analytics-replay.ts b/apps/hosted/cloud/web/src/implementation/analytics-replay.ts index 228317ff4d..9ee184a552 100644 --- a/apps/hosted/cloud/web/src/implementation/analytics-replay.ts +++ b/apps/hosted/cloud/web/src/implementation/analytics-replay.ts @@ -62,9 +62,11 @@ export const dashboardReplay: SessionRecordingOptions = { collectFonts: false, inlineStylesheet: true, captureCanvas: { recordCanvas: false }, - // rrweb also uses this callback for its required page metadata. Keep only a fixed route label. - maskCapturedNetworkRequestFn: (request) => - Object.keys(request).length === 1 && typeof request.name === "string" - ? { ...request, name: `${location.origin}/dashboard` } - : null, + // rrweb also uses this callback for its required page metadata. Keep only the page's path, which + // replayPageAllowed has already vetted; drop the query string and hash. + maskCapturedNetworkRequestFn: (request) => { + if (Object.keys(request).length !== 1 || typeof request.name !== "string") return null; + const url = new URL(request.name, location.origin); + return { ...request, name: `${url.origin}${url.pathname}` }; + }, }; From e2dcb6157dd6dcf2f477349dec5f90234f23a7e0 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:24:55 +0000 Subject: [PATCH 013/127] Export 2c930372 --- apps/hosted/cloud/alchemy.github.ts | 14 + .../cloud/src/implementation/auth-options.ts | 3 + .../cloud/src/implementation/migrations.ts | 17 + .../src/implementation/v1-session-handoff.ts | 313 ++++++++++++++++++ apps/hosted/cloud/src/infrastructure/auth.ts | 55 ++- .../cloud/src/infrastructure/secrets.ts | 40 ++- .../src/infrastructure/v1-import-settings.ts | 7 + apps/hosted/cloud/src/main.ts | 3 + e2e/support/cloud-environment.ts | 9 + e2e/support/v1-session-handoff.ts | 31 ++ e2e/test-plan.ts | 12 + e2e/tests/cloud-v1-session-handoff.spec.ts | 288 ++++++++++++++++ 12 files changed, 767 insertions(+), 25 deletions(-) create mode 100644 apps/hosted/cloud/src/implementation/v1-session-handoff.ts create mode 100644 e2e/support/v1-session-handoff.ts create mode 100644 e2e/tests/cloud-v1-session-handoff.spec.ts diff --git a/apps/hosted/cloud/alchemy.github.ts b/apps/hosted/cloud/alchemy.github.ts index af77c04f22..5028b18865 100644 --- a/apps/hosted/cloud/alchemy.github.ts +++ b/apps/hosted/cloud/alchemy.github.ts @@ -31,6 +31,7 @@ import { missingV1McpForwardSecret, v1ImportWriterName, v1McpForwardSecretName, + v1SessionHandoffSecretName, v1ImportOperatorTokenMinLength, missingV1ImportOperatorToken, v1FirstPartyClientsName, @@ -513,6 +514,19 @@ export default Alchemy.Stack( environment: production, }).pipe(retain()); + /** + * The secret v1 signs session handoffs with (`v1-session-handoff.ts`), the same value as v1's + * `V2_SESSION_HANDOFF_SECRET`. Without it a member v1 sends to a moved workspace signs in to v2. + */ + const v1Handoff = yield* nonEmptySecret(v1SessionHandoffSecretName); + if (Option.isSome(v1Handoff)) + yield* GitHub.Secret(`production-${v1SessionHandoffSecretName}`, { + ...target, + name: v1SessionHandoffSecretName, + value: v1Handoff.value, + environment: production, + }).pipe(retain()); + // Rulesets are unavailable on private repositories under the GitHub Free plan, so // `CI_RULESET_ENFORCEMENT=disabled` skips the resource instead of asking GitHub for it. const ruleset = diff --git a/apps/hosted/cloud/src/implementation/auth-options.ts b/apps/hosted/cloud/src/implementation/auth-options.ts index 6f83e0b9e5..ac5369d0bb 100644 --- a/apps/hosted/cloud/src/implementation/auth-options.ts +++ b/apps/hosted/cloud/src/implementation/auth-options.ts @@ -24,6 +24,7 @@ import { emulatedSocialProviders } from "./emulated-auth.ts"; import { cloudSso, ssoVerifiedEmail } from "./sso.ts"; import { chatGptSettings, chatGptSignIn } from "./chatgpt-sign-in.ts"; import { cloudMemberLimit } from "./member-limit.ts"; +import { v1HandoffSessions } from "./v1-session-handoff.ts"; /** The better-auth endpoint that creates accounts from a verified email code. */ const emailCodeSignInPath = "/sign-in/email-otp"; @@ -361,6 +362,8 @@ export const cloudAuthOptions = ( ), }), ...(billing === undefined ? [] : [cloudMemberLimit(billing)]), + // Server only: sessions for members v1 vouches for (`v1-session-handoff.ts`). + v1HandoffSessions(), // The native migrator creates tables in plugin order; SSO references organization. cloudSso(billing), emailOTP({ diff --git a/apps/hosted/cloud/src/implementation/migrations.ts b/apps/hosted/cloud/src/implementation/migrations.ts index 8fa83bc342..7a860477ce 100644 --- a/apps/hosted/cloud/src/implementation/migrations.ts +++ b/apps/hosted/cloud/src/implementation/migrations.ts @@ -163,6 +163,22 @@ export const migrateV1ImportOrgs = Effect.gen(function* () { )`; }).pipe(Effect.mapError(migrationFailed("product"))); +/** + * The v1 session handoff tokens v2 has accepted (`v1-session-handoff.ts`), so each is used once. + * A row is needed only until its token expires; the handoff deletes expired rows. Additive: the + * running server never reads it. + */ +export const migrateV1SessionHandoffs = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`create table if not exists private_v1_session_handoff ( + jti text primary key, + expires_at timestamptz not null, + used_at timestamptz not null default now() + )`; + yield* sql`create index if not exists private_v1_session_handoff_expires + on private_v1_session_handoff (expires_at)`; +}).pipe(Effect.mapError(migrationFailed("product"))); + /** * The onboarding experiment's assignments (`notes/posthog.md#onboarding-order-experiment`): one * row per member who created their first team through team setup and was assigned a variant, @@ -202,6 +218,7 @@ export const migrateCloudDatabase = Effect.scoped( "5_v1_import_map": migrateV1ImportMap, "6_setup_assignments": migrateSetupAssignments, "7_v1_import_orgs": migrateV1ImportOrgs, + "8_v1_session_handoffs": migrateV1SessionHandoffs, }); }); yield* migrateHostedDatabase(setup.options, cloudMigrations).pipe( diff --git a/apps/hosted/cloud/src/implementation/v1-session-handoff.ts b/apps/hosted/cloud/src/implementation/v1-session-handoff.ts new file mode 100644 index 0000000000..30dd59ae0c --- /dev/null +++ b/apps/hosted/cloud/src/implementation/v1-session-handoff.ts @@ -0,0 +1,313 @@ +/** + * Sign-in handoff from v1 for members of moved workspaces (`notes/v1-import.md`, Session handoff). + * + * v2 offers no WorkOS sign-in and never starts one. v1's dashboard sends a signed-in member of an + * organization that has moved to `/api/v1-handoff?token=…` after it has verified its own WorkOS + * session and the member's membership. The token is v1 vouching for that session, signed with a + * secret only v1 (`V2_SESSION_HANDOFF_SECRET`) and v2 (`V1_SESSION_HANDOFF_SECRET`) hold: + * + * v1hnd.."> + * + * The claims name the WorkOS user, the v1 organization and the v2 organization's slug, with `iat`, + * an `exp` at most a minute later and a random `jti`. v2 accepts each `jti` once, maps the WorkOS + * user to a v2 user only through the import map, requires that user's membership of the + * organization the v1 organization was imported as, and starts an ordinary Better Auth session. + * Every refusal opens the v2 sign-in page, returning to the organization. Nothing here reads v1. + */ +import { AuthenticationUnavailable } from "@executor-js/hosted-server"; +import { OrganizationSlug } from "@executor-js/hosted-server/organization"; +import type { BetterAuthPlugin } from "better-auth"; +import { APIError, createAuthEndpoint } from "better-auth/api"; +import { setSessionCookie } from "better-auth/cookies"; +import { Clock, Effect, Option, Redacted, Result, Schema } from "effect"; +import { Base64Url } from "effect/encoding"; +import { HttpServerRequest, HttpServerResponse } from "effect/http"; +import { SqlClient } from "effect/sql"; + +/** The route v1's dashboard redirects to. */ +export const v1HandoffPath = "/api/v1-handoff"; + +const prefix = "v1hnd."; +const audience = "v2-session-handoff"; +/** v1 gives a token a minute; one that claims longer is refused. */ +const maxLifetimeSeconds = 60; +/** How far ahead of this Worker's clock v1's may run. */ +const clockSkewSeconds = 30; + +const Claims = Schema.Struct({ + v: Schema.Literal(1), + aud: Schema.Literal(audience), + /** The WorkOS user whose v1 session v1 verified. */ + user: Schema.NonEmptyString, + /** The v1 organization that moved, a WorkOS organization ID. */ + organization: Schema.NonEmptyString, + /** The v2 organization to open, as v1's moved flag names it. */ + slug: OrganizationSlug, + iat: Schema.Int, + exp: Schema.Int, + jti: Schema.String.check(Schema.isMinLength(16), Schema.isMaxLength(128)), +}); +type Claims = typeof Claims.Type; + +/** Only what a refusal needs to read from a token it did not accept: where to return to. */ +const Destination = Schema.Struct({ slug: Schema.optionalKey(Schema.Unknown) }); + +/** Why a handoff opened the sign-in page; logged, never shown, and never with the token. */ +const V1HandoffRefusal = Schema.Literals([ + "disabled", + "malformed", + "signature", + "version", + "audience", + "expired", + "lifetime", + "replayed", + "unmapped_user", + "unknown_organization", + "organization_mismatch", + "not_member", + "other_session", +]); +type V1HandoffRefusal = typeof V1HandoffRefusal.Type; + +export class V1HandoffRefused extends Schema.TaggedError()("V1HandoffRefused", { + reason: V1HandoffRefusal, +}) {} + +const refuse = (reason: V1HandoffRefusal) => new V1HandoffRefused({ reason }); + +/** Better Auth could not start the session. */ +export class V1HandoffSessionFailed extends Schema.TaggedError()( + "V1HandoffSessionFailed", + {}, +) {} + +const encoder = new TextEncoder(); + +/** The token's parts, when it has the shape v1 signs. */ +const split = (token: string) => { + if (!token.startsWith(prefix)) return undefined; + const [payload, signature, ...rest] = token.slice(prefix.length).split("."); + if (payload === undefined || signature === undefined || rest.length > 0) return undefined; + return { payload, signature }; +}; + +/** The decoded JSON of a payload, or undefined. */ +const payloadJson = (payload: string) => + Result.match(Base64Url.decodeString(payload), { + onFailure: () => undefined, + onSuccess: (json) => Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown))(json), + }); + +/** + * The claims of a token v1 signed with `secret` that is current. The MAC is checked with + * `crypto.subtle.verify`, which compares in constant time, before anything in the payload is read. + */ +const verify = (secret: Redacted.Redacted, token: string) => + Effect.gen(function* () { + const parts = split(token); + if (parts === undefined) return yield* refuse("malformed"); + const mac = Base64Url.decode(parts.signature); + if (Result.isFailure(mac)) return yield* refuse("malformed"); + const valid = yield* Effect.promise(async () => { + const key = await crypto.subtle.importKey( + "raw", + encoder.encode(Redacted.value(secret)), + { name: "HMAC", hash: "SHA-256" }, + false, + ["verify"], + ); + return crypto.subtle.verify( + "HMAC", + key, + new Uint8Array(mac.success), + encoder.encode(`${prefix}${parts.payload}`), + ); + }); + if (!valid) return yield* refuse("signature"); + const json = payloadJson(parts.payload); + if (json === undefined || Option.isNone(json)) return yield* refuse("malformed"); + const loose = Schema.decodeUnknownOption( + Schema.Struct({ v: Schema.Unknown, aud: Schema.Unknown }), + )(json.value); + if (Option.isNone(loose)) return yield* refuse("malformed"); + if (loose.value.v !== 1) return yield* refuse("version"); + if (loose.value.aud !== audience) return yield* refuse("audience"); + const claims = yield* Schema.decodeUnknownEffect(Claims)(json.value).pipe( + Effect.mapError(() => refuse("malformed")), + ); + const now = Math.floor((yield* Clock.currentTimeMillis) / 1000); + if (claims.exp <= now) return yield* refuse("expired"); + if ( + claims.exp <= claims.iat || + claims.exp - claims.iat > maxLifetimeSeconds || + claims.iat > now + clockSkewSeconds + ) + return yield* refuse("lifetime"); + return claims; + }); + +/** + * Accept the token's `jti` once. Rows of expired tokens are deleted first: an expired token is + * refused before it gets here, so its row no longer guards anything. The margin covers the clock + * difference between this Worker and the database. + */ +const consume = (sql: SqlClient.SqlClient, claims: Claims) => + Effect.gen(function* () { + yield* sql` + delete from private_v1_session_handoff where expires_at < now() - interval '5 minutes'`; + const accepted = yield* sql` + insert into private_v1_session_handoff (jti, expires_at) + values (${claims.jti}, to_timestamp(${claims.exp})) + on conflict (jti) do nothing + returning jti`; + if (accepted.length === 0) return yield* refuse("replayed"); + }); + +const Target = Schema.Struct({ + userId: Schema.NullOr(Schema.String), + imported: Schema.NullOr(Schema.String), + organization: Schema.NullOr(Schema.String), + role: Schema.NullOr(Schema.String), +}); + +/** + * The v2 user and organization the claims stand for, in one statement. The user is the import + * map's record of the WorkOS user, created or merged by the import; nothing is matched by email. + * The organization is the slug's, which must be the one the v1 organization was imported as. + */ +const target = (sql: SqlClient.SqlClient, claims: Claims) => + Effect.gen(function* () { + const rows = yield* sql` + with u as ( + select v2_id from private_v1_import_map + where v1_kind = 'user' and v1_id = ${claims.user} and status = 'imported' + and v2_kind in ('user', 'merged_user') + ), o as ( + select id from organization where slug = ${claims.slug} + ) + select + (select v2_id from u) as "userId", + (select v2_id from private_v1_import_map + where v1_kind = 'organization' and v1_id = ${claims.organization} + and status = 'imported') as imported, + (select id from o) as organization, + (select role from member + where "userId" = (select v2_id from u) and "organizationId" = (select id from o)) as role`; + const row = yield* Schema.decodeUnknownEffect(Target)(rows[0]); + if (row.userId === null) return yield* refuse("unmapped_user"); + if (row.organization === null) return yield* refuse("unknown_organization"); + if (row.imported !== row.organization) return yield* refuse("organization_mismatch"); + if (row.role === null) return yield* refuse("not_member"); + return { user: row.userId, organization: row.organization }; + }); + +/** Where every response sends the browser: never back to a URL that carries the token. */ +const redirect = (location: string, cookies: ReadonlyArray = []) => { + const headers = new Headers({ + location, + "referrer-policy": "no-referrer", + "cache-control": "no-store", + }); + for (const cookie of cookies) headers.append("set-cookie", cookie); + return HttpServerResponse.fromWeb(new Response(null, { status: 302, headers })); +}; + +/** The sign-in page, returning to the organization the token named when it names one. */ +const signIn = (token: string) => { + const parts = split(token); + const json = parts === undefined ? undefined : payloadJson(parts.payload); + const slug = + json === undefined || Option.isNone(json) + ? undefined + : Option.getOrUndefined(Schema.decodeUnknownOption(Destination)(json.value))?.slug; + const returnTo = Schema.is(OrganizationSlug)(slug) ? `/org/${slug}` : "/"; + return redirect(`/login?redirect=${encodeURIComponent(returnTo)}`); +}; + +export interface V1HandoffServices { + readonly secret: Option.Option>; + readonly withSql: (effect: Effect.Effect) => Effect.Effect; + /** The user of the session the request already carries, if any. */ + readonly currentUser: ( + headers: Headers, + ) => Effect.Effect; + /** Start a Better Auth session for the user; its `Set-Cookie` values. */ + readonly startSession: ( + headers: Headers, + user: string, + organization: string, + ) => Effect.Effect, V1HandoffSessionFailed, R>; +} + +/** `GET /api/v1-handoff?token=…`: a session for the member v1 vouches for, or the sign-in page. */ +export const v1SessionHandoff = (services: V1HandoffServices) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const token = new URL(request.url, "http://localhost").searchParams.get("token") ?? ""; + const headers = new Headers(request.headers); + return yield* Effect.gen(function* () { + if (Option.isNone(services.secret)) return yield* refuse("disabled"); + const claims = yield* verify(services.secret.value, token); + const { user, organization } = yield* services.withSql( + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* consume(sql, claims); + return yield* target(sql, claims); + }), + ); + const current = yield* services.currentUser(headers); + // A browser already signed in to v2 keeps its session: as this user it needs no other, and + // another user's session is never replaced by a handoff. + if (current !== undefined && current !== user) return yield* refuse("other_session"); + const cookies = + current === user ? [] : yield* services.startSession(headers, user, organization); + return redirect(`/org/${claims.slug}`, cookies); + }).pipe( + Effect.catchTag("V1HandoffRefused", (error) => + Effect.logInfo("v1 session handoff refused", { reason: error.reason }).pipe( + Effect.as(signIn(token)), + ), + ), + // Storage, decoding or Better Auth failed: the member can still sign in. + Effect.catch(() => + Effect.logWarning("v1 session handoff unavailable", { reason: "unavailable" }).pipe( + Effect.as(signIn(token)), + ), + ), + Effect.withSpan("auth.v1SessionHandoff"), + ); + }); + +/** + * Better Auth's half: a session for a user the handoff has already authorized, created and set as + * every sign-in's is, so the session hooks (verified email, sign-in analytics) run too. Server only: + * it has no URL, and only `v1SessionHandoff` calls it. + */ +export const v1HandoffSessions = () => + ({ + id: "executor-v1-session-handoff", + endpoints: { + startV1HandoffSession: createAuthEndpoint( + "/v1-handoff/session", + { + method: "POST", + requireHeaders: true, + body: Schema.toStandardSchemaV1( + Schema.Struct({ userId: Schema.NonEmptyString, organizationId: Schema.NonEmptyString }), + ), + metadata: { SERVER_ONLY: true }, + }, + async (ctx) => { + const user = await ctx.context.internalAdapter.findUserById(ctx.body.userId); + if (!user) throw new APIError("NOT_FOUND"); + const session = await ctx.context.internalAdapter.createSession(user.id, false, { + activeOrganizationId: ctx.body.organizationId, + }); + if (!session) throw new APIError("INTERNAL_SERVER_ERROR"); + await setSessionCookie(ctx, { session, user }); + return ctx.json({ userId: user.id }); + }, + ), + }, + }) satisfies BetterAuthPlugin; diff --git a/apps/hosted/cloud/src/infrastructure/auth.ts b/apps/hosted/cloud/src/infrastructure/auth.ts index 24ff2fa73d..e252e751ae 100644 --- a/apps/hosted/cloud/src/infrastructure/auth.ts +++ b/apps/hosted/cloud/src/infrastructure/auth.ts @@ -41,7 +41,8 @@ import * as Cloudflare from "alchemy/Cloudflare"; import { Context, Effect, Layer, Option, Redacted, Schema, type Scope } from "effect"; import { HttpBody, HttpServerRequest, HttpServerResponse } from "effect/http"; import type { SendAuthEmail } from "../contracts/email.ts"; -import { cloudSecrets, v1McpForwardSecret } from "./secrets.ts"; +import { cloudSecrets, v1McpForwardSecret, v1SessionHandoffSecret } from "./secrets.ts"; +import { V1HandoffSessionFailed, v1SessionHandoff } from "../implementation/v1-session-handoff.ts"; import { cloudAuthRateLimit } from "./stage.ts"; import { AuthDatabase, appSessionsPerCall, boundAuthAdapter } from "./auth-database.ts"; import { invocationSql } from "./invocation-database.ts"; @@ -60,6 +61,7 @@ export const cloudAuth = (send: SendAuthEmail, onboarding: typeof Onboarding.Ser }; const secrets = yield* cloudSecrets.pipe(Effect.orDie); const v1Forward = yield* v1McpForwardSecret; + const v1Handoff = yield* v1SessionHandoffSecret; const meter = yield* BillingMeter.pipe(Effect.provide(yield* billingLive)); // Better Auth invokes Promise callbacks. Carry the calling request's scope, // bindings and cancellation through that boundary, isolated per invocation. @@ -313,17 +315,48 @@ export const cloudAuth = (send: SendAuthEmail, onboarding: typeof Onboarding.Ser ), Effect.map(HttpServerResponse.setHeader("cache-control", "no-store")), ); + const browserSession = (headers: Headers) => + nativeCall((instance) => + instance.api.getSession({ + headers, + query: { disableRefresh: true, disableCookieCache: true }, + }), + ).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(BrowserSession)), + Effect.mapError(() => new AuthenticationUnavailable()), + ); + const v1HandoffHandler = v1SessionHandoff({ + secret: v1Handoff, + withSql, + currentUser: (headers) => + browserSession(headers).pipe(Effect.map((session) => session?.user.id)), + // Inside the auth callbacks' context, like a request to the handler: the session hooks + // record the sign-in. + startSession: (headers, userId, organizationId) => + Effect.gen(function* () { + const [instance, bind] = yield* bound; + const context = yield* Effect.context< + RuntimeContext | HttpServerRequest.HttpServerRequest | Scope.Scope + >(); + const result = yield* Effect.tryPromise({ + try: (signal) => + callbacks.run({ context, signal }, () => + bind(() => + instance.api.startV1HandoffSession({ + headers, + body: { userId, organizationId }, + returnHeaders: true, + }), + ), + ), + catch: () => new V1HandoffSessionFailed(), + }); + return result.headers.getSetCookie(); + }), + }); return { - browserSession: (headers: Headers) => - nativeCall((instance) => - instance.api.getSession({ - headers, - query: { disableRefresh: true, disableCookieCache: true }, - }), - ).pipe( - Effect.flatMap(Schema.decodeUnknownEffect(BrowserSession)), - Effect.mapError(() => new AuthenticationUnavailable()), - ), + browserSession, + v1Handoff: v1HandoffHandler, identity, mcpIdentity, apiIdentity, diff --git a/apps/hosted/cloud/src/infrastructure/secrets.ts b/apps/hosted/cloud/src/infrastructure/secrets.ts index ff0281478e..3f5aa848c9 100644 --- a/apps/hosted/cloud/src/infrastructure/secrets.ts +++ b/apps/hosted/cloud/src/infrastructure/secrets.ts @@ -2,7 +2,7 @@ import { Random } from "alchemy"; import { Config, Effect, Option, Redacted, Schema } from "effect"; import { testStage } from "./stage.ts"; -import { v1McpForwardSecretName } from "./v1-import-settings.ts"; +import { v1McpForwardSecretName, v1SessionHandoffSecretName } from "./v1-import-settings.ts"; const SigningSecret = Schema.Redacted(Schema.String.check(Schema.isMinLength(32))); /** AES-GCM credentials need exactly 32 raw bytes, hex encoded. */ @@ -52,19 +52,31 @@ export const cloudSecrets = Effect.gen(function* () { return { authSecret: yield* authSecret.text, encryptionKey } satisfies CloudSecrets; }); +/** + * A secret v1 shares with this deployment. GitHub Actions passes an unset secret as an empty + * string, so empty means unset; a deployment without it accepts nothing v1 signs with it. + */ +const v1SharedSecret = (name: string) => + Effect.gen(function* () { + const secret = (yield* Config.Redacted(name).pipe(Config.option)).pipe( + Option.filter((value) => Redacted.value(value) !== ""), + ); + if ( + Option.isSome(secret) && + Option.isNone(Schema.decodeUnknownOption(SigningSecret)(secret.value)) + ) + return yield* Effect.die(new Error(`${name} must be at least 32 characters`)); + return secret; + }); + /** * The secret v1 signs the MCP requests it forwards with (`v1-mcp-forward.ts`), shared with v1's - * `V2_MCP_FORWARD_SECRET`. A deployment without it accepts no forwarded request. GitHub Actions - * passes an unset secret as an empty string, so empty means unset. + * `V2_MCP_FORWARD_SECRET`. */ -export const v1McpForwardSecret = Effect.gen(function* () { - const secret = (yield* Config.Redacted(v1McpForwardSecretName).pipe(Config.option)).pipe( - Option.filter((value) => Redacted.value(value) !== ""), - ); - if ( - Option.isSome(secret) && - Option.isNone(Schema.decodeUnknownOption(SigningSecret)(secret.value)) - ) - return yield* Effect.die(new Error(`${v1McpForwardSecretName} must be at least 32 characters`)); - return secret; -}); +export const v1McpForwardSecret = v1SharedSecret(v1McpForwardSecretName); + +/** + * The secret v1 signs session handoffs with (`v1-session-handoff.ts`), shared with v1's + * `V2_SESSION_HANDOFF_SECRET`. Without it every handoff opens the sign-in page. + */ +export const v1SessionHandoffSecret = v1SharedSecret(v1SessionHandoffSecretName); diff --git a/apps/hosted/cloud/src/infrastructure/v1-import-settings.ts b/apps/hosted/cloud/src/infrastructure/v1-import-settings.ts index f0e6596014..d6c56178f3 100644 --- a/apps/hosted/cloud/src/infrastructure/v1-import-settings.ts +++ b/apps/hosted/cloud/src/infrastructure/v1-import-settings.ts @@ -77,6 +77,13 @@ export const missingV1ImportWriter = `v1 imports that write need ${v1ImportWrite export const v1McpForwardSecretName = "V1_MCP_FORWARD_SECRET"; export const missingV1McpForwardSecret = `v1 imports that write need ${v1McpForwardSecretName}, the secret v1 signs forwarded MCP requests with (v1's V2_MCP_FORWARD_SECRET): without it a moved workspace's MCP URLs stop working`; +/** + * The secret v1 signs session handoffs with, shared with v1's `V2_SESSION_HANDOFF_SECRET`: v1's + * dashboard sends a signed-in member of a moved organization to `/api/v1-handoff` with a token + * signed with it, so they need not sign in again (`v1-session-handoff.ts`). + */ +export const v1SessionHandoffSecretName = "V1_SESSION_HANDOFF_SECRET"; + /** * Where a non-production Cloud reads v1. `live` is v1's production WorkOS, database, blob bucket * and Autumn account, read-only, from local Cloud dev on a loopback origin only. It never writes: diff --git a/apps/hosted/cloud/src/main.ts b/apps/hosted/cloud/src/main.ts index 8bb6bfbd5b..0aca285d0c 100644 --- a/apps/hosted/cloud/src/main.ts +++ b/apps/hosted/cloud/src/main.ts @@ -59,6 +59,7 @@ import { Config, Effect, Layer, Option, Path, Ref } from "effect"; import { HttpRouter, HttpServer, HttpServerResponse } from "effect/http"; import { cloudAuth } from "./infrastructure/auth.ts"; import { cloudAccountCallback } from "./implementation/account-callback.ts"; +import { v1HandoffPath } from "./implementation/v1-session-handoff.ts"; import { cloudOnboarding, cloudV1ImportSettings } from "./infrastructure/onboarding.ts"; import { V1ImportConfiguration, @@ -600,6 +601,8 @@ export default Api.make( auth.handler.pipe(Effect.flatMap(hideRemovedOrganizations), Effect.provide(executor)), ), HttpRouter.add("*", "/api/auth/*", dispatchAfterWrites(auth.handler)), + // A member of a moved v1 workspace, signed in to v1, opens it here without signing in again. + HttpRouter.add("GET", v1HandoffPath, auth.v1Handoff), HttpRouter.add("*", "/api/email/unsubscribe", welcomeEmails.unsubscribe), HttpRouter.add( "GET", diff --git a/e2e/support/cloud-environment.ts b/e2e/support/cloud-environment.ts index 092fbdf0d9..e62111b797 100644 --- a/e2e/support/cloud-environment.ts +++ b/e2e/support/cloud-environment.ts @@ -22,6 +22,7 @@ import { v1DatabaseWriter, } from "./cloud-postgres.ts"; import { v1McpForwardFile } from "./v1-mcp-forward.ts"; +import { v1SessionHandoffFile } from "./v1-session-handoff.ts"; import { isAlchemyDevFailure, outputLines } from "./alchemy-dev-output.ts"; import { startFixtureControl, fixtureRequest } from "../sdk/fixtures.ts"; import { roleHost } from "./role-hosts.ts"; @@ -224,6 +225,13 @@ export const startCloudEnvironment = (input: { mode: 0o600, }); yield* Effect.addFinalizer(() => fs.remove(v1Forward).pipe(Effect.orDie)); + // The secret the synthetic v1 signs session handoffs with; scenarios read it from here. + const v1HandoffSecret = randomBytes(32).toString("hex"); + const v1Handoff = `${directory}/${v1SessionHandoffFile}`; + yield* fs.writeFileString(v1Handoff, JSON.stringify({ secret: v1HandoffSecret }), { + mode: 0o600, + }); + yield* Effect.addFinalizer(() => fs.remove(v1Handoff).pipe(Effect.orDie)); const env = { PATH: [path.join(cloud, "node_modules/.bin"), process.env.PATH ?? ""].join( process.platform === "win32" ? ";" : ":", @@ -275,6 +283,7 @@ export const startCloudEnvironment = (input: { // Imports freeze and cut over each organization in the emulated v1 through this login. V1_IMPORT_WRITER_DATABASE_URL: `postgresql://${v1DatabaseWriter}:${v1WriterPassword}@127.0.0.1:${input.databasePort}/executor_v1?sslmode=disable`, V1_MCP_FORWARD_SECRET: v1ForwardSecret, + V1_SESSION_HANDOFF_SECRET: v1HandoffSecret, // Serve the role hosts as production does, at `app.`, `mcp.` and `api.` of the origin's host. EXECUTOR_ROLE_HOSTS_DOMAIN: new URL(input.origin).hostname, CLOUD_BROWSER_ORIGIN: input.browserOrigin, diff --git a/e2e/support/v1-session-handoff.ts b/e2e/support/v1-session-handoff.ts new file mode 100644 index 0000000000..11ad0c493e --- /dev/null +++ b/e2e/support/v1-session-handoff.ts @@ -0,0 +1,31 @@ +import { createHmac, randomBytes } from "node:crypto"; +import { Schema } from "effect"; + +/** The secret a managed Cloud run shares with its synthetic v1 for session handoffs. */ +export const V1SessionHandoffSecret = Schema.Struct({ secret: Schema.String }); +/** Where a managed Cloud run records the secret it verifies v1 session handoffs with. */ +export const v1SessionHandoffFile = "v1-session-handoff.json"; + +/** What v1 vouches for in a session handoff; times are Unix seconds. */ +export interface V1HandoffClaims { + readonly user: string; + readonly organization: string; + readonly slug: string; + readonly iat: number; + readonly exp: number; + readonly jti: string; + readonly v?: number; + readonly aud?: string; +} + +/** A handoff token as v1's dashboard signs it, before redirecting a moved workspace's member. */ +export const v1HandoffToken = (secret: string, claims: V1HandoffClaims) => { + const payload = Buffer.from( + JSON.stringify({ v: 1, aud: "v2-session-handoff", ...claims }), + ).toString("base64url"); + const signature = createHmac("sha256", secret).update(`v1hnd.${payload}`).digest("base64url"); + return `v1hnd.${payload}.${signature}`; +}; + +/** A fresh token id, as v1 draws one for each handoff. */ +export const v1HandoffId = () => randomBytes(18).toString("base64url"); diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index caed05bc63..7d50ef03f8 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -7633,6 +7633,18 @@ export const scenarios = plan({ "Cloud v1 import keeps a workspace that fails verification frozen in v1, and accepts MCP requests v1 forwards for an imported workspace, with a new signed bearer for each request of a session, scopes a toolkit's bearer to the caller's own copy, and refuses a wrong signature, an expired or too long-lived bearer, an account, organization or toolkit copy the import never mapped, and a toolkit claim without its connection", targets: { ...cloudOnboarding, cloud: managedCloud }, }, + v1HandoffSignIn: { + file: "cloud-v1-session-handoff.spec.ts", + title: + "Cloud v1 import hands a moved workspace's member from their v1 session to v2: the handoff starts a session for the user the import mapped and opens the organization with no referrer, a browser signed in as them is only redirected, one signed in as another member keeps its session, and a used token opens sign-in", + targets: { ...cloudOnboarding, cloud: managedCloud }, + }, + v1HandoffRefusals: { + file: "cloud-v1-session-handoff.spec.ts", + title: + "Cloud v1 import opens v2 sign-in without a session for a handoff with a wrong signature, an expired or too long-lived token, another audience or version, a WorkOS user the import never mapped, or a member who has left the organization", + targets: { ...cloudOnboarding, cloud: managedCloud }, + }, v1ImportToolkitCalls: { file: "cloud-v1-import.spec.ts", title: diff --git a/e2e/tests/cloud-v1-session-handoff.spec.ts b/e2e/tests/cloud-v1-session-handoff.spec.ts new file mode 100644 index 0000000000..562a9ab15b --- /dev/null +++ b/e2e/tests/cloud-v1-session-handoff.spec.ts @@ -0,0 +1,288 @@ +/** + * v1 session handoff: v1's dashboard redirects a signed-in member of a workspace that has moved to + * v2's `/api/v1-handoff` with a token it signs for that member, and v2 starts its own session for + * the user the import mapped them to (`notes/v1-import.md`, Session handoff). The scenarios move a + * synthetic workspace with the operator's import, then sign tokens as v1 does with the secret the + * managed Cloud shares with its synthetic v1. + */ +import { randomUUID } from "node:crypto"; +import { expect, layer } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Schema } from "effect"; +import { TestLive, withCase } from "../support/case.ts"; +import { Browser } from "../support/browser.ts"; +import { writeRows } from "../support/cloud-rows.ts"; +import { Emulators } from "../support/emulators.ts"; +import { Onboarding } from "../support/onboarding.ts"; +import { driver, Target } from "../support/platform.ts"; +import { targetHosts } from "../support/role-hosts.ts"; +import { v1ImportOperator, v1RunId } from "../support/v1-import.ts"; +import { + V1SessionHandoffSecret, + v1HandoffId, + v1HandoffToken, + v1SessionHandoffFile, + type V1HandoffClaims, +} from "../support/v1-session-handoff.ts"; +import { scenarios } from "../test-plan.ts"; + +const services = Layer.mergeAll(Onboarding.layer, Emulators.layer); + +const Session = Schema.NullOr(Schema.Struct({ user: Schema.Struct({ email: Schema.String }) })); + +/** + * A two-member v1 workspace, moved to v2 by the operator's import and cut over. Both members are + * new to v2, so the import created their users; neither has signed in to v2. + */ +const movedWorkspace = Effect.gen(function* () { + const emulators = yield* Emulators, + operator = yield* v1ImportOperator, + fs = yield* FileSystem.FileSystem, + target = yield* Target; + const { secret } = yield* fs + .readFileString(`${target.directory}/${v1SessionHandoffFile}`) + .pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(V1SessionHandoffSecret))), + ); + const ownerEmail = `handoff-owner-${randomUUID().slice(0, 8)}@example.test`; + const memberEmail = `handoff-member-${randomUUID().slice(0, 8)}@example.test`; + const workspace = yield* emulators.v1Workspace([ownerEmail, memberEmail]); + const member = yield* emulators.v1User(memberEmail); + yield* writeRows("v1-database.json", [ + { + sql: "insert into organizations (id, name, slug) values ($1, 'Synthetic moved workspace', $2)", + params: [workspace.organization, `v1-${randomUUID().slice(0, 8)}`], + }, + ]); + const run = v1RunId("handoff"); + const started = yield* operator.start({ + mode: "import", + runId: run, + organizations: [workspace.organization], + }); + expect(started.status, JSON.stringify(started.body)).toBe(200); + const report = yield* operator.waitForReport(run, workspace.organization); + expect(report.status, report.reason ?? "").toBe("imported"); + expect(report.cutover).toBe(true); + const moved = report.v2Organization; + if (moved === undefined) return yield* Effect.die("The import named no organization"); + /** A token as v1 signs it for one handoff, a minute long. */ + const sign = (claims: Partial = {}, signedWith = secret) => { + const now = Math.floor(Date.now() / 1000); + return v1HandoffToken(signedWith, { + user: workspace.user, + organization: workspace.organization, + slug: moved.slug, + iat: now, + exp: now + 60, + jti: v1HandoffId(), + ...claims, + }); + }; + return { + sign, + slug: moved.slug, + v2Organization: moved.id, + owner: { user: workspace.user, email: ownerEmail }, + member: { user: member, email: memberEmail }, + }; +}); + +const handoffPath = (token: string) => `/api/v1-handoff?token=${encodeURIComponent(token)}`; +const signInFor = (slug: string) => `/login?redirect=${encodeURIComponent(`/org/${slug}`)}`; + +/** One handoff request from a browser with no session, without following its redirect. */ +const signedOut = (token: string) => + Effect.gen(function* () { + const target = yield* Target; + const url = `${targetHosts(target).browser}${handoffPath(token)}`; + return yield* driver("Open a v1 handoff signed out", (signal) => + fetch(url, { redirect: "manual", signal }).then((response) => + response.text().then(() => ({ + status: response.status, + location: response.headers.get("location"), + referrerPolicy: response.headers.get("referrer-policy"), + cookies: response.headers.getSetCookie().length, + })), + ), + ); + }); + +/** One handoff request from the signed-in browser, with its cookies, without following it. */ +const fromBrowser = (step: string, token: string) => + Effect.gen(function* () { + const browser = yield* Browser; + return yield* browser.use(step, (page) => + page + .context() + .request.get(handoffPath(token), { maxRedirects: 0 }) + .then((response) => ({ + status: response.status(), + location: response.headers()["location"] ?? null, + cookies: response.headersArray().filter((header) => header.name === "set-cookie").length, + })), + ); + }); + +/** The email of the browser's v2 session, or null when it has none. */ +const sessionEmail = (step: string) => + Effect.gen(function* () { + const browser = yield* Browser; + const session = yield* browser + .use(step, (page) => + page.evaluate(() => fetch("/api/auth/get-session").then((response) => response.json())), + ) + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Session))); + return session?.user.email ?? null; + }); + +layer(TestLive, { excludeTestServices: true })("Cloud v1 session handoff", (it) => { + it.effect(scenarios.v1HandoffSignIn.title, (context) => + withCase( + context, + Effect.gen(function* () { + const browser = yield* Browser; + const moved = yield* movedWorkspace; + const token = moved.sign(); + + // The browser follows the redirects; the handoff's own response is the first of them. + const handoff = yield* browser.use("Open the handoff v1 redirects to", (page) => + page.goto(handoffPath(token)).then((response) => { + let request = response?.request() ?? null; + while (request !== null && new URL(request.url()).pathname !== "/api/v1-handoff") + request = request.redirectedFrom(); + return request?.response() ?? null; + }), + ); + if (handoff === null) return yield* Effect.die("The handoff did not redirect"); + const headers = yield* browser.use("Read the handoff's response", () => + handoff.headersArray(), + ); + const header = (name: string) => + headers.filter((entry) => entry.name.toLowerCase() === name).map((entry) => entry.value); + const landed = yield* browser.use( + "The handoff opens the organization's dashboard", + (page) => + page + .getByRole("heading", { name: /^Apps(?:\s*\d+)?$/ }) + .waitFor({ state: "visible" }) + .then(() => new URL(page.url()).pathname), + ); + expect(landed.startsWith(`/org/${moved.slug}/`)).toBe(true); + const opened = { + status: handoff.status(), + location: header("location")[0], + referrerPolicy: header("referrer-policy")[0], + cacheControl: header("cache-control")[0], + sessionCookie: header("set-cookie").length, + }; + expect(opened).toMatchObject({ + status: 302, + location: `/org/${moved.slug}`, + referrerPolicy: "no-referrer", + cacheControl: "no-store", + }); + expect(opened.sessionCookie).toBeGreaterThan(0); + expect(yield* sessionEmail("Read the session the handoff started")).toBe(moved.owner.email); + yield* browser.checkpoint("signed in through the v1 handoff"); + + // Signed in as the same user, a new handoff only redirects. + expect(yield* fromBrowser("Hand off the same user again", moved.sign())).toEqual({ + status: 302, + location: `/org/${moved.slug}`, + cookies: 0, + }); + + // Another user's handoff never replaces the session the browser holds. + expect( + yield* fromBrowser( + "Hand off another member to the signed-in browser", + moved.sign({ user: moved.member.user }), + ), + ).toEqual({ status: 302, location: signInFor(moved.slug), cookies: 0 }); + expect(yield* sessionEmail("The browser keeps its own session")).toBe(moved.owner.email); + + // The token is used: replayed from anywhere, it opens sign-in and starts no session. + expect(yield* signedOut(token)).toEqual({ + status: 302, + location: signInFor(moved.slug), + referrerPolicy: "no-referrer", + cookies: 0, + }); + }).pipe(Effect.provide(services)), + ), + ); + + it.effect(scenarios.v1HandoffRefusals.title, (context) => + withCase( + context, + Effect.gen(function* () { + const moved = yield* movedWorkspace; + const now = Math.floor(Date.now() / 1000); + const refused = { + status: 302, + location: signInFor(moved.slug), + referrerPolicy: "no-referrer", + cookies: 0, + }; + // Not signed by v1, expired, longer-lived than v1 makes them, or for another audience. + expect(yield* signedOut(moved.sign({}, "x".repeat(64)))).toEqual(refused); + expect(yield* signedOut(moved.sign({ iat: now - 120, exp: now - 60 }))).toEqual(refused); + expect(yield* signedOut(moved.sign({ exp: now + 3_600 }))).toEqual(refused); + expect(yield* signedOut(moved.sign({ aud: "v2-mcp-forward" }))).toEqual(refused); + expect(yield* signedOut(moved.sign({ v: 2 }))).toEqual(refused); + // A WorkOS user the import never mapped, even one in no other workspace. + expect(yield* signedOut(moved.sign({ user: `user_${randomUUID()}` }))).toEqual(refused); + // The organization's slug claimed for a v1 organization that was not imported as it. + expect(yield* signedOut(moved.sign({ organization: `org_${randomUUID()}` }))).toEqual( + refused, + ); + // The endpoint that starts the session is server-only: over HTTP it does not exist, even + // for the mapped user and their organization. + const ownerRows = yield* writeRows("sso-database.json", [ + { + sql: `select v2_id from private_v1_import_map where v1_kind = 'user' and v1_id = $1`, + params: [moved.owner.user], + }, + ]); + const owner = yield* Schema.decodeUnknownEffect(Schema.Struct({ v2_id: Schema.String }))( + ownerRows[0]?.[0], + ); + const direct = yield* Effect.gen(function* () { + const target = yield* Target; + const url = `${targetHosts(target).browser}/api/auth/v1-handoff/session`; + return yield* driver("Call the handoff's session endpoint directly", (signal) => + fetch(url, { + method: "POST", + redirect: "manual", + signal, + headers: { "content-type": "application/json", origin: targetHosts(target).browser }, + body: JSON.stringify({ userId: owner.v2_id, organizationId: moved.v2Organization }), + }).then((response) => + response.text().then(() => ({ + status: response.status, + cookies: response.headers.getSetCookie().length, + })), + ), + ); + }); + expect(direct).toEqual({ status: 404, cookies: 0 }); + // A member who has left the v2 organization since it moved. + yield* writeRows("sso-database.json", [ + { + sql: `delete from member where "organizationId" = $1 and "userId" = (select v2_id from private_v1_import_map where v1_kind = 'user' and v1_id = $2) returning id`, + params: [moved.v2Organization, moved.member.user], + }, + ]).pipe(Effect.map((rows) => expect(rows[0]).toHaveLength(1))); + expect(yield* signedOut(moved.sign({ user: moved.member.user }))).toEqual(refused); + // The workspace itself hands off: what was refused was each token. + const accepted = yield* signedOut(moved.sign()); + expect(accepted).toMatchObject({ + status: 302, + location: `/org/${moved.slug}`, + referrerPolicy: "no-referrer", + }); + expect(accepted.cookies).toBeGreaterThan(0); + }).pipe(Effect.provide(services)), + ), + ); +}); From 5beb4dd51426f5675882c2a90b24b8a8ea4b4a72 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:03:31 +0000 Subject: [PATCH 014/127] Export 07c90955 --- .../src/implementation/v1-import/items.ts | 22 +++++++++++-------- e2e/tests/cloud-v1-import.spec.ts | 9 ++++++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/apps/hosted/cloud/src/implementation/v1-import/items.ts b/apps/hosted/cloud/src/implementation/v1-import/items.ts index 9ec533a649..859158c1a4 100644 --- a/apps/hosted/cloud/src/implementation/v1-import/items.ts +++ b/apps/hosted/cloud/src/implementation/v1-import/items.ts @@ -487,10 +487,12 @@ const rank = { approve: 1, default: 1.5, require_approval: 2, block: 3 } as cons * restrictive, where a connection without a matching policy keeps the tool's own default. * `aliases` maps a dropped v1 name to the name the tool keeps. * - * A toolkit's own policies applied only to calls through the toolkit, but v2 keeps approvals in - * the app for every caller. A toolkit that asks before a tool it grants therefore makes the app - * ask; its `approve` never lowers the workspace's result, and its `block` only leaves the tool out - * of the toolkit's connection. + * A toolkit's own policies applied only to calls through the toolkit, and v1 ignored the + * workspace's policies there, but v2 keeps approvals in the app for every caller. A toolkit that + * asks before a tool it grants therefore makes the app ask. One that grants it without a rule of its + * own left it to the tool's default, so the workspace's `approve` cannot lower it below that + * default either. The toolkit's `approve` never lowers the workspace's result, and its `block` only + * leaves the tool out of the toolkit's connection. */ export const toolActions = ( snapshot: V1Snapshot, @@ -503,11 +505,13 @@ export const toolActions = ( for (const tool of tools) { const id = `${integration.slug}.${tool.owner}.${tool.connection}.${tool.name}`; const own = resolvePolicy(id, snapshot.policies, snapshot.multiMember) ?? "default"; - const action = - rank[own] < rank.require_approval && - toolkits.some((toolkit) => toolkitAction(toolkit, id) === "require_approval") - ? "require_approval" - : own; + // The least the app may do for each toolkit that grants the tool: ask when the toolkit asks, + // keep the tool's default when the toolkit left it to the default. + const action = toolkits.reduce((current, toolkit) => { + const granted = toolkitAction(toolkit, id); + const floor = granted === "require_approval" || granted === "default" ? granted : undefined; + return floor !== undefined && rank[floor] > rank[current] ? floor : current; + }, own); const name = Object.hasOwn(aliases, tool.name) ? (aliases[tool.name] ?? tool.name) : tool.name; const current = actions.get(name); if (current === undefined || rank[action] > rank[current]) actions.set(name, action); diff --git a/e2e/tests/cloud-v1-import.spec.ts b/e2e/tests/cloud-v1-import.spec.ts index ee16aefabd..44f266d7ff 100644 --- a/e2e/tests/cloud-v1-import.spec.ts +++ b/e2e/tests/cloud-v1-import.spec.ts @@ -1192,6 +1192,12 @@ const seedToolkits = (organization: string, user: string, slug: string) => organization, `${hosted}.*.*.purge_items`, ), + // The workspace never asks before listing, but v1 ignored that inside the toolkit. + row( + "insert into tool_policy (tenant, owner, subject, pattern, action, position) values ($1, 'org', '', $2, 'approve', 'a1')", + organization, + `${hosted}.*.*.list_items`, + ), toolkitRow("user", "toolkits", { id: "tk_support", name: "Support", slug: "support" }), toolkitRow("user", "toolkitConnections", { id: "tkconn_1", @@ -3410,6 +3416,9 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import", (it) => { const rules = source.files.find((file) => file.path === "tools.ts")?.content ?? ""; expect(rules).toContain('"create_item": "always"'); expect(rules).not.toContain('"delete_item"'); + // The toolkit granted listing without a rule of its own, so v1 used the tool's default there; + // the workspace's approval does not become `never`, and the app keeps the default. + expect(rules).not.toContain('"list_items"'); // Artifacts become apps with source and no version, and say what needs repair. const dashboard = results.get("Dashboard"); From 2070e93fee98de31773abfed66884ea571193b3f Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:12:26 +0000 Subject: [PATCH 015/127] Export 18be383e --- .../src/implementation/v1-import/items.ts | 309 +++++++++++++----- .../cloud/src/implementation/v1-import/run.ts | 187 ++++++----- e2e/test-plan.ts | 2 +- e2e/tests/cloud-v1-import-run.spec.ts | 75 ++++- e2e/tests/cloud-v1-import.spec.ts | 106 ++++-- packages/sdk/src/contracts/oauth.ts | 27 +- packages/sdk/src/implementation/create.ts | 6 + packages/sdk/src/implementation/oauth.ts | 243 ++++++++------ 8 files changed, 654 insertions(+), 301 deletions(-) diff --git a/apps/hosted/cloud/src/implementation/v1-import/items.ts b/apps/hosted/cloud/src/implementation/v1-import/items.ts index 859158c1a4..b2b922c57e 100644 --- a/apps/hosted/cloud/src/implementation/v1-import/items.ts +++ b/apps/hosted/cloud/src/implementation/v1-import/items.ts @@ -43,8 +43,10 @@ import { SourceFiles, StorageHost, ToolName, + type AccountRequirement, type DeployedApp, type Executor, + type ImportOAuthGrant, type OAuthGrantNotImportable, type OwnerId, type OAuthSetupFailed, @@ -123,6 +125,12 @@ export interface ItemContext { readonly firstParty: FirstPartyClients; } +/** The v1 rows a connection's account is decided from. */ +export type ConnectionSnapshot = Pick< + V1Snapshot, + "tenant" | "subjects" | "integrations" | "connections" | "clients" +>; + /** The v1 rows an item reads are org rows or one member's own (`subject`). */ type Owned = { readonly owner: "org" | "user"; readonly subject: string }; @@ -398,7 +406,7 @@ export const fixedRequirement = "fixed"; * in `dropped` are left out. */ export const appMethods = ( - snapshot: V1Snapshot, + snapshot: ConnectionSnapshot, firstParty: FirstPartyClients, integration: IntegrationRow, dropped: readonly DroppedFixed[] = [], @@ -1015,6 +1023,15 @@ export const importIntegration = ( Effect.provideService(CurrentUserId, context.owner), ); +/** The app's fixed account's field names and values, from v1's configuration (decision 22). */ +const fixedAccount = (planned: AppAccounts) => + planned.fixed === undefined + ? fail("Its imported app takes fixed credentials its integration no longer has.") + : Effect.succeed({ + names: planned.fixed.account.fields, + values: Redacted.make(Object.fromEntries(planned.fixed.configured)), + }); + /** * Save the account that holds the integration's fixed credentials, from v1's configuration, and * link it to the app's `fixed` requirement for everyone (decision 22), when the app declares one. @@ -1033,12 +1050,8 @@ export const importFixedAccount = (context: ItemContext, integration: Integratio integration, yield* redundantFixed(context, integration), ); - if (planned.fixed === undefined) - return yield* fail("Its imported app takes fixed credentials its integration no longer has."); - const hash = yield* sourceHash({ - provider: requirement.provider, - fields: planned.fixed.account.fields, - }); + const fixed = yield* fixedAccount(planned); + const hash = yield* sourceHash({ provider: requirement.provider, fields: fixed.names }); const existing = yield* context.store.find("fixed_credentials", integration.row_id); const record = (result: MapResult) => context.store.record({ @@ -1048,7 +1061,6 @@ export const importFixedAccount = (context: ItemContext, integration: Integratio result, hash, }); - const fields = Object.fromEntries(planned.fixed.configured); const account = existing?.status === "imported" && existing.v2_id !== null && existing.source_hash === hash ? AccountId.make(existing.v2_id) @@ -1060,7 +1072,7 @@ export const importFixedAccount = (context: ItemContext, integration: Integratio provider: requirement.provider, method: fixedRequirement, label: `${integrationLabel(integration)} fixed credentials`, - fields: Redacted.make(fields), + fields: fixed.values, }) .pipe(accountDestination({ kind: "shared", audience: { kind: "everyone" } })); yield* record(imported("account", added.id)); @@ -1104,7 +1116,7 @@ export const checkIntegration = ( ); const generated = yield* generate(context, integration, plan.value, planned); const name = integrationLabel(integration); - yield* context.executor.apps.build({ + const built = yield* context.executor.apps.build({ owner: context.owner, name, files: yield* scopeGeneratedPackage(generated.files, context.slug, name).pipe( @@ -1117,9 +1129,36 @@ export const checkIntegration = ( stale: generated.stale, dropped: generated.dropped, health: generated.health, + accounts: built.requirements.accounts, }; }).pipe(Effect.catchTags(buildFailures)); +/** + * Check the account `importFixedAccount` would save for the app, against the requirements its + * build declares, and save nothing. + */ +export const checkFixedAccount = ( + context: Pick, + integration: IntegrationRow, + accounts: AppAccountRequirements, +) => + Effect.gen(function* () { + const requirement = accounts[fixedRequirement]; + if (requirement === undefined) return; + const planned = appMethods( + context.snapshot, + context.firstParty, + integration, + yield* redundantFixed(context, integration), + ); + yield* context.executor[OAuthGrantImport].checkAccountFields({ + provider: requirement.provider, + definition: requirement.definition, + method: fixedRequirement, + fields: (yield* fixedAccount(planned)).values, + }); + }); + // --------------------------------------------------------------------------------------------- // Connections. @@ -1242,7 +1281,7 @@ const secretSubjects = ( row.owner === "org" && itemOwner(itemId) === "user" ? snapshot.subjects : [row.subject]; /** The account's name; an org connection yields its name to a personal one. */ -export const accountLabel = (snapshot: V1Snapshot, row: ConnectionRow) => +export const accountLabel = (snapshot: Pick, row: ConnectionRow) => row.owner === "org" && snapshot.connections.some( (other) => @@ -1255,39 +1294,52 @@ export const accountLabel = (snapshot: V1Snapshot, row: ConnectionRow) => export const connectionRequirement = (row: Pick) => row.owner === "org" ? "workspace" : "service"; +/** The accounts an integration's app takes, by requirement: a deployed app's or a build's. */ +export type AppAccountRequirements = DeployedApp["requirements"]["accounts"]; + /** - * Save the connection's account: an org connection's is shared with everyone and created by the - * owner, a member's is their own. The map row commits with it. + * The account a connection's import saves, decided from v1's rows and secrets and the app's + * requirements, writing nothing: the requirement it fills and the grant or fields it holds, or + * why it cannot be saved. The import saves it (`importAccount`) and the dry run checks it + * (`checkAccount`), so a dry run refuses every account an import would, with the same reason. */ -export const importAccount = ( - context: ItemContext, - plan: ConnectionPlan, - app: Pick, - record: (result: MapResult) => Effect.Effect, -) => - Effect.gen(function* () { - const actor = actorOf(context, plan.row); - if (actor === undefined) return yield* fail("Its member is not imported."); - return yield* saveAccount(context, plan, app, record, actor).pipe( - Effect.provideService(CurrentUserId, actor), - ); - }); +type AccountWrite = + | { + readonly kind: "grant"; + readonly requirement: AccountRequirement; + readonly input: ImportOAuthGrant; + /** The operator's client a first-party grant moves to, which its refusal names. */ + readonly firstParty?: { + readonly name: string; + readonly client: { readonly placement: { readonly hosts: readonly string[] } }; + readonly hosts: readonly string[]; + }; + } + | { + readonly kind: "fields"; + readonly requirement: AccountRequirement; + readonly input: { + readonly owner: OwnerId; + readonly provider: AccountRequirement["provider"]; + readonly method: string; + readonly label: string; + readonly fields: Redacted.Redacted>; + }; + }; -const saveAccount = ( - context: ItemContext, +/** Reads one of a connection's secrets from v1's Vault, as `readSecret` does. */ +export type SecretReader = (itemId: string) => ReturnType; + +const accountWrite = ( + context: Pick & { readonly snapshot: ConnectionSnapshot }, plan: ConnectionPlan, - app: Pick, - record: (result: MapResult) => Effect.Effect, - actor: string, + accounts: AppAccountRequirements, + owner: OwnerId, + secret: SecretReader, ) => Effect.gen(function* () { - const requirement = app.requirements.accounts[connectionRequirement(plan.row)]; + const requirement = accounts[connectionRequirement(plan.row)]; if (requirement === undefined) return yield* fail("Its imported app takes no account."); - const creation = - plan.row.owner === "org" - ? accountDestination({ kind: "shared", audience: { kind: "everyone" } }) - : personalAccountCreation(actor); - const secret = (itemId: string) => readSecret(context, plan.row, itemId); const planned = appMethods( context.snapshot, context.firstParty, @@ -1296,41 +1348,34 @@ const saveAccount = ( ); const method = planned.methodOf.get(plan.row.row_id); if (method === undefined) return yield* fail("Its sign-in method is not in the imported app."); - const owner = organizationOwner(context.target.id); - const label = accountLabel(context.snapshot, plan.row); + const base = { + owner, + provider: requirement.provider, + method, + label: accountLabel(context.snapshot, plan.row), + }; + const expiresAt = plan.row.expires_at === null ? {} : { expiresAt: plan.row.expires_at }; const access = plan.access; if (access.kind === "first_party") { const accessToken = yield* secret(access.accessItem); const refreshToken = plan.row.refresh_item_id === null ? undefined : yield* secret(plan.row.refresh_item_id); - const appName = firstPartyName(access.client.id); const hosts = requirement.definition.hosts ?? []; - const account = yield* context.executor[OAuthGrantImport] - .importOAuthGrant({ - owner, - provider: requirement.provider, - method, - label, + return { + kind: "grant", + requirement, + input: { + ...base, redirectUri: v1OAuthCallback, firstParty: access.client.id, hosts, accessToken, refreshToken, - ...(plan.row.expires_at === null ? {} : { expiresAt: plan.row.expires_at }), + ...expiresAt, renewal: access.renewal, - }) - .pipe( - creation, - Effect.catchTag("OAuthSetupFailed", (error) => fail(oauthSetupReason(error.reason))), - Effect.catchTag("AuthMethodInvalid", () => - fail("Its sign-in method is not in the imported app."), - ), - Effect.catchTag("OAuthGrantNotImportable", (error) => - fail(firstPartyRefusal(appName, access.client, hosts, error)), - ), - ); - yield* record(imported("account", account.id)); - return account.id; + }, + firstParty: { name: firstPartyName(access.client.id), client: access.client, hosts }, + } satisfies AccountWrite as AccountWrite; } if (access.kind !== "oauth") { // Values from v1's configuration first, then the connection's own from v1's Vault. @@ -1340,21 +1385,11 @@ const saveAccount = ( if (access.kind === "secrets") for (const [variable, item] of access.items) fields[variable] = Redacted.value(yield* secret(item)); - return yield* context.executor[StorageHost].transaction( - Effect.gen(function* () { - const account = yield* context.executor.managedAccounts - .add({ - owner, - provider: requirement.provider, - method, - label, - fields: Redacted.make(fields), - }) - .pipe(creation); - yield* record(imported("account", account.id)); - return account.id; - }), - ); + return { + kind: "fields", + requirement, + input: { ...base, fields: Redacted.make(fields) }, + } satisfies AccountWrite as AccountWrite; } const { client, grant } = access; const clientSecret = @@ -1374,33 +1409,129 @@ const saveAccount = ( grant === "authorization_code" && plan.row.refresh_item_id !== null ? yield* secret(plan.row.refresh_item_id) : undefined; - const account = yield* context.executor[OAuthGrantImport] - .importOAuthGrant({ - owner, - provider: requirement.provider, - method, - label, + return { + kind: "grant", + requirement, + input: { + ...base, ...(grant === "authorization_code" ? { redirectUri: client.origin_redirect_uri ?? v1OAuthCallback } : {}), client: Redacted.make(registration), accessToken, refreshToken, - ...(plan.row.expires_at === null ? {} : { expiresAt: plan.row.expires_at }), + ...expiresAt, renewal: access.renewal, - }) - .pipe( - creation, - Effect.catchTag("OAuthSetupFailed", (error) => fail(oauthSetupReason(error.reason))), - Effect.catchTag("AuthMethodInvalid", () => - fail("Its sign-in method is not in the imported app."), - ), + }, + } satisfies AccountWrite as AccountWrite; + }); + +/** Why v2 refused to save a connection's grant, as the user reads it, for an import and a check. */ +const grantRefusals = (write: Extract) => + Effect.catchTags({ + OAuthSetupFailed: (error: OAuthSetupFailed) => fail(oauthSetupReason(error.reason)), + AuthMethodInvalid: () => fail("Its sign-in method is not in the imported app."), + OAuthGrantNotImportable: (error: OAuthGrantNotImportable) => + write.firstParty === undefined + ? fail("Its sign-in could not be carried over to v2. Reconnect it in v2.") + : fail( + firstPartyRefusal( + write.firstParty.name, + write.firstParty.client, + write.firstParty.hosts, + error, + ), + ), + }); + +/** Whether a connection's app is public: it takes no account, so the connection needs none. */ +export const needsNoAccount = (plan: ConnectionPlan, accounts: AppAccountRequirements) => + plan.access.kind === "none" && accounts.service === undefined && accounts.workspace === undefined; + +/** + * Save the connection's account: an org connection's is shared with everyone and created by the + * owner, a member's is their own. The map row commits with it. + */ +export const importAccount = ( + context: ItemContext, + plan: ConnectionPlan, + app: Pick, + record: (result: MapResult) => Effect.Effect, +) => + Effect.gen(function* () { + const actor = actorOf(context, plan.row); + if (actor === undefined) return yield* fail("Its member is not imported."); + return yield* saveAccount(context, plan, app, record, actor).pipe( + Effect.provideService(CurrentUserId, actor), + ); + }); + +const saveAccount = ( + context: ItemContext, + plan: ConnectionPlan, + app: Pick, + record: (result: MapResult) => Effect.Effect, + actor: string, +) => + Effect.gen(function* () { + const creation = + plan.row.owner === "org" + ? accountDestination({ kind: "shared", audience: { kind: "everyone" } }) + : personalAccountCreation(actor); + const write = yield* accountWrite( + context, + plan, + app.requirements.accounts, + organizationOwner(context.target.id), + (itemId) => readSecret(context, plan.row, itemId), + ); + if (write.kind === "fields") + return yield* context.executor[StorageHost].transaction( + Effect.gen(function* () { + const account = yield* context.executor.managedAccounts.add(write.input).pipe(creation); + yield* record(imported("account", account.id)); + return account.id; + }), ); + const account = yield* context.executor[OAuthGrantImport] + .importOAuthGrant(write.input) + .pipe(creation, grantRefusals(write)); // Discovery ran outside a transaction; the account is recorded as soon as it exists. yield* record(imported("account", account.id)); return account.id; }); +/** + * Check the connection's account as `importAccount` would save it, against the requirements its + * app's build declares, and save nothing: the dry run's check that every account moves. Its + * refusals are the import's, with the same reasons. `secret` reads what the import would read + * from v1's Vault, such as the values the dry run has already read. + */ +export const checkAccount = ( + context: Pick & { + readonly snapshot: ConnectionSnapshot; + /** Names the check in its errors; nothing is saved under it. */ + readonly owner: OwnerId; + }, + plan: ConnectionPlan, + accounts: AppAccountRequirements, + secret: SecretReader, +) => + Effect.gen(function* () { + const write = yield* accountWrite(context, plan, accounts, context.owner, secret); + const host = context.executor[OAuthGrantImport]; + if (write.kind === "fields") + return yield* host.checkAccountFields({ + provider: write.input.provider, + definition: write.requirement.definition, + method: write.input.method, + fields: write.input.fields, + }); + return yield* host + .checkOAuthGrant({ ...write.input, definition: write.requirement.definition }) + .pipe(grantRefusals(write)); + }); + /** * Link the connection's account to its app as linking it in v2 does: an org connection's to the * app's `workspace` requirement for everyone, by the owner; a member's to its `service` diff --git a/apps/hosted/cloud/src/implementation/v1-import/run.ts b/apps/hosted/cloud/src/implementation/v1-import/run.ts index 3f9f189685..a579d6ebeb 100644 --- a/apps/hosted/cloud/src/implementation/v1-import/run.ts +++ b/apps/hosted/cloud/src/implementation/v1-import/run.ts @@ -52,6 +52,8 @@ import { import { ItemFailed, artifactMapping, + checkAccount, + checkFixedAccount, checkIntegration, connectionTarget, importAccount, @@ -61,11 +63,13 @@ import { importToolkit, linkConnectionAccount, mapLookup, + needsNoAccount, readSecretCompared, refused, toolkitCopyKey, toolkitMembers, toolkitPolicy, + type AppAccountRequirements, type DroppedFixed, type ImportedLookup, type ItemContext, @@ -105,6 +109,9 @@ export const noToolMatches = "It matches no tool v1 last listed for its integrat export const personalApproveDropped = "In a workspace with more than one member, a member's own approve rules are not imported, so these tools ask again."; +/** Why a connection or rule of an integration whose app was not imported cannot move. */ +const integrationNotImported = "Its integration was not imported."; + /** The v2 kind of an app imported as source without a version, which needs repair. */ const undeployedApp = "undeployed_app"; @@ -460,6 +467,11 @@ interface Outcome { readonly staleTools?: readonly StaleTool[] | undefined; /** Fixed credentials an integration's app leaves out (decision 22). */ readonly droppedCredentials?: readonly DroppedFixed[] | undefined; + /** + * The accounts a dry run's build of an integration's app takes, which its connections' accounts + * are checked against as an import would save them. Provider definitions only, no secret. + */ + readonly accounts?: AppAccountRequirements | undefined; /** * An org connection whose user-scoped secret members hold with different values: how many hold * it and how many distinct values (decision 23). @@ -719,17 +731,38 @@ const dryIntegration = (assessment: Assessment, organization: string, id: string }, row, ).pipe( - Effect.map((checked): Outcome => ({ - kind: "integration", - id, - label, - context: null, - status: "planned", - note: checked.note, - traced: checked.traced, - staleTools: checked.stale, - droppedCredentials: checked.dropped, - })), + Effect.flatMap((checked) => { + const outcome: Outcome = { + kind: "integration", + id, + label, + context: null, + status: "planned", + note: checked.note, + traced: checked.traced, + staleTools: checked.stale, + droppedCredentials: checked.dropped, + accounts: checked.accounts, + }; + // As an import's `withFixedAccount`: a refused fixed account fails the integration, whose + // app its connections still get. + return checkFixedAccount( + { executor, snapshot, source, firstParty }, + row, + checked.accounts, + ).pipe( + Effect.as(outcome), + Effect.catch((error) => + transient(error) + ? Effect.fail(error) + : Effect.succeed({ + ...outcome, + status: "failed", + reason: clip(refusal(error)), + }), + ), + ); + }), Effect.catch((error) => transient(error) ? Effect.fail(error) @@ -744,42 +777,35 @@ const dryIntegration = (assessment: Assessment, organization: string, id: string ); }); -/** The Vault items a connection's import reads. */ -const connectionSecrets = ( - plan: Extract, { ok: true }>["value"], -) => { - const access = plan.access; - switch (access.kind) { - case "none": - return []; - case "secrets": - return access.items.map(([, item]) => item); - case "first_party": - return [ - access.accessItem, - ...(plan.row.refresh_item_id === null ? [] : [plan.row.refresh_item_id]), - ]; - case "oauth": - return [ - access.accessItem, - ...(access.grant === "authorization_code" && plan.row.refresh_item_id !== null - ? [plan.row.refresh_item_id] - : []), - ...(access.client.client_secret_item_id === null - ? [] - : [access.client.client_secret_item_id]), - ]; - } -}; +/** + * The apps a dry run built, by integration row id: the accounts each takes. An integration that + * did not build has none, as an import would have imported no app for it. + */ +const builtApps = (outcomes: readonly Outcome[]): ReadonlyMap => + new Map( + outcomes.flatMap((outcome) => + outcome.kind === "integration" && outcome.accounts !== undefined + ? [[outcome.id, outcome.accounts] as const] + : [], + ), + ); +/** + * Decide each connection as `importConnectionItem` would, writing nothing: its plan, its app, and + * the account `importAccount` would save, checked against its app's build (`checkAccount`), so + * every account an import would refuse is refused here with the same reason. Each secret the + * import reads is read too and kept nowhere; an org connection's secret a member saved is read in + * every member's partition, to report members who hold different values, compared in memory. + */ const dryConnections = ( assessment: Assessment, organization: string, ids: readonly string[], stale: StaleIntegrations, + apps: ReadonlyMap, ) => Effect.gen(function* () { - const { source, snapshot, firstParty, mismatchOf } = yield* dryContext( + const { source, snapshot, executor, firstParty, mismatchOf } = yield* dryContext( assessment, organization, ["integrations", "connections", "clients"], @@ -818,41 +844,47 @@ const dryConnections = ( reason: plan.reason, stale: plan.stale, } satisfies Outcome); - // Read every secret the import would, and keep none of them. An org connection's secret - // a member saved is read in every member's partition, to report members who hold - // different values; they are compared in memory. - const reads = yield* Effect.forEach(connectionSecrets(plan.value), (item) => - readSecretCompared({ source, snapshot }, row, item).pipe( - Effect.map((read) => ({ read, failed: undefined })), - Effect.catchTag("ItemFailed", (failed) => Effect.succeed({ read: undefined, failed })), - ), - ); - const failures = reads.map((read) => read.failed); - const failed = - failures.find((failure) => failure?.vault === "unavailable") ?? - failures.find((failure) => failure !== undefined); - const differing = reads.flatMap(({ read }) => - read !== undefined && read.values > 1 - ? [{ members: read.holders, values: read.values }] - : [], + const accounts = integration === undefined ? undefined : apps.get(integration.row_id); + if (accounts === undefined) + return { ...base, status: "failed", reason: integrationNotImported } satisfies Outcome; + if (needsNoAccount(plan.value, accounts)) + return { ...base, status: "planned" } satisfies Outcome; + const differing: Array<{ readonly members: number; readonly values: number }> = []; + const refusedBy = yield* checkAccount( + { executor, source, snapshot, firstParty, owner: OwnerId.make(`v1:${organization}`) }, + plan.value, + accounts, + (item) => + readSecretCompared({ source, snapshot }, row, item).pipe( + Effect.map((read) => { + if (read.values > 1) differing.push({ members: read.holders, values: read.values }); + return read.value; + }), + ), + ).pipe( + Effect.as(undefined), + Effect.catch((error) => (transient(error) ? Effect.fail(error) : Effect.succeed(error))), ); - if (failed === undefined) + if (refusedBy !== undefined) return { - kind: "connection", - id, - label: row.name, - context, - status: "planned", - ...(differing[0] === undefined - ? {} - : { - memberSecrets: { - members: Math.max(...differing.map((entry) => entry.members)), - values: Math.max(...differing.map((entry) => entry.values)), - }, - }), + ...base, + ...refusalOutcome(refusedBy), + ...(refusedBy instanceof ItemFailed && refusedBy.vault !== undefined + ? { vaultFailed: refusedBy.vault } + : {}), } satisfies Outcome; - return { ...base, ...refusalOutcome(failed), vaultFailed: failed.vault } satisfies Outcome; + return { + ...base, + status: "planned", + ...(differing.length === 0 + ? {} + : { + memberSecrets: { + members: Math.max(...differing.map((entry) => entry.members)), + values: Math.max(...differing.map((entry) => entry.values)), + }, + }), + } satisfies Outcome; }), ); }); @@ -1269,17 +1301,13 @@ const importConnectionItem = (context: ImportContext, id: string, stale: StaleIn ? undefined : yield* store.find("integration", integration.row_id); if (appRow?.status !== "imported" || appRow.v2_id === null) - return yield* new ItemFailed({ reason: "Its integration was not imported." }); + return yield* new ItemFailed({ reason: integrationNotImported }); const app = yield* executor.apps.get({ owner: organizationOwner(context.target.id), app: AppId.make(appRow.v2_id), }); // A public app needs no account or link. - if ( - plan.value.access.kind === "none" && - app.requirements.accounts.service === undefined && - app.requirements.accounts.workspace === undefined - ) { + if (needsNoAccount(plan.value, app.requirements.accounts)) { const result: MapResult = { status: "imported", v2Kind: "app", v2Id: app.id }; yield* recorded(context, "profile", id, hash)(result); return outcomeOf(base, result); @@ -1347,7 +1375,7 @@ const importPolicyItem = ( const matches = yield* policyMatches(context.source, snapshot, row.pattern, dropped); const result: MapResult = apps[0] === undefined - ? { status: "failed", reason: "Its integration was not imported." } + ? { status: "failed", reason: integrationNotImported } : { status: "imported", v2Kind: "app", @@ -2077,11 +2105,12 @@ export const runV1Import = ( ), ); const skippedIntegrations = staleIntegrations(outcomes); + const apps = builtApps(outcomes); for (const [index, batch] of chunks(items.connections, connectionBatch).entries()) outcomes.push( ...(yield* step( `connections ${index}`, - dryConnections(assessment, organization, batch, skippedIntegrations).pipe( + dryConnections(assessment, organization, batch, skippedIntegrations, apps).pipe( unlessUnreadable((reason) => batch.map((id) => failedOutcome("connection", id, reason)), ), diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index 7d50ef03f8..e1b137b743 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -7673,7 +7673,7 @@ export const scenarios = plan({ v1ImportFirstParty: { file: "cloud-v1-import.spec.ts", title: - "Cloud v1 import moves connections that signed in through Executor's own apps to v2's operator client of the same name, including a GitHub-shaped MCP server that names the scopeless client's endpoints, which renews them with the granted scopes, sends the token to each of a service's hosts the client reaches, and refuses an app v2 does not offer, a scope it does not allow and a host its tokens never reach", + "Cloud v1 import moves connections that signed in through Executor's own apps to v2's operator client of the same name, including a GitHub-shaped MCP server that names the scopeless client's endpoints, which renews them with the granted scopes, sends the token to each of a service's hosts the client reaches, and refuses an app v2 does not offer, a scope it does not allow, a host its tokens never reach, a server that does not sign in with it and one whose resource metadata names another resource, each of which its dry run refuses first for the same reason", targets: { ...cloudOnboarding, cloud: managedCloud }, }, v1ImportBuilds: { diff --git a/e2e/tests/cloud-v1-import-run.spec.ts b/e2e/tests/cloud-v1-import-run.spec.ts index 435696792b..ce164ef27c 100644 --- a/e2e/tests/cloud-v1-import-run.spec.ts +++ b/e2e/tests/cloud-v1-import-run.spec.ts @@ -4,9 +4,12 @@ * v1 rows into the managed Cloud's emulated v1 database, members into the emulated v1 WorkOS and * secrets into its Vault, and read v2's own rows to prove what was and was not written. */ +import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"; import { randomUUID } from "node:crypto"; +import { createServer } from "node:http"; import { expect, layer } from "@effect/vitest"; import { Effect, Layer, Schedule, Schema } from "effect"; +import { HttpRouter, HttpServer, HttpServerRequest, HttpServerResponse } from "effect/http"; import { TestLive, withCase } from "../support/case.ts"; import { Browser } from "../support/browser.ts"; import { writeRows, type RowStatement } from "../support/cloud-rows.ts"; @@ -18,6 +21,68 @@ import { scenarios } from "../test-plan.ts"; const services = Layer.mergeAll(Onboarding.layer, Emulators.layer); +/** + * OAuth MCP servers at `//mcp` on loopback, for each of `names`, as far as a sign-in reaches + * them: each refuses a request without a token with a challenge naming its resource metadata, + * which names this origin's authorization server. The import discovers them before it saves an + * account, and the dry run does too. + */ +const mcpSignInServers = (names: readonly string[]) => + Effect.gen(function* () { + const at = Effect.map( + HttpServerRequest.HttpServerRequest, + (request) => `http://${request.headers.host}`, + ); + const routes = Layer.mergeAll( + HttpRouter.add( + "GET", + "/.well-known/oauth-authorization-server", + Effect.flatMap(at, (origin) => + HttpServerResponse.json({ + issuer: origin, + authorization_endpoint: `${origin}/authorize`, + token_endpoint: `${origin}/token`, + response_types_supported: ["code"], + code_challenge_methods_supported: ["S256"], + token_endpoint_auth_methods_supported: ["client_secret_post"], + }), + ), + ), + ...names.flatMap((name) => [ + HttpRouter.add( + "*", + `/${name}/mcp`, + Effect.map(at, (origin) => + HttpServerResponse.empty({ + status: 401, + headers: { + "www-authenticate": `Bearer resource_metadata="${origin}/.well-known/oauth-protected-resource/${name}/mcp"`, + }, + }), + ), + ), + HttpRouter.add( + "GET", + `/.well-known/oauth-protected-resource/${name}/mcp`, + Effect.flatMap(at, (origin) => + HttpServerResponse.json({ + resource: `${origin}/${name}/mcp`, + authorization_servers: [origin], + }), + ), + ), + ]), + ); + const built = yield* Layer.build( + HttpRouter.serve(routes, { disableLogger: true, disableListenLog: true }).pipe( + Layer.provideMerge(NodeHttpServer.layer(createServer, { host: "127.0.0.1", port: 0 })), + ), + ); + const server = yield* HttpServer.HttpServer.pipe(Effect.provideContext(built)); + if (!("port" in server.address)) return yield* Effect.die("Fixture must listen on TCP"); + return `127.0.0.1:${server.address.port}`; + }); + const row = (sql: string, ...params: string[]): RowStatement => ({ sql, params }); const Rows = Schema.Array(Schema.Array(Schema.Unknown)); @@ -1103,9 +1168,11 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import runs", (it) => { clientId, secretItem ?? "", ); - const oauthMcp = (headers: Record, host: string) => ({ + // Reachable, so their sign-ins can be discovered as the import discovers them. + const signIn = yield* mcpSignInServers(["pasted", "gated"]); + const oauthMcp = (headers: Record, name: string) => ({ transport: "remote", - endpoint: `https://${host}/mcp`, + endpoint: `http://${signIn}/${name}/mcp`, remoteTransport: "auto", headers, authenticationTemplate: [{ slug: "oauth", kind: "oauth2" }], @@ -1196,7 +1263,7 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import runs", (it) => { CLIENT_ID: "synthetic-stale-client-0123456789", CLIENT_SECRET: "synthetic-stale-secret-0123456789", }, - "pasted.example.com", + "pasted", ), ), // A self-hosted MCP server behind a gateway that takes its own client ID and secret. @@ -1209,7 +1276,7 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import runs", (it) => { "CF-Access-Client-Id": "synthetic0gateway1client2id.access", "CF-Access-Client-Secret": "synthetic-gateway-secret-0123456789", }, - "gated.example.com", + "gated", ), ), row( diff --git a/e2e/tests/cloud-v1-import.spec.ts b/e2e/tests/cloud-v1-import.spec.ts index 44f266d7ff..082e69af42 100644 --- a/e2e/tests/cloud-v1-import.spec.ts +++ b/e2e/tests/cloud-v1-import.spec.ts @@ -2417,6 +2417,10 @@ const firstPartyService = (port: number) => HttpRouter.add("GET", "/slack/mcp", mcpGet("slack")), HttpRouter.add("POST", "/github/mcp", mcp("github")), HttpRouter.add("GET", "/github/mcp", mcpGet("github")), + // GitHub's MCP server at a sibling path, whose challenge still names the metadata of the + // resource at `/github/mcp`, which a sign-in here cannot use. + HttpRouter.add("POST", "/github/mcp-moved", mcp("github")), + HttpRouter.add("GET", "/github/mcp-moved", mcpGet("github")), // As GitHub's MCP server: its authorization server is the GitHub client's, under an issuer // path (GitHub's is `https://github.com/login/oauth`), and it advertises scopes a GitHub App, // which has none, never asks for. @@ -2484,7 +2488,9 @@ const firstPartyService = (port: number) => * calls; the rest have not. * `beyond` was granted a scope the Google client does not allow, `elsewhere` and `mcp-elsewhere` * call a host the GitHub client's tokens never reach, and `notion` signed in through an app v2 - * does not offer. + * does not offer. `wrong-server` signed in with the GitHub App to Slack's MCP server, whose + * authorization server is not GitHub's, and `moved` to an MCP server whose resource metadata names + * another resource, so neither sign-in can be carried over. * `both` and `apionly` use APIs on two hosts of one service, like GitHub's API and upload hosts: * `both`'s client sends its tokens to both, and `apionly`'s to the API host only. */ @@ -2501,6 +2507,8 @@ const seedFirstParty = (organization: string, user: string, slug: string, origin apiOnly: `apionly_${suffix}`, githubMcp: `github_mcp_${suffix}`, mcpElsewhere: `mcp_elsewhere_${suffix}`, + wrongServer: `wrong_server_${suffix}`, + moved: `moved_${suffix}`, }; const githubHosts = v1GitHubHosts(origin); const userScope = `executor/user/${encodeURIComponent(organization)}/${encodeURIComponent(user)}`; @@ -2539,6 +2547,8 @@ const seedFirstParty = (organization: string, user: string, slug: string, origin }, { name: "github-mcp", integration: slugs.githubMcp, client: "v1-github", scope: "" }, { name: "mcp-elsewhere", integration: slugs.mcpElsewhere, client: "v1-github", scope: "" }, + { name: "wrong-server", integration: slugs.wrongServer, client: "v1-github", scope: "" }, + { name: "moved", integration: slugs.moved, client: "v1-github", scope: "" }, ]; for (const connection of connections) { const item = access(connection.integration, connection.name); @@ -2711,6 +2721,18 @@ const seedFirstParty = (organization: string, user: string, slug: string, origin connection: "mcp-elsewhere", endpoint: `${origin.replace("127.0.0.1", "localhost")}/github/mcp`, }, + { + slug: slugs.wrongServer, + name: "GitHub MCP on Slack's server", + connection: "wrong-server", + endpoint: `${origin}/slack/mcp`, + }, + { + slug: slugs.moved, + name: "GitHub MCP moved", + connection: "moved", + endpoint: `${origin}/github/mcp-moved`, + }, ].flatMap((server) => [ row( "insert into integration (tenant, slug, plugin_id, name, config) values ($1, $2, 'mcp', $3, $4::json)", @@ -4246,18 +4268,66 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import", (it) => { yield* seedFirstParty(own.organization, own.user, `v1-${randomUUID().slice(0, 8)}`, origin); yield* googleSignIn(identity.email); - const { target: organization, results } = yield* importOnServer(own.organization); - // v2 does not offer Notion's app, and the Google client allows no scope beyond its menu. - expect(results.get("notion")).toEqual({ - status: "failed", - reason: - "It signed in through Executor's own Notion app, which Executor v2 does not offer yet. Reconnect it in v2.", + // The dry run refuses every connection whose account the import would refuse, with the + // reason the import gives, so the organization is blocked before it is ever frozen. + const importer = yield* v1ImportOperator; + const dry = v1RunId("first-party-dry"); + const started = yield* importer.start({ + mode: "dry-run", + runId: dry, + organizations: [own.organization], }); - expect(results.get("beyond")).toEqual({ - status: "failed", - reason: + expect(started.status, JSON.stringify(started.body)).toBe(200); + const planned = yield* importer.waitForReport(dry, own.organization); + expect(planned.status).toBe("blocked"); + expect(planned.integrationFailures).toEqual([]); + expect(planned.vaultFailures).toEqual([]); + expect(planned.staleItems).toEqual([]); + const predicted = Object.fromEntries( + planned.connectionFailures.flatMap((failure) => + failure.connections.map((connection) => [connection.label, failure.reason] as const), + ), + ); + const githubHosts = v1GitHubHosts(origin); + const elsewhere = `Executor v2's own V1-github app sends its tokens only to ${new URL(origin).host}, and its integration calls none of them. Reconnect it in v2.`; + expect(predicted).toEqual({ + notion: + "It signed in through Executor's own Notion app, which Executor v2 does not offer yet. Reconnect it in v2.", + beyond: "It was granted permissions Executor v2's own V1-google app does not allow. Reconnect it in v2.", + // The GitHub client's tokens go only to its pinned host, which neither this API nor + // this MCP server is on. + elsewhere, + "mcp-elsewhere": elsewhere, + // A client whose tokens reach only one of an API's two hosts would leave the other's + // calls refused, so the account is not imported, and the reason names that host. + apionly: `Executor v2's own V1-github-api app sends its tokens only to ${githubHosts.api}, and its integration also calls ${githubHosts.uploads}, where its calls would be refused. Reconnect it in v2.`, + // Slack's MCP server names Slack's authorization server, not the GitHub client's. + "wrong-server": + "Its integration does not sign in where Executor v2's own V1-github app does. Reconnect it in v2.", + // Its resource metadata names the resource at `/github/mcp`, not this server. + moved: "Its sign-in could not be carried over to v2. Reconnect it in v2.", }); + + const { target: organization, results, items } = yield* importOnServer(own.organization); + // The import refuses exactly the accounts its dry run did, for the same reasons, and + // imports every other connection. + const connections = items.filter((item) => item.kind === "connection"); + expect(connections).toHaveLength(12); + expect( + Object.fromEntries( + connections + .filter((item) => item.status !== "imported") + .map((item) => [item.label, resultOf(item)] as const), + ), + ).toEqual( + Object.fromEntries( + Object.entries(predicted).map(([label, reason]) => [ + label, + { status: "failed", reason }, + ]), + ), + ); for (const name of [ "Synthetic Google", "Synthetic GitHub", @@ -4267,6 +4337,8 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import", (it) => { "GitHub one host", "Synthetic GitHub MCP", "GitHub MCP elsewhere", + "GitHub MCP on Slack's server", + "GitHub MCP moved", "google", "github", "slack", @@ -4274,20 +4346,6 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import", (it) => { "github-mcp", ]) expect(results.get(name)?.status, results.get(name)?.reason).toBe("imported"); - // The GitHub client's tokens go only to its pinned host, which neither this API nor this - // MCP server is on. - for (const label of ["elsewhere", "mcp-elsewhere"]) - expect(results.get(label), label).toMatchObject({ - status: "failed", - reason: `Executor v2's own V1-github app sends its tokens only to ${new URL(origin).host}, and its integration calls none of them. Reconnect it in v2.`, - }); - // A client whose tokens reach only one of an API's two hosts would leave the other's calls - // refused, so the account is not imported, and the reason names the host it cannot reach. - const githubHosts = v1GitHubHosts(origin); - expect(results.get("apionly")).toMatchObject({ - status: "failed", - reason: `Executor v2's own V1-github-api app sends its tokens only to ${githubHosts.api}, and its integration also calls ${githubHosts.uploads}, where its calls would be refused. Reconnect it in v2.`, - }); const inventory = yield* call( "Read the new organization", diff --git a/packages/sdk/src/contracts/oauth.ts b/packages/sdk/src/contracts/oauth.ts index a30e0d6478..a5b6b64212 100644 --- a/packages/sdk/src/contracts/oauth.ts +++ b/packages/sdk/src/contracts/oauth.ts @@ -2,7 +2,12 @@ import { UserFacingError, type ErrorPresentation } from "@executor-js/utils/user import { ApiError } from "@executor-js/utils/api-error"; /** Host-owned OAuth configuration and encrypted protocol records. */ import type { UrlPolicy } from "@executor-js/utils/url-policy"; -import { AuthMethodName, type AuthMethodInvalid, type ProviderNotFound } from "./provider.ts"; +import { + AuthMethodName, + type AuthMethodInvalid, + type ProviderDefinition, + type ProviderNotFound, +} from "./provider.ts"; import { AccountConnectionId, type CredentialsError, type StorageError } from "./shared.ts"; import { Schema, type Effect, type Redacted } from "effect"; import { @@ -14,7 +19,7 @@ import { OAuthTokenResponse, PlacementHeaderName, } from "apps/contracts"; -import { Account } from "./account.ts"; +import { Account, type AccountFieldsInvalid } from "./account.ts"; export { OAuthClientAuth } from "apps/contracts"; import type { HttpClient } from "effect/http"; import { AccountId, HttpUrl, JsonObject, OwnerId, ProviderId } from "./shared.ts"; @@ -1516,6 +1521,24 @@ export interface OAuthGrantImport { | OAuthSetupFailed | OAuthGrantNotImportable >; + /** + * Check the grant as `importOAuthGrant` would, against `definition`, the provider an app that is + * not deployed declares, and save nothing: the v1 import's dry run refuses exactly the grants an + * import would. Runs the provider's discovery, but no token request. + */ + readonly checkOAuthGrant: ( + input: ImportOAuthGrant & { readonly definition: ProviderDefinition }, + ) => Effect.Effect; + /** + * Check an account's fields as `managedAccounts.add` would, against `definition`, the provider an + * app that is not deployed declares, and save nothing. + */ + readonly checkAccountFields: (input: { + readonly provider: ProviderId; + readonly definition: ProviderDefinition; + readonly method: string; + readonly fields: Redacted.Redacted; + }) => Effect.Effect; /** The operator's client `id` when it is offered here, which its credentials being set decides. */ readonly firstPartyClient: (id: string) => FirstPartyOAuthClientDescription | undefined; /** diff --git a/packages/sdk/src/implementation/create.ts b/packages/sdk/src/implementation/create.ts index 6d6d705468..5d0cc1f8c8 100644 --- a/packages/sdk/src/implementation/create.ts +++ b/packages/sdk/src/implementation/create.ts @@ -33,6 +33,7 @@ import { makeSchedules } from "./schedules.ts"; import { makeTools } from "./tools.ts"; import { makeSkills } from "./skills.ts"; import { toEffectRuntime } from "./runtime.ts"; +import { validateFields } from "./provider.ts"; import { database } from "./database.ts"; import { makeOAuth } from "./oauth.ts"; import { makeDeclarationCache, makeDeclarations } from "./declarations.ts"; @@ -222,6 +223,11 @@ export const createExecutor = ( }, [OAuthGrantImport]: { importOAuthGrant: oauth.importGrant, + checkOAuthGrant: oauth.checkGrant, + checkAccountFields: (input) => + validateFields(input.provider, input.definition, input.method, input.fields).pipe( + Effect.asVoid, + ), firstPartyClient: oauth.firstPartyClient, credentialsReadable: credentialsReadable(db, credentials), }, diff --git a/packages/sdk/src/implementation/oauth.ts b/packages/sdk/src/implementation/oauth.ts index c7fdded551..4f34916f7b 100644 --- a/packages/sdk/src/implementation/oauth.ts +++ b/packages/sdk/src/implementation/oauth.ts @@ -755,17 +755,30 @@ export const makeOAuth = ( Effect.gen(function* () { if (protocol === undefined || options === undefined) return yield* new OAuthSetupFailed({ reason: "unsupported" }); - const row = yield* query(() => - db.findFirst("providers", { where: (b) => b("id", "=", input.provider) }), - ); - if (row === null) return yield* new ProviderNotFound({ provider: input.provider }); - const provider = yield* decode(Provider, row); + return yield* setupOf(yield* storedProvider(input.provider), input); + }); + + const storedProvider = (id: ProviderId) => + Effect.gen(function* () { + const row = yield* query(() => db.findFirst("providers", { where: (b) => b("id", "=", id) })); + if (row === null) return yield* new ProviderNotFound({ provider: id }); + return yield* decode(Provider, row); + }); + + /** `resolveSetup` for a provider already read, or one an undeployed app declares. */ + const setupOf = ( + provider: Provider, + input: { readonly method: string; readonly redirectUri?: string | undefined }, + ) => + Effect.gen(function* () { + if (protocol === undefined || options === undefined) + return yield* new OAuthSetupFailed({ reason: "unsupported" }); yield* Effect.annotateCurrentSpan("oauth.provider.id", provider.id); const method = Object.hasOwn(provider.definition.auth, input.method) ? provider.definition.auth[input.method] : undefined; if (method === undefined || method.type !== "oauth2") - return yield* new AuthMethodInvalid({ provider: input.provider, method: input.method }); + return yield* new AuthMethodInvalid({ provider: provider.id, method: input.method }); const redirect = method.grant === "client_credentials" || input.redirectUri === undefined ? undefined @@ -3045,20 +3058,15 @@ export const makeOAuth = ( }); /** - * Save an account whose grant another product issued (`OAuthGrantImport`). A grant to a client - * the product registered keeps that client and renews as that product did - * (`ImportOAuthGrant.renewal`), whatever the provider's method declares; an authorization-code - * client also becomes a stored client of the owner, as an entered client a sign-in keeps does, - * so a reconnect can name it. A grant to one of the operator's clients is saved as a sign-in - * with it saves one: it keeps only the client's ID, and no client is stored for the owner. + * Everything an imported grant is checked against before anything is saved, with the provider + * its account's app declares: the method and callback a sign-in would use, the provider's + * discovery, the operator's client for a grant it issued, the token response's fields, the token + * endpoint renewals use and the grant itself. `importGrant` saves what this returns, and + * `checkGrant` stops here, so a check refuses exactly what an import would. */ - const importGrant = (input: ImportOAuthGrant) => + const preparedGrant = (provider: Provider, input: ImportOAuthGrant) => Effect.gen(function* () { - const { method, discovered, issuer } = yield* resolveSetup({ - provider: input.provider, - method: input.method, - redirectUri: input.redirectUri, - }); + const { method, discovered, issuer } = yield* setupOf(provider, input); const { renewal } = input; const firstParty = "firstParty" in input @@ -3074,16 +3082,6 @@ export const makeOAuth = ( ? {} : { refreshToken: Redacted.value(input.refreshToken) }; const scopes = renewal.scopes.length === 0 ? {} : { scopes: renewal.scopes }; - const createdAt = yield* Clock.currentTimeMillis; - const account = yield* decode(Account, { - id: AccountId.make(`acc_${yield* nextId}`), - provider: input.provider, - owner: input.owner, - label: input.label, - description: null, - method: input.method, - createdAt: new Date(createdAt), - }); const registered = "client" in input ? yield* Effect.gen(function* () { @@ -3099,77 +3097,120 @@ export const makeOAuth = ( const server = { ...discovered.server, token_endpoint: tokenEndpoint.href }; const resource = input.renewal.resource === undefined ? {} : { resource: input.renewal.resource }; - const stored = - discovered.grant === "client_credentials" - ? undefined - : yield* Effect.gen(function* () { - const id = OAuthClientId.make(`oac_${yield* nextId}`); - const details = registrationCredentials(client); - return { - row: { - id, - owner: input.owner, - provider: null, - issuer, - clientId: client.client_id, - source: "manual" as const, - hasSecret: client.token_endpoint_auth_method !== "none", - automaticKey: null, - confirmed: true, - createdAt: new Date(createdAt), - encrypted: yield* encrypt(id, details), - }, - digest: yield* digestOf({ owner: input.owner, issuer }, details), - }; - }); - const grantOf = (oauthClient: OAuthClientId | undefined) => - discovered.grant === "client_credentials" - ? { - ...Struct.omit(discovered, ["resource"]), - server, - scopes: input.renewal.scopes, - ...resource, - client, - fields, - response: method.response, - generation: 0, - ...expires, - } - : { - server, - client, - ...(oauthClient === undefined ? {} : { oauthClient }), - response: method.response, - ...scopes, - ...resource, - ...(discovered.tokenRequestFormat === undefined - ? {} - : { tokenRequestFormat: discovered.tokenRequestFormat }), - ...(discovered.tokenResponse === undefined - ? {} - : { tokenResponse: discovered.tokenResponse }), - fields, - ...refresh, - generation: 0, - ...expires, - }; - return { stored, grantOf }; + return { client, server, resource }; }) : undefined; - const encryptGrant = (grant: unknown) => - decode(OAuthGrant, grant).pipe( + /** The grant, naming the owner's stored client a sign-in with it would keep. */ + const grantOf = (oauthClient: OAuthClientId | undefined) => + registered === undefined + ? { + grant: "authorization_code", + firstParty: firstParty?.id, + response: method.response, + fields, + ...refresh, + ...scopes, + generation: 0, + ...expires, + } + : discovered.grant === "client_credentials" + ? { + ...Struct.omit(discovered, ["resource"]), + server: registered.server, + scopes: renewal.scopes, + ...registered.resource, + client: registered.client, + fields, + response: method.response, + generation: 0, + ...expires, + } + : { + server: registered.server, + client: registered.client, + ...(oauthClient === undefined ? {} : { oauthClient }), + response: method.response, + ...scopes, + ...registered.resource, + ...(discovered.tokenRequestFormat === undefined + ? {} + : { tokenRequestFormat: discovered.tokenRequestFormat }), + ...(discovered.tokenResponse === undefined + ? {} + : { tokenResponse: discovered.tokenResponse }), + fields, + ...refresh, + generation: 0, + ...expires, + }; + const grantOrRefused = (oauthClient: OAuthClientId | undefined) => + decode(OAuthGrant, grantOf(oauthClient)).pipe( Effect.mapError(() => new OAuthSetupFailed({ reason: "invalid_client" })), - Effect.flatMap((decoded) => encrypt(account.id, decoded)), ); + // A stored client's ID changes nothing the schema checks, so a grant that decodes without + // one decodes with it. + yield* grantOrRefused(undefined); + return { discovered, issuer, fields, registered, grantOrRefused }; + }); + + /** + * Save an account whose grant another product issued (`OAuthGrantImport`). A grant to a client + * the product registered keeps that client and renews as that product did + * (`ImportOAuthGrant.renewal`), whatever the provider's method declares; an authorization-code + * client also becomes a stored client of the owner, as an entered client a sign-in keeps does, + * so a reconnect can name it. A grant to one of the operator's clients is saved as a sign-in + * with it saves one: it keeps only the client's ID, and no client is stored for the owner. + */ + const importGrant = (input: ImportOAuthGrant) => + Effect.gen(function* () { + if (protocol === undefined || options === undefined) + return yield* new OAuthSetupFailed({ reason: "unsupported" }); + const { discovered, issuer, fields, registered, grantOrRefused } = yield* preparedGrant( + yield* storedProvider(input.provider), + input, + ); + const createdAt = yield* Clock.currentTimeMillis; + const account = yield* decode(Account, { + id: AccountId.make(`acc_${yield* nextId}`), + provider: input.provider, + owner: input.owner, + label: input.label, + description: null, + method: input.method, + createdAt: new Date(createdAt), + }); + const stored = + registered === undefined || discovered.grant === "client_credentials" + ? undefined + : yield* Effect.gen(function* () { + const id = OAuthClientId.make(`oac_${yield* nextId}`); + const details = registrationCredentials(registered.client); + return { + row: { + id, + owner: input.owner, + provider: null, + issuer, + clientId: registered.client.client_id, + source: "manual" as const, + hasSecret: registered.client.token_endpoint_auth_method !== "none", + automaticKey: null, + confirmed: true, + createdAt: new Date(createdAt), + encrypted: yield* encrypt(id, details), + }, + digest: yield* digestOf({ owner: input.owner, issuer }, details), + }; + }); const encryptedCredentials = yield* encrypt(account.id, fields); const ready = `ready_${yield* nextId}`; yield* transaction(db, (tx) => Effect.gen(function* () { // Credentials another stored client holds are that client; nothing is stored twice. const saved = - registered?.stored === undefined + stored === undefined ? undefined - : yield* createStoredClient(tx, registered.stored.row, registered.stored.digest); + : yield* createStoredClient(tx, stored.row, stored.digest); const oauthClient = saved === undefined ? undefined : saved.stored ? saved.client.id : saved.holder; yield* query(() => @@ -3177,20 +3218,7 @@ export const makeOAuth = ( ); if (lifecycle) yield* lifecycle.accountCreated(account); const state = { - encrypted: yield* encryptGrant( - registered === undefined - ? { - grant: "authorization_code", - firstParty: firstParty?.id, - response: method.response, - fields, - ...refresh, - ...scopes, - generation: 0, - ...expires, - } - : registered.grantOf(oauthClient), - ), + encrypted: yield* encrypt(account.id, yield* grantOrRefused(oauthClient)), status: ready, updatedAt: new Date(createdAt), }; @@ -3206,11 +3234,22 @@ export const makeOAuth = ( return account; }).pipe(Effect.withSpan("oauth.importGrant")); + /** + * Check an imported grant as `importGrant` would, against the provider an app that is not + * deployed declares, and save nothing (`OAuthGrantImport.checkOAuthGrant`). + */ + const checkGrant = (input: ImportOAuthGrant & { readonly definition: ProviderDefinition }) => + preparedGrant({ id: input.provider, definition: input.definition }, input).pipe( + Effect.asVoid, + Effect.withSpan("oauth.checkGrant"), + ); + return { revocation, clients, connections: { oauthSetup, startOAuth, completeOAuth }, importGrant, + checkGrant, /** The operator's client `id` as the importer checks a grant against it, without its secret. */ firstPartyClient: (id: string) => { const client = [...firstPartyClients.values()].find((candidate) => candidate.id === id); From d42a326c1de3f604e522989768a2d252a3f201aa Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:29:33 +0000 Subject: [PATCH 016/127] Export 7a7b3731 --- apps/hosted/cloud/alchemy.github.ts | 19 ---- .../cloud/scripts/check-v1-import-access.ts | 58 ++++------- apps/hosted/cloud/scripts/v1-import.ts | 7 +- .../cloud/src/implementation/migrations.ts | 17 ++++ .../cloud/src/implementation/v1-import/run.ts | 17 +++- .../src/implementation/v1-import/writes.ts | 79 +++++++++++---- .../cloud/src/infrastructure/onboarding.ts | 20 ++-- .../src/infrastructure/v1-import-settings.ts | 40 +------- .../src/infrastructure/v1-import-workflow.ts | 13 +-- apps/hosted/cloud/src/main.ts | 5 +- e2e/support/v1-import.ts | 15 +++ e2e/test-plan.ts | 6 ++ e2e/tests/cloud-v1-import-run.spec.ts | 99 ++++++++++++++++++- e2e/tests/cloud-v1-import.spec.ts | 3 +- e2e/tests/cloud-v1-session-handoff.spec.ts | 3 +- 15 files changed, 259 insertions(+), 142 deletions(-) diff --git a/apps/hosted/cloud/alchemy.github.ts b/apps/hosted/cloud/alchemy.github.ts index 5028b18865..447ae678a3 100644 --- a/apps/hosted/cloud/alchemy.github.ts +++ b/apps/hosted/cloud/alchemy.github.ts @@ -24,9 +24,6 @@ import { V1FirstPartyClients, invalidV1FirstPartyClients, missingV1FirstPartyClient, - invalidV1ImportAllowedOrgs, - parseV1ImportAllowedOrgs, - v1ImportAllowedOrgsName, missingV1ImportWriter, missingV1McpForwardSecret, v1ImportWriterName, @@ -328,22 +325,6 @@ export default Alchemy.Stack( value: v1ImportWrites, environment: production, }).pipe(retain()); - /** - * The v1 organizations production may import (`v1-import-settings.ts`): with writes on, an - * import names its organizations and each must be on this list. `.env.ci.op` sets it; unset - * or empty imports none, and stores nothing, as GitHub refuses an empty variable. It is not - * retained: emptying the list removes the variable, so production imports none again. - */ - const allowedOrgs = yield* Config.String(v1ImportAllowedOrgsName).pipe(Config.withDefault("")); - if (parseV1ImportAllowedOrgs(allowedOrgs) === undefined) - return yield* Effect.die(new Error(invalidV1ImportAllowedOrgs)); - if (allowedOrgs.trim() !== "") - yield* GitHub.Variable(`production-${v1ImportAllowedOrgsName}`, { - ...target, - name: v1ImportAllowedOrgsName, - value: allowedOrgs, - environment: production, - }); /** * The URL of v1's `v1_import_writer` role (`moved-flag.ts`), which freezes and cuts over each * imported organization. Imports that write need it, so this stack refuses writes without it. diff --git a/apps/hosted/cloud/scripts/check-v1-import-access.ts b/apps/hosted/cloud/scripts/check-v1-import-access.ts index c5e9affe73..bdb49ef94f 100644 --- a/apps/hosted/cloud/scripts/check-v1-import-access.ts +++ b/apps/hosted/cloud/scripts/check-v1-import-access.ts @@ -10,19 +10,15 @@ * with writes allowed; * - production's source runs on any other stage; * - the emulated Cloud the E2E runner uses stops allowing imports that write; - * - an import over v1's real data runs without naming its organizations, or for one the allow-list - * (`CLOUD_V1_IMPORT_ALLOWED_ORGS`) does not list, or the allow-list reads wrongly. + * - an import runs without naming its organizations, or for one without a row in the allow table + * (`private_v1_import_allowed`), given the rows read from it. */ import { Stage } from "alchemy/Stage"; -import { ConfigProvider, Effect, Exit, Schema } from "effect"; -import { V1OrganizationId } from "../src/contracts/v1-import.ts"; +import { ConfigProvider, Effect, Exit } from "effect"; import type { V1MovedFlag } from "../src/implementation/v1-import/moved-flag.ts"; -import { V1ImportWrites, importRefusal } from "../src/implementation/v1-import/writes.ts"; +import { V1ImportWrites, refusalGiven } from "../src/implementation/v1-import/writes.ts"; import { productionStage } from "../src/infrastructure/stage.ts"; -import { - parseV1ImportAllowedOrgs, - v1ImportAccess, -} from "../src/infrastructure/v1-import-settings.ts"; +import { v1ImportAccess } from "../src/infrastructure/v1-import-settings.ts"; const stages = [productionStage, "dev", "dev-checkout", "staging", "test-e2e-check", "test-pr-1"]; const loopbackOrigins = ["http://localhost:4411", "http://127.0.0.1:4411", "http://[::1]:4411"]; @@ -119,48 +115,32 @@ const e2e = evaluate({ if (!Exit.isSuccess(e2e) || e2e.value.source !== "emulated" || !e2e.value.writes) failures.push("Emulated Cloud no longer allows imports that write."); -// Which organizations an import may write. The flag is never written here. +// Which organizations an import may write, given the allow table's rows. The flag is never +// written here. const flag: V1MovedFlag = { set: () => Effect.void }; -const id = Schema.decodeUnknownSync(V1OrganizationId); -const [listedOrg, otherOrg] = [id("org_listed1"), id("org_other1")]; -const listed = V1ImportWrites.Listed({ organizations: new Set([listedOrg]), flag }); +const [listedOrg, otherOrg] = ["org_listed1", "org_other1"]; +const listed = V1ImportWrites.Listed({ flag }); +const rows = new Set([listedOrg]); const writeRules: ReadonlyArray = [ - ["a real import that pages through v1", importRefusal(listed, undefined), false], - ["a real import that names no organization", importRefusal(listed, []), false], - ["a real import of a listed organization", importRefusal(listed, [listedOrg]), true], - ["a real import of an unlisted organization", importRefusal(listed, [otherOrg]), false], - ["a real import that adds an unlisted one", importRefusal(listed, [listedOrg, otherOrg]), false], + ["an import that pages through v1", refusalGiven(listed, undefined, rows), false], + ["an import that names no organization", refusalGiven(listed, [], rows), false], + ["an import of an organization with a row", refusalGiven(listed, [listedOrg], rows), true], + ["an import of an organization without a row", refusalGiven(listed, [otherOrg], rows), false], [ - "a real import with an empty allow-list", - importRefusal(V1ImportWrites.Listed({ organizations: new Set(), flag }), [listedOrg]), + "an import that adds an organization without a row", + refusalGiven(listed, [listedOrg, otherOrg], rows), false, ], - ["an emulated import that pages", importRefusal(V1ImportWrites.Any({ flag }), undefined), true], + ["an import with an empty allow table", refusalGiven(listed, [listedOrg], new Set()), false], [ "a deployment that refuses imports", - importRefusal(V1ImportWrites.Refused({ reason: "off" }), [listedOrg]), + refusalGiven(V1ImportWrites.Refused({ reason: "off" }), [listedOrg], rows), false, ], ]; for (const [rule, refusal, allowed] of writeRules) if ((refusal === undefined) !== allowed) failures.push(`${rule} is ${refusal === undefined ? "allowed" : "refused"}`); -const lists: ReadonlyArray = [ - ["", []], - [" ", []], - ["org_a1", ["org_a1"]], - [" org_a1 , org_b2,", ["org_a1", "org_b2"]], - ["org_a1,not-an-org", undefined], - ["org_a1;org_b2", undefined], -]; -for (const [value, expected] of lists) { - const parsed = parseV1ImportAllowedOrgs(value); - const got = parsed === undefined ? undefined : [...parsed].toSorted(); - if (JSON.stringify(got) !== JSON.stringify(expected)) - failures.push( - `CLOUD_V1_IMPORT_ALLOWED_ORGS=${JSON.stringify(value)} reads as ${JSON.stringify(got)}`, - ); -} if (failures.length > 0) { console.error(`v1 import access rules failed in ${failures.length} combinations:`); @@ -168,5 +148,5 @@ if (failures.length > 0) { process.exit(1); } console.log( - `v1 import access: ${cases.length} combinations checked, ${live} run the live source; ${writeRules.length} write rules and ${lists.length} allow-lists checked.`, + `v1 import access: ${cases.length} combinations checked, ${live} run the live source; ${writeRules.length} write rules checked.`, ); diff --git a/apps/hosted/cloud/scripts/v1-import.ts b/apps/hosted/cloud/scripts/v1-import.ts index 2f0c702167..acaa6a7db3 100644 --- a/apps/hosted/cloud/scripts/v1-import.ts +++ b/apps/hosted/cloud/scripts/v1-import.ts @@ -15,8 +15,9 @@ * Raise the bound only while it stays rare. Dry runs write only their reports. * `--mode import` freezes each organization in v1, moves it and cuts it over. It * needs the deployment's CLOUD_V1_IMPORT_WRITES=on and the organizations named with - * --org; production imports only those on its CLOUD_V1_IMPORT_ALLOWED_ORGS and - * refuses the rest with the reason. + * --org; the deployment imports only those with a row in its + * private_v1_import_allowed table and refuses the rest with the reason. This + * command never writes that table: rows are added by direct SQL. * aggregate Combine a run's reports, a page per request, into * v1-import/runs//aggregate.json and print it. Repeat it until `complete` is * true. @@ -74,7 +75,7 @@ const program = Effect.gen(function* () { HttpClientRequest.bearerToken(Redacted.value(token)), ), ), - // A refused request names why (`{ error }`), such as an organization off the allow-list. + // A refused request names why (`{ error }`), such as an organization without a row in the allow table. HttpClient.transformResponse( Effect.flatMap((response) => response.status >= 200 && response.status < 300 diff --git a/apps/hosted/cloud/src/implementation/migrations.ts b/apps/hosted/cloud/src/implementation/migrations.ts index 7a860477ce..eeb1a70e3b 100644 --- a/apps/hosted/cloud/src/implementation/migrations.ts +++ b/apps/hosted/cloud/src/implementation/migrations.ts @@ -179,6 +179,22 @@ export const migrateV1SessionHandoffs = Effect.gen(function* () { on private_v1_session_handoff (expires_at)`; }).pipe(Effect.mapError(migrationFailed("product"))); +/** + * The v1 organizations an import may write (`notes/v1-import.md`, `v1-import/writes.ts`): an import + * of an organization without a row is refused. A guard apart from the operator token, which alone + * allows nothing. No route, operator command or app path writes it; rows are added only by direct + * SQL from someone with access to the database. It starts empty and copies nothing, so until rows + * are inserted every import is refused. Additive: the running server never reads it. + */ +export const migrateV1ImportAllowed = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`create table if not exists private_v1_import_allowed ( + v1_org_id text primary key, + added_at timestamptz not null default now(), + note text + )`; +}).pipe(Effect.mapError(migrationFailed("product"))); + /** * The onboarding experiment's assignments (`notes/posthog.md#onboarding-order-experiment`): one * row per member who created their first team through team setup and was assigned a variant, @@ -219,6 +235,7 @@ export const migrateCloudDatabase = Effect.scoped( "6_setup_assignments": migrateSetupAssignments, "7_v1_import_orgs": migrateV1ImportOrgs, "8_v1_session_handoffs": migrateV1SessionHandoffs, + "9_v1_import_allowed": migrateV1ImportAllowed, }); }); yield* migrateHostedDatabase(setup.options, cloudMigrations).pipe( diff --git a/apps/hosted/cloud/src/implementation/v1-import/run.ts b/apps/hosted/cloud/src/implementation/v1-import/run.ts index a579d6ebeb..4dbd9866e1 100644 --- a/apps/hosted/cloud/src/implementation/v1-import/run.ts +++ b/apps/hosted/cloud/src/implementation/v1-import/run.ts @@ -142,7 +142,7 @@ export class V1ImportEnvironment extends Context.Service< readonly reports: V1ImportReports; /** * Which organizations this deployment may import, writing v2 (`writes.ts`); dry runs never - * write. The operator routes check it before starting, and the run again. + * write. The operator routes check it and the allow table before starting, and the run again. */ readonly writes: V1ImportWrites; /** @@ -1976,8 +1976,8 @@ export const reportPeopleUnread = "Its members' emails could not be read from v1's WorkOS, so the report leaves out its email matches and the owner its API keys act as."; /** - * Move or check one organization. Import mode needs the deployment's write switch and, on v1's - * real data, the organization on the allow-list (`writes.ts`), and holds back what a dry run only + * Move or check one organization. Import mode needs the deployment's write switch and the + * organization's row in `private_v1_import_allowed` (`writes.ts`), and holds back what a dry run only * reports. It follows "Cutover per organization": it freezes the organization in v1 before reading * anything it copies, imports it, verifies it, and only then marks it moved, so v1 forwards it. An * organization it copies nothing from is unfrozen again; one that fails after copying stays frozen @@ -1994,8 +1994,17 @@ export const runV1Import = ( ) => Effect.gen(function* () { const environment = yield* V1ImportEnvironment; + // Checked again in its own step, so the allow table is read once per instance, not on replay. const refusal = - params.mode === "import" ? importRefusal(environment.writes, [params.v1OrgId]) : undefined; + params.mode === "import" + ? yield* step("allowed", importRefusal(environment.writes, [params.v1OrgId])).pipe( + Effect.catchTag("V1ImportStepFailed", (error) => + Effect.succeed( + `Its row in the allow table could not be read: v2's database kept failing (${error.failure}).`, + ), + ), + ) + : undefined; if (refusal !== undefined) return yield* Effect.die(new Error(refusal)); const organization = params.v1OrgId; diff --git a/apps/hosted/cloud/src/implementation/v1-import/writes.ts b/apps/hosted/cloud/src/implementation/v1-import/writes.ts index b2c916b646..9f535ab4d7 100644 --- a/apps/hosted/cloud/src/implementation/v1-import/writes.ts +++ b/apps/hosted/cloud/src/implementation/v1-import/writes.ts @@ -3,54 +3,97 @@ * runs"). Dry runs are never limited by it: they write nothing but their report. * * - `Refused`: no import writes here, and `reason` says which setting stops it. - * - `Any`: emulated Cloud's v1, whose organizations are all synthetic. - * - `Listed`: v1's real data on production. Only the organizations on the allow-list - * (`CLOUD_V1_IMPORT_ALLOWED_ORGS`) move, each named explicitly, never by paging through v1. + * - `Listed`: an import names each organization, never pages through v1, and only those with a + * row in `private_v1_import_allowed` move. Production and emulated Cloud both check the table; + * with no rows, nothing is imported. + * + * The table is a guard apart from the operator token: holding the token is not enough to allow an + * organization. No route, operator command or app path writes it. Rows are added only by direct + * SQL from someone with access to the database; this code only reads it. * * Every import freezes its organization in v1 first and cuts it over at the end, so a deployment * that writes always holds the moved flag's writer (`flag`). */ -import { Data } from "effect"; -import type { V1OrganizationId } from "../../contracts/v1-import.ts"; +import { Data, Effect, Schema } from "effect"; +import { V1ImportDatabase, V1ImportUnavailable } from "../../contracts/v1-import.ts"; import type { V1MovedFlag } from "./moved-flag.ts"; export type V1ImportWrites = Data.TaggedEnum<{ Refused: { readonly reason: string }; - Any: { readonly flag: V1MovedFlag }; - Listed: { readonly organizations: ReadonlySet; readonly flag: V1MovedFlag }; + Listed: { readonly flag: V1MovedFlag }; }>; export const V1ImportWrites = Data.taggedEnum(); -/** Why a real v1 import that does not name its organizations is refused. */ +/** The table of v1 organizations an import may write; see this module's comment. */ +export const v1ImportAllowedTable = "private_v1_import_allowed"; + +/** Why an import that does not name its organizations is refused. */ export const v1ImportMustName = - "An import over v1's real data names its organizations with `organizations`; it never pages through v1 with `after` or `limit`."; + "An import names its organizations with `organizations`; it never pages through v1 with `after` or `limit`."; -/** Why an organization missing from the allow-list is refused. */ +/** Why an organization without a row in the allow table is refused. */ export const v1ImportNotAllowed = (organizations: readonly string[]) => - `CLOUD_V1_IMPORT_ALLOWED_ORGS does not list ${organizations.join(", ")}, so it is not imported.`; + `${v1ImportAllowedTable} has no row for ${organizations.join(", ")}, so it is not imported.`; /** - * Why a request to import is refused here, or undefined when it may run. `organizations` is what - * the request names, undefined when it pages through v1 instead. + * Why a request to import is refused, given which of its organizations have a row in the allow + * table, or undefined when it may run. `organizations` is what the request names, undefined when + * it pages through v1 instead. */ -export const importRefusal = ( +export const refusalGiven = ( writes: V1ImportWrites, - organizations: readonly V1OrganizationId[] | undefined, + organizations: readonly string[] | undefined, + allowed: ReadonlySet, ): string | undefined => V1ImportWrites.$match(writes, { Refused: ({ reason }) => reason, - Any: () => undefined, - Listed: ({ organizations: allowed }) => { + Listed: () => { if (organizations === undefined || organizations.length === 0) return v1ImportMustName; const missing = organizations.filter((organization) => !allowed.has(organization)); return missing.length === 0 ? undefined : v1ImportNotAllowed(missing); }, }); +const AllowedRows = Schema.Array(Schema.Struct({ v1_org_id: Schema.String })); + +/** Which of these organizations have a row in the allow table. Only reads it. */ +const allowedOf = (organizations: readonly string[]) => + Effect.gen(function* () { + const sql = yield* Effect.flatten(V1ImportDatabase); + const rows = yield* sql` + select v1_org_id from private_v1_import_allowed + where ${sql.in("v1_org_id", organizations)} + `.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(AllowedRows)), + Effect.mapError(() => new V1ImportUnavailable()), + ); + return new Set(rows.map((row) => row.v1_org_id)); + }); + +/** + * Why a request to import is refused here, or undefined when it may run, reading the allow table + * for the organizations it names (`refusalGiven`). + */ +export const importRefusal = ( + writes: V1ImportWrites, + organizations: readonly string[] | undefined, +): Effect.Effect => + writes._tag === "Listed" && organizations !== undefined && organizations.length > 0 + ? allowedOf(organizations).pipe( + Effect.map((allowed) => refusalGiven(writes, organizations, allowed)), + ) + : Effect.succeed(refusalGiven(writes, organizations, new Set())); + +/** Why imports never write here, or undefined where they may. */ +export const writesRefused = (writes: V1ImportWrites): string | undefined => + V1ImportWrites.$match(writes, { + Refused: ({ reason }) => reason, + Listed: () => undefined, + }); + /** The moved flag's writer, where imports write. */ export const movedFlag = (writes: V1ImportWrites): V1MovedFlag | undefined => V1ImportWrites.$match(writes, { Refused: () => undefined, - Any: ({ flag }) => flag, Listed: ({ flag }) => flag, }); diff --git a/apps/hosted/cloud/src/infrastructure/onboarding.ts b/apps/hosted/cloud/src/infrastructure/onboarding.ts index 3a3a7a4183..26b2dae11e 100644 --- a/apps/hosted/cloud/src/infrastructure/onboarding.ts +++ b/apps/hosted/cloud/src/infrastructure/onboarding.ts @@ -31,7 +31,6 @@ import { v1ImportAccess, missingV1ImportWriter, missingV1McpForwardSecret, - v1ImportAllowedOrgsSetting, v1ImportOperatorTokenMinLength, v1ImportWriterName, v1LiveSettings, @@ -327,7 +326,7 @@ export interface V1ImportSettings { readonly source: Effect.Effect; /** * Which organizations an operator may import, writing v2: none without - * `CLOUD_V1_IMPORT_WRITES`, and on v1's real data only those `CLOUD_V1_IMPORT_ALLOWED_ORGS` lists. + * `CLOUD_V1_IMPORT_WRITES`, and otherwise only those with a row in `private_v1_import_allowed`. */ readonly writes: V1ImportWrites; /** Whether the source is v1's real data: production's, or the live source of a local dry run. */ @@ -359,13 +358,13 @@ export interface V1ImportSettings { * * - Emulated Cloud with a v1 database (managed local Cloud) reads the emulated v1 WorkOS and its * Vault, the v1 database (blobs from its `blob` table, as v1 keeps them without R2) and Autumn, - * and allows imports that write. + * and allows imports that write, of the organizations with a row in `private_v1_import_allowed`. * - Production runs it only when `CLOUD_V1_IMPORT=on` (`v1-import-settings.ts`, off by default). * It then reads v1's WorkOS (`V1_WORKOS_API_KEY`, also the sign-in check's), v1's database * through the read-only role (`V1_DATABASE_URL`), the Autumn account v2 bills from (v1 billed * there too), and v1's blob bucket through the `V1_BLOBS` binding. It starts only dry runs - * unless `CLOUD_V1_IMPORT_WRITES=on` too, and then imports only the organizations - * `CLOUD_V1_IMPORT_ALLOWED_ORGS` lists. + * unless `CLOUD_V1_IMPORT_WRITES=on` too, and then imports only the organizations with a row + * in `private_v1_import_allowed`, which only direct SQL writes (`writes.ts`). * - Local Cloud dev with `CLOUD_V1_IMPORT_SOURCE=live` reads v1's production data read-only * (`cloudV1LiveSource`) and runs only dry runs. Any other stage or origin refuses the setting. * - Everywhere else, and on production while the switch is off, there is no importer. Another @@ -417,14 +416,9 @@ export const cloudV1ImportSettings = Effect.gen(function* () { }) : Option.isNone(writer) ? V1ImportWrites.Refused({ reason: missingV1ImportWriter }) - : access.source === "production" - ? V1ImportWrites.Listed({ - organizations: yield* v1ImportAllowedOrgsSetting, - flag: v1MovedFlagPostgres(writer.value), - }) - : access.source === "emulated" - ? V1ImportWrites.Any({ flag: v1MovedFlagPostgres(writer.value) }) - : V1ImportWrites.Refused({ reason: "This deployment has no v1 source that writes." }); + : access.source === "production" || access.source === "emulated" + ? V1ImportWrites.Listed({ flag: v1MovedFlagPostgres(writer.value) }) + : V1ImportWrites.Refused({ reason: "This deployment has no v1 source that writes." }); const secrets = yield* cloudSecrets.pipe(Effect.orDie); return Option.some({ source: source.value, diff --git a/apps/hosted/cloud/src/infrastructure/v1-import-settings.ts b/apps/hosted/cloud/src/infrastructure/v1-import-settings.ts index d6c56178f3..bc1cd4e12f 100644 --- a/apps/hosted/cloud/src/infrastructure/v1-import-settings.ts +++ b/apps/hosted/cloud/src/infrastructure/v1-import-settings.ts @@ -8,16 +8,15 @@ * binds none of it and answers every operator request 404. * - `CLOUD_V1_IMPORT_WRITES`: `on` also lets the operator start imports that write v2. `off`, the * default, refuses them, and requires `CLOUD_V1_IMPORT=on` to mean anything. - * - `CLOUD_V1_IMPORT_ALLOWED_ORGS`: the v1 organizations production may import, comma separated. - * With writes on, an import must name its organizations and each must be listed here. Empty, the - * default, imports none. + * - Which v1 organizations may be imported is no setting: with writes on, an import names its + * organizations and each needs a row in the database table `private_v1_import_allowed`, which + * only direct SQL writes (`writes.ts`). With no rows, none is imported. * - `CLOUD_V1_IMPORT_SOURCE`: `live` points local Cloud dev at v1's production data, read-only, * for an operator's dry run from their own machine. It runs only dry runs; `emulated`, the * default, keeps the emulated v1. */ import { isLoopbackHostname } from "@executor-js/utils/url-policy"; import { Config, Effect, Option, Schema } from "effect"; -import { V1OrganizationId } from "../contracts/v1-import.ts"; import { cloudOrigin, productionStage, stageName, testStage } from "./stage.ts"; export const V1ImportSwitch = Schema.Literals(["off", "on"]); @@ -34,37 +33,6 @@ export const v1ImportWritesSetting = Config.Literals( "CLOUD_V1_IMPORT_WRITES", ).pipe(Config.withDefault("off")); -export const v1ImportAllowedOrgsName = "CLOUD_V1_IMPORT_ALLOWED_ORGS"; -/** What to fix when the allow-list holds something other than v1 organization ids. */ -export const invalidV1ImportAllowedOrgs = `${v1ImportAllowedOrgsName} must be v1 organization ids (org_…), comma separated`; - -/** - * Parse the allow-list: v1 organization ids separated by commas, with optional spaces. Empty is - * none. Undefined when any entry is not a v1 organization id. - */ -export const parseV1ImportAllowedOrgs = ( - value: string, -): ReadonlySet | undefined => { - const entries = value - .split(",") - .map((entry) => entry.trim()) - .filter((entry) => entry !== ""); - const parsed = entries.map((entry) => Schema.decodeUnknownOption(V1OrganizationId)(entry)); - if (parsed.some(Option.isNone)) return undefined; - return new Set(parsed.flatMap((entry) => (Option.isSome(entry) ? [entry.value] : []))); -}; - -/** The allow-list this deployment imports from; unset is empty. Dies on an entry that is no id. */ -export const v1ImportAllowedOrgsSetting = Config.String(v1ImportAllowedOrgsName).pipe( - Config.withDefault(""), - Effect.flatMap((value) => { - const parsed = parseV1ImportAllowedOrgs(value); - return parsed === undefined - ? Effect.die(new Error(invalidV1ImportAllowedOrgs)) - : Effect.succeed(parsed); - }), -); - /** * The secret holding the URL of v1's `v1_import_writer` role, which may only insert and update * `org_moved_to_v2` (and read the columns its upsert compares). Imports that write need it: they @@ -175,7 +143,7 @@ export const missingV1FirstPartyClient = (client: string) => * - `none`: no importer. * * `realV1` is true for both real sources. The live source never writes; production writes only - * for the organizations on `CLOUD_V1_IMPORT_ALLOWED_ORGS`, each named by the operator. + * for the organizations with a row in `private_v1_import_allowed`, each named by the operator. * `scripts/check-v1-import-access.ts` checks these rules over every combination. */ export const v1ImportAccess = (emulated: boolean) => diff --git a/apps/hosted/cloud/src/infrastructure/v1-import-workflow.ts b/apps/hosted/cloud/src/infrastructure/v1-import-workflow.ts index 4f042960ab..9c6fd6607b 100644 --- a/apps/hosted/cloud/src/infrastructure/v1-import-workflow.ts +++ b/apps/hosted/cloud/src/infrastructure/v1-import-workflow.ts @@ -43,7 +43,7 @@ import { import { makeV1Source } from "../implementation/v1-import/source.ts"; import { V1MovedState } from "../implementation/v1-import/moved-flag.ts"; import { makeImportStore } from "../implementation/v1-import/store.ts"; -import { importRefusal, movedFlag } from "../implementation/v1-import/writes.ts"; +import { importRefusal, movedFlag, writesRefused } from "../implementation/v1-import/writes.ts"; import { AppBuilds } from "./blobs.ts"; import type { V1ImportSettings } from "./onboarding.ts"; @@ -254,8 +254,9 @@ const aggregationId = /^[0-9a-f]{32}$/u; * reads of v1's database and WorkOS and the builds: a start takes only as many as there is room * for, defers the rest, and reports how many are running and which of the run's stopped without * a report. An id that is not a v1 organization id, or one the run already has running, is - * skipped. An import needs the deployment's write switch; over v1's real data it must list its - * organizations, and only those on `CLOUD_V1_IMPORT_ALLOWED_ORGS` (`writes.ts`). + * skipped. An import needs the deployment's write switch, must list its organizations, and + * each needs a row in `private_v1_import_allowed`, which only direct SQL writes (`writes.ts`). + * The token alone allows no organization. * - `GET /api/internal/v1-import/runs/:run/orgs/:organization` returns one organization's report * and its instance's status. * - `POST /api/internal/v1-import/orgs/:organization/rollback` gives an organization back to v1 @@ -383,10 +384,10 @@ export const v1ImportRoutes = (workflow: Workflow) => ); if (Option.isNone(body)) return yield* json(400, { error: "invalid request" }); const input = body.value; - // A start that pages through v1 names no organizations, which only emulated v1 may import. + // A start that pages through v1 names no organizations, which no import may do. const refusal = input.mode === "import" - ? importRefusal( + ? yield* importRefusal( settings.writes, input.after === undefined && input.limit === undefined ? input.organizations @@ -597,7 +598,7 @@ export const v1ImportRoutes = (workflow: Workflow) => const flag = movedFlag(found.value.settings.writes); if (flag === undefined) return yield* json(409, { - error: importRefusal(found.value.settings.writes, undefined) ?? "unavailable", + error: writesRefused(found.value.settings.writes) ?? "unavailable", }); const decoded = Schema.decodeUnknownOption(V1OrganizationId)( (yield* HttpRouter.params).organization, diff --git a/apps/hosted/cloud/src/main.ts b/apps/hosted/cloud/src/main.ts index 0aca285d0c..f2ddc50963 100644 --- a/apps/hosted/cloud/src/main.ts +++ b/apps/hosted/cloud/src/main.ts @@ -539,7 +539,10 @@ export default Api.make( const routes = Layer.mergeAll( HttpRouter.add("POST", "/api/internal/app-domains/resume", appDomains.control("resume")), HttpRouter.add("POST", "/api/internal/app-domains/drain", appDomains.control("drain")), - HttpRouter.add("POST", "/api/internal/v1-import/start", v1Import.start), + // Starting an import reads the allow table (`writes.ts`) from the product database. + HttpRouter.add("POST", "/api/internal/v1-import/start", v1Import.start).pipe( + HttpRouter.provideRequest(executor), + ), HttpRouter.add( "GET", "/api/internal/v1-import/runs/:run/orgs/:organization", diff --git a/e2e/support/v1-import.ts b/e2e/support/v1-import.ts index 34ee6573f1..fce04279c3 100644 --- a/e2e/support/v1-import.ts +++ b/e2e/support/v1-import.ts @@ -5,6 +5,7 @@ import { randomUUID } from "node:crypto"; import { Effect, FileSystem, Schedule, Schema } from "effect"; import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; +import { writeRows } from "./cloud-rows.ts"; import { Target } from "./platform.ts"; const Operator = Schema.Struct({ token: Schema.String }); @@ -167,6 +168,20 @@ const AggregatePage = Schema.Union([ Schema.Struct({ cursor: Schema.Null, aggregate: Aggregate }), ]); +/** + * Allow these v1 organizations to be imported, as a person with database access does: a row each + * in Cloud's `private_v1_import_allowed`, which no route writes. An import of an organization + * without a row is refused. + */ +export const allowV1Import = (...organizations: readonly string[]) => + writeRows( + "sso-database.json", + organizations.map((organization) => ({ + sql: "insert into private_v1_import_allowed (v1_org_id, note) values ($1, 'E2E scenario') on conflict (v1_org_id) do nothing", + params: [organization], + })), + ).pipe(Effect.asVoid); + /** A run id no other scenario uses. */ export const v1RunId = (prefix: string) => `${prefix}-${randomUUID().slice(0, 8)}`; diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index e1b137b743..d6f1dc966f 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -7694,6 +7694,12 @@ export const scenarios = plan({ "Cloud v1 import moves a shared workspace on the server and cuts it over once it verifies: the oldest admin owns it, members keep their roles without a welcome email, org connections become shared accounts linked for everyone and members' their own, a member's approve rule is dropped while their block applies to everyone, every member gets their own copy of an org toolkit, and a former member's rows stay behind", targets: { ...cloudOnboarding, cloud: managedCloud }, }, + v1ImportAllowed: { + file: "cloud-v1-import-run.spec.ts", + title: + "Cloud v1 import writes only organizations with a row in its allow table, refuses one without a row, a start that adds one and a start that pages through v1, writing nothing to v1 or v2, and runs dry runs whatever the table holds", + targets: { ...cloudOnboarding, cloud: managedCloud }, + }, v1ImportDryRun: { file: "cloud-v1-import-run.spec.ts", title: diff --git a/e2e/tests/cloud-v1-import-run.spec.ts b/e2e/tests/cloud-v1-import-run.spec.ts index ce164ef27c..37eb5779a1 100644 --- a/e2e/tests/cloud-v1-import-run.spec.ts +++ b/e2e/tests/cloud-v1-import-run.spec.ts @@ -16,7 +16,7 @@ import { writeRows, type RowStatement } from "../support/cloud-rows.ts"; import { Emulators } from "../support/emulators.ts"; import { Telemetry } from "../support/evidence.ts"; import { Onboarding } from "../support/onboarding.ts"; -import { v1ImportOperator, v1RunId } from "../support/v1-import.ts"; +import { allowV1Import, v1ImportOperator, v1RunId } from "../support/v1-import.ts"; import { scenarios } from "../test-plan.ts"; const services = Layer.mergeAll(Onboarding.layer, Emulators.layer); @@ -323,6 +323,7 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import runs", (it) => { // The first member already has a v2 account with the same verified email. yield* signInOnV1(first.email); + yield* allowV1Import(workspace.organization); const run = v1RunId("shared"); const started = yield* operator.start({ mode: "import", @@ -561,6 +562,7 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import runs", (it) => { ]); yield* signInOnV1(identity.email); + yield* allowV1Import(workspace.organization); const run = v1RunId("rollback"); const started = yield* operator.start({ mode: "import", @@ -619,6 +621,101 @@ layer(TestLive, { excludeTestServices: true })("Cloud v1 import runs", (it) => { ), ); + it.effect(scenarios.v1ImportAllowed.title, (context) => + withCase( + context, + Effect.gen(function* () { + const emulators = yield* Emulators; + const operator = yield* v1ImportOperator; + const workspace = (label: string) => + Effect.gen(function* () { + const created = yield* emulators.v1Workspace([ + `${label}-${randomUUID().slice(0, 8)}@example.test`, + ]); + yield* writeRows("v1-database.json", [ + row( + "insert into organizations (id, name, slug) values ($1, $2, $3)", + created.organization, + `Synthetic ${label} workspace`, + `v1-${randomUUID().slice(0, 8)}`, + ), + ]); + return created.organization; + }); + const unlisted = yield* workspace("unlisted"); + const listed = yield* workspace("listed"); + yield* allowV1Import(listed); + const notAllowed = (organization: string) => ({ + error: `private_v1_import_allowed has no row for ${organization}, so it is not imported.`, + }); + const nothingWritten = (organization: string) => + Effect.gen(function* () { + expect(yield* movedFlag(organization)).toEqual([]); + expect(yield* importState(organization)).toEqual([]); + }); + + // The operator token alone allows nothing: without a row, the import is refused. + const refused = yield* operator.start({ + mode: "import", + runId: v1RunId("unlisted"), + organizations: [unlisted], + }); + expect(refused.status).toBe(409); + expect(refused.body).toEqual(notAllowed(unlisted)); + yield* nothingWritten(unlisted); + + // One organization without a row refuses the whole start, naming only that one. + const mixed = yield* operator.start({ + mode: "import", + runId: v1RunId("mixed"), + organizations: [listed, unlisted], + }); + expect(mixed.status).toBe(409); + expect(mixed.body).toEqual(notAllowed(unlisted)); + yield* nothingWritten(listed); + yield* nothingWritten(unlisted); + + // An import never pages through v1, whatever the table holds. + const paged = yield* operator.start({ + mode: "import", + runId: v1RunId("paged"), + after: listed, + limit: 1, + }); + expect(paged.status).toBe(409); + expect(paged.body).toEqual({ + error: + "An import names its organizations with `organizations`; it never pages through v1 with `after` or `limit`.", + }); + + // A dry run ignores the table: it runs for the organization without a row. + const dry = v1RunId("unlisted-dry"); + const dryStarted = yield* operator.start({ + mode: "dry-run", + runId: dry, + organizations: [unlisted], + }); + expect(dryStarted.status, JSON.stringify(dryStarted.body)).toBe(200); + const dryReport = yield* operator.waitForReport(dry, unlisted); + expect(dryReport.status, dryReport.reason ?? "").toBe("planned"); + yield* nothingWritten(unlisted); + + // With its row, the organization imports and is cut over. + const run = v1RunId("listed"); + const started = yield* operator.start({ + mode: "import", + runId: run, + organizations: [listed], + }); + expect(started.status, JSON.stringify(started.body)).toBe(200); + const report = yield* operator.waitForReport(run, listed); + expect(report.status, report.reason ?? "").toBe("imported"); + expect(report.cutover).toBe(true); + expect((yield* movedFlag(listed))[0]?.state).toBe("moved"); + }).pipe(Effect.provide(services)), + ), + ); + it.effect(scenarios.v1ImportDryRun.title, (context) => withCase( context, diff --git a/e2e/tests/cloud-v1-import.spec.ts b/e2e/tests/cloud-v1-import.spec.ts index 082e69af42..feaf753b2a 100644 --- a/e2e/tests/cloud-v1-import.spec.ts +++ b/e2e/tests/cloud-v1-import.spec.ts @@ -44,7 +44,7 @@ import { v1SlackScopes, } from "../support/operator-oauth.ts"; import { Target } from "../support/platform.ts"; -import { v1ImportOperator, v1RunId, type V1Report } from "../support/v1-import.ts"; +import { allowV1Import, v1ImportOperator, v1RunId, type V1Report } from "../support/v1-import.ts"; import { scenarios } from "../test-plan.ts"; const services = Layer.mergeAll(Onboarding.layer, Emulators.layer); @@ -712,6 +712,7 @@ const importOnServer = ( ) => Effect.gen(function* () { const operator = yield* v1ImportOperator; + yield* allowV1Import(organization); const run = v1RunId("import"); const started = yield* operator.start({ mode: "import", diff --git a/e2e/tests/cloud-v1-session-handoff.spec.ts b/e2e/tests/cloud-v1-session-handoff.spec.ts index 562a9ab15b..ecdb2ce6ab 100644 --- a/e2e/tests/cloud-v1-session-handoff.spec.ts +++ b/e2e/tests/cloud-v1-session-handoff.spec.ts @@ -15,7 +15,7 @@ import { Emulators } from "../support/emulators.ts"; import { Onboarding } from "../support/onboarding.ts"; import { driver, Target } from "../support/platform.ts"; import { targetHosts } from "../support/role-hosts.ts"; -import { v1ImportOperator, v1RunId } from "../support/v1-import.ts"; +import { allowV1Import, v1ImportOperator, v1RunId } from "../support/v1-import.ts"; import { V1SessionHandoffSecret, v1HandoffId, @@ -53,6 +53,7 @@ const movedWorkspace = Effect.gen(function* () { params: [workspace.organization, `v1-${randomUUID().slice(0, 8)}`], }, ]); + yield* allowV1Import(workspace.organization); const run = v1RunId("handoff"); const started = yield* operator.start({ mode: "import", From b630c1eaf4f345cc0df60eeb9435b1e339264ce4 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:33:24 +0000 Subject: [PATCH 017/127] Export 8d7675c1 --- apps/hosted/cloud/src/agent-mail.ts | 2 +- apps/hosted/cloud/src/app-data.ts | 2 +- apps/hosted/cloud/src/app-domains.ts | 2 +- apps/hosted/cloud/src/app-ui.ts | 2 +- .../hosted/cloud/src/artifacts-credentials.ts | 2 +- apps/hosted/cloud/src/authoring.ts | 2 +- apps/hosted/cloud/src/compiler.ts | 2 +- apps/hosted/cloud/src/dashboard.ts | 2 +- apps/hosted/cloud/src/edge.ts | 2 +- apps/hosted/cloud/src/formatter.ts | 2 +- .../cloud/src/infrastructure/app-outbound.ts | 5 +- apps/hosted/cloud/src/invocation-telemetry.ts | 2 +- apps/hosted/cloud/src/main.ts | 7 +- apps/hosted/cloud/src/marketing.ts | 2 +- apps/hosted/cloud/src/mcp-server.ts | 7 +- apps/hosted/self-host/Dockerfile | 4 +- .../self-host/scripts/package-runtime.ts | 6 +- e2e/ci-selection.ts | 2 +- e2e/support/egress-upstream.ts | 13 ++++ e2e/test-plan.ts | 14 ++++ e2e/tests/app-fetch-brotli.spec.ts | 72 +++++++++++++++++++ packages/app-data/src/cloudflare.ts | 2 +- packages/sdk/src/implementation/app-runner.ts | 2 +- .../sdk/src/implementation/workerd-apps.ts | 2 +- packages/telemetry/scripts/build-motel.ts | 4 +- scripts/releases/bun-runtime/README.md | 42 ++++++----- scripts/releases/bun-runtime/build.patch | 16 ----- scripts/releases/bun-runtime/lockdown.patch | 63 +++++++++------- scripts/releases/bun-runtime/setup-linux.sh | 14 ++-- 29 files changed, 202 insertions(+), 97 deletions(-) create mode 100644 e2e/tests/app-fetch-brotli.spec.ts delete mode 100644 scripts/releases/bun-runtime/build.patch diff --git a/apps/hosted/cloud/src/agent-mail.ts b/apps/hosted/cloud/src/agent-mail.ts index c5ce120cc6..24896cbe52 100644 --- a/apps/hosted/cloud/src/agent-mail.ts +++ b/apps/hosted/cloud/src/agent-mail.ts @@ -18,7 +18,7 @@ export default AgentMailReceiver.make( Effect.succeed({ main: import.meta.url, workersDev: false, - compatibility: { date: "2026-09-08", flags: ["nodejs_compat"] }, + compatibility: { date: "2026-09-08", flags: ["nodejs_compat", "no_brotli_content_encoding"] }, }), Effect.gen(function* () { const mailbox = yield* Cloudflare.KV.WriteNamespace(AgentMailbox); diff --git a/apps/hosted/cloud/src/app-data.ts b/apps/hosted/cloud/src/app-data.ts index 82fe85f88d..05879ac44c 100644 --- a/apps/hosted/cloud/src/app-data.ts +++ b/apps/hosted/cloud/src/app-data.ts @@ -289,7 +289,7 @@ export default AppData.make( build: workerBuild("app-data"), ...(yield* cloudObservability), workersDev: false, - compatibility: { date: "2026-09-08", flags: ["nodejs_compat"] }, + compatibility: { date: "2026-09-08", flags: ["nodejs_compat", "no_brotli_content_encoding"] }, env: { ...(yield* telemetryBindings), ...(yield* appOutboundBindings), diff --git a/apps/hosted/cloud/src/app-domains.ts b/apps/hosted/cloud/src/app-domains.ts index 93cdce47ee..5749fe5fb1 100644 --- a/apps/hosted/cloud/src/app-domains.ts +++ b/apps/hosted/cloud/src/app-domains.ts @@ -242,7 +242,7 @@ export default AppDomainController.make( main: import.meta.url, ...(yield* cloudObservability), workersDev: false, - compatibility: { date: "2026-09-08", flags: ["nodejs_compat"] }, + compatibility: { date: "2026-09-08", flags: ["nodejs_compat", "no_brotli_content_encoding"] }, env: yield* telemetryBindings, }; }), diff --git a/apps/hosted/cloud/src/app-ui.ts b/apps/hosted/cloud/src/app-ui.ts index 88163dc0ee..8eda7d40e8 100644 --- a/apps/hosted/cloud/src/app-ui.ts +++ b/apps/hosted/cloud/src/app-ui.ts @@ -56,7 +56,7 @@ export default class AppPages extends Cloudflare.Worker()( }, compatibility: { date: "2026-09-08", - flags: ["nodejs_compat", "global_fetch_strictly_public"], + flags: ["nodejs_compat", "global_fetch_strictly_public", "no_brotli_content_encoding"], }, ...(dev ? {} diff --git a/apps/hosted/cloud/src/artifacts-credentials.ts b/apps/hosted/cloud/src/artifacts-credentials.ts index eeee431d36..473aed5120 100644 --- a/apps/hosted/cloud/src/artifacts-credentials.ts +++ b/apps/hosted/cloud/src/artifacts-credentials.ts @@ -421,7 +421,7 @@ export default ArtifactsCredentials.make( main: import.meta.url, ...(yield* cloudObservability), workersDev: false, - compatibility: { date: "2026-09-08", flags: ["nodejs_compat"] }, + compatibility: { date: "2026-09-08", flags: ["nodejs_compat", "no_brotli_content_encoding"] }, env: yield* telemetryBindings, }; }), diff --git a/apps/hosted/cloud/src/authoring.ts b/apps/hosted/cloud/src/authoring.ts index c1221c3731..f65e9e498a 100644 --- a/apps/hosted/cloud/src/authoring.ts +++ b/apps/hosted/cloud/src/authoring.ts @@ -18,7 +18,7 @@ export default Authoring.make( workersDev: false, build: workerBuild("authoring"), // No placement: a service binding runs this Worker beside its caller, the placed API Worker. - compatibility: { date: "2026-09-08", flags: ["nodejs_compat"] }, + compatibility: { date: "2026-09-08", flags: ["nodejs_compat", "no_brotli_content_encoding"] }, }; }), Effect.succeed(Authoring.of({ files: () => Effect.succeed(authoring) })), diff --git a/apps/hosted/cloud/src/compiler.ts b/apps/hosted/cloud/src/compiler.ts index b87f77642e..6939bbfd71 100644 --- a/apps/hosted/cloud/src/compiler.ts +++ b/apps/hosted/cloud/src/compiler.ts @@ -30,7 +30,7 @@ export default AppCompiler.make( ...(yield* cloudObservability), workersDev: false, build: workerBuild("compiler"), - compatibility: { date: "2026-09-08", flags: ["nodejs_compat"] }, + compatibility: { date: "2026-09-08", flags: ["nodejs_compat", "no_brotli_content_encoding"] }, env: yield* telemetryBindings, ...(Option.isSome(framework) ? { assets: { directory: framework.value } } : {}), }; diff --git a/apps/hosted/cloud/src/dashboard.ts b/apps/hosted/cloud/src/dashboard.ts index 5d5ade5cd0..bb716ad16d 100644 --- a/apps/hosted/cloud/src/dashboard.ts +++ b/apps/hosted/cloud/src/dashboard.ts @@ -80,7 +80,7 @@ export default Dashboard.make( build: workerBuild("dashboard"), env: { EXECUTOR_SITE_BUILD: site.hash.output }, // No placement: a service binding runs this Worker beside its caller, the placed API Worker. - compatibility: { date: "2026-09-08", flags: ["nodejs_compat"] }, + compatibility: { date: "2026-09-08", flags: ["nodejs_compat", "no_brotli_content_encoding"] }, }; }), Effect.succeed( diff --git a/apps/hosted/cloud/src/edge.ts b/apps/hosted/cloud/src/edge.ts index ee4cfb8768..f60841bada 100644 --- a/apps/hosted/cloud/src/edge.ts +++ b/apps/hosted/cloud/src/edge.ts @@ -36,7 +36,7 @@ export default Edge.make( main: import.meta.url, workersDev: false, build: workerBuild("edge"), - compatibility: { date: "2026-09-08", flags: ["nodejs_compat"] }, + compatibility: { date: "2026-09-08", flags: ["nodejs_compat", "no_brotli_content_encoding"] }, env: siteTelemetryBinding( (yield* postHogBindings).analyticsProxy, (yield* sentryBindings).errorTunnel, diff --git a/apps/hosted/cloud/src/formatter.ts b/apps/hosted/cloud/src/formatter.ts index 5de4e02660..d33cc217c3 100644 --- a/apps/hosted/cloud/src/formatter.ts +++ b/apps/hosted/cloud/src/formatter.ts @@ -23,7 +23,7 @@ export default Formatter.make( workersDev: false, build: workerBuild("formatter"), // No placement: a service binding runs this Worker beside its caller, the placed API Worker. - compatibility: { date: "2026-09-08", flags: ["nodejs_compat"] }, + compatibility: { date: "2026-09-08", flags: ["nodejs_compat", "no_brotli_content_encoding"] }, env: yield* telemetryBindings, }; }), diff --git a/apps/hosted/cloud/src/infrastructure/app-outbound.ts b/apps/hosted/cloud/src/infrastructure/app-outbound.ts index 9469bf4425..efdb5397f9 100644 --- a/apps/hosted/cloud/src/infrastructure/app-outbound.ts +++ b/apps/hosted/cloud/src/infrastructure/app-outbound.ts @@ -19,7 +19,10 @@ export const AppOutbound = Effect.gen(function* () { if (globalThis.__ALCHEMY_RUNTIME__) return yield* Cloudflare.Worker.ref("AppOutbound"); return yield* Cloudflare.Worker("AppOutbound", { workersDev: false, - compatibility: { date: "2026-09-08", flags: ["global_fetch_strictly_public"] }, + compatibility: { + date: "2026-09-08", + flags: ["global_fetch_strictly_public", "no_brotli_content_encoding"], + }, // This private service has no application bindings or credentials. script: `export default { fetch(request) { return fetch(request); } diff --git a/apps/hosted/cloud/src/invocation-telemetry.ts b/apps/hosted/cloud/src/invocation-telemetry.ts index 112e2cec0d..ce8dfbf201 100644 --- a/apps/hosted/cloud/src/invocation-telemetry.ts +++ b/apps/hosted/cloud/src/invocation-telemetry.ts @@ -11,7 +11,7 @@ export default InvocationTelemetry.make( return { main: import.meta.url, workersDev: false, - compatibility: { date: "2026-09-08", flags: ["nodejs_compat"] }, + compatibility: { date: "2026-09-08", flags: ["nodejs_compat", "no_brotli_content_encoding"] }, env: yield* telemetryBindings, // No tail consumer or native trace export here: the receiver must not observe itself. }; diff --git a/apps/hosted/cloud/src/main.ts b/apps/hosted/cloud/src/main.ts index f2ddc50963..a122b12993 100644 --- a/apps/hosted/cloud/src/main.ts +++ b/apps/hosted/cloud/src/main.ts @@ -194,7 +194,12 @@ export default Api.make( limits: { subrequests: 50_000 }, compatibility: { date: "2026-09-08", - flags: ["nodejs_compat", "global_fetch_strictly_public", "enable_request_signal"], + flags: [ + "nodejs_compat", + "global_fetch_strictly_public", + "enable_request_signal", + "no_brotli_content_encoding", + ], }, dev: { host: "127.0.0.1", diff --git a/apps/hosted/cloud/src/marketing.ts b/apps/hosted/cloud/src/marketing.ts index 95706ff5a4..4e9d266c3d 100644 --- a/apps/hosted/cloud/src/marketing.ts +++ b/apps/hosted/cloud/src/marketing.ts @@ -32,7 +32,7 @@ export default Marketing.make( workersDev: false, build: workerBuild("marketing"), ...(yield* cloudObservability), - compatibility: { date: "2026-09-08", flags: ["nodejs_compat"] }, + compatibility: { date: "2026-09-08", flags: ["nodejs_compat", "no_brotli_content_encoding"] }, // Only explicitly public paths below can read this binding. Dashboard files stay private. assets: { directory: site.outdir, diff --git a/apps/hosted/cloud/src/mcp-server.ts b/apps/hosted/cloud/src/mcp-server.ts index bce60d0754..40163e2c0d 100644 --- a/apps/hosted/cloud/src/mcp-server.ts +++ b/apps/hosted/cloud/src/mcp-server.ts @@ -51,7 +51,12 @@ export default McpServer.make( // The API Worker's flags, under which session objects ran before they moved here. compatibility: { date: "2026-09-08", - flags: ["nodejs_compat", "global_fetch_strictly_public", "enable_request_signal"], + flags: [ + "nodejs_compat", + "global_fetch_strictly_public", + "enable_request_signal", + "no_brotli_content_encoding", + ], }, env: { ...(yield* postHogBindings).env, diff --git a/apps/hosted/self-host/Dockerfile b/apps/hosted/self-host/Dockerfile index 858376133f..89fefe7f02 100644 --- a/apps/hosted/self-host/Dockerfile +++ b/apps/hosted/self-host/Dockerfile @@ -1,5 +1,5 @@ -FROM node:24.18.0-bookworm-slim AS build -COPY --from=oven/bun:1.4.2 /usr/local/bin/bun /usr/local/bin/bun +FROM node:24.22.1-bookworm-slim AS build +COPY --from=oven/bun:1.4.4 /usr/local/bin/bun /usr/local/bin/bun ARG EXECUTOR_BUILD_VERSION RUN test -n "$EXECUTOR_BUILD_VERSION" || { echo "EXECUTOR_BUILD_VERSION is required. Pass --build-arg EXECUTOR_BUILD_VERSION=, or set it in the environment for docker compose build." >&2; exit 1; } ENV EXECUTOR_BUILD_VERSION=$EXECUTOR_BUILD_VERSION EXECUTOR_ENVIRONMENT=self-host diff --git a/apps/hosted/self-host/scripts/package-runtime.ts b/apps/hosted/self-host/scripts/package-runtime.ts index 2d7a91e9b2..ada09e53bc 100644 --- a/apps/hosted/self-host/scripts/package-runtime.ts +++ b/apps/hosted/self-host/scripts/package-runtime.ts @@ -271,18 +271,18 @@ const config :Workerd.Config = ( extensions=[(modules=[(name="cloudflare-runtime:workflows-wrapped-binding",internal=true,esModule=embed "@@RUNTIME@@/workflow-binding.mjs")])], services=[ (name="product",worker=( - compatibilityDate="2026-09-01",compatibilityFlags=["nodejs_compat"], + compatibilityDate="2026-09-01",compatibilityFlags=["nodejs_compat","no_brotli_content_encoding"], modules=[(name="product.mjs",esModule=embed "@@RUNTIME@@/product.mjs"),(name="executor:pglite.wasm",wasm=embed "@@RUNTIME@@/pglite.wasm"),(name="executor:initdb.wasm",wasm=embed "@@RUNTIME@@/initdb.wasm"),(name="executor:pglite.data",data=embed "@@RUNTIME@@/pglite.data")], bindings=[(name="PRODUCT",durableObjectNamespace="ExecutorProduct"),(name="NATIVE",service="native"),(name="LEGACY_DATABASE",service="legacy-data"),(name="BLOBS",service="builds"),(name="DASHBOARD",service="dashboard"),(name="PUBLIC_FETCH",service="public"),(name="PRIVATE_FETCH",service="internet"),(name="MOTEL",service="motel"),(name="SELF",service=(name="product",entrypoint="SelfOrigin")),(name="APPS",service="apps"),(name="UNSAFE_EVAL",unsafeEval=void)], durableObjectNamespaces=[(className="ExecutorProduct",uniqueKey="executor-product",enableSql=true,preventEviction=true)],durableObjectStorage=(localDisk="product-data") )), (name="apps",worker=( - compatibilityDate="2026-07-30",compatibilityFlags=["nodejs_compat"],modules=[${moduleConfig.join(",")}], + compatibilityDate="2026-07-30",compatibilityFlags=["nodejs_compat","no_brotli_content_encoding"],modules=[${moduleConfig.join(",")}], bindings=[(name="LOADER",workerLoader=()),(name="DATA",durableObjectNamespace="AppDataSupervisor"),(name="AUTH",text="service-binding"),(name="CREDENTIAL_SECRET",fromEnvironment="EXECUTOR_CREDENTIAL_HANDLE_SECRET"),(name="APPS_PRIVATE_FETCH",json="@@APPS_PRIVATE_FETCH@@"),(name="APP_WORKERS",json="@@APP_WORKERS@@"),(name="APP_WORKER_IDLE_SECONDS",json="@@APP_WORKER_IDLE_SECONDS@@"),(name="PUBLIC_FETCH",service="public"),(name="SELF_ORIGIN",text=@@SELF_ORIGIN@@),(name="NPM_REGISTRY",text=@@NPM_REGISTRY@@),(name="SELF",service=(name="product",entrypoint="SelfOrigin")),(name="HOST",service=(name="product",entrypoint="WorkflowCallbacks")),(name="RUNS",wrapped=(moduleName="cloudflare-runtime:workflows-wrapped-binding",innerBindings=[(name="binding",service=(name="workflows",entrypoint="WorkflowBinding"))]))], durableObjectNamespaces=[(className="AppDataSupervisor",uniqueKey="executor-app-data",enableSql=true)],durableObjectStorage=(localDisk="app-data") )), (name="workflows",worker=( - compatibilityDate="2026-09-01",compatibilityFlags=["experimental","nodejs_compat"],modules=[${workflowModules.join(",")}], + compatibilityDate="2026-09-01",compatibilityFlags=["experimental","nodejs_compat","no_brotli_content_encoding"],modules=[${workflowModules.join(",")}], bindings=[(name="ENGINE",durableObjectNamespace="Engine"),(name="USER_WORKFLOW",service=(name="apps",entrypoint="AppWorkflows")),(name="BINDING_NAME",json=${JSON.stringify(JSON.stringify("executor-app-workflows"))}),(name="WORKFLOW_NAME",json=${JSON.stringify(JSON.stringify("executor-app-workflows"))})], durableObjectNamespaces=[${workflowEngines}],durableObjectStorage=(localDisk="workflow-data") )), diff --git a/e2e/ci-selection.ts b/e2e/ci-selection.ts index 10c748fcbb..39a4960275 100644 --- a/e2e/ci-selection.ts +++ b/e2e/ci-selection.ts @@ -92,7 +92,7 @@ const jobs = { runner: 1, pattern: - "Cloud onboarding|Cloud sign-in (?:keeps a new v1|skips the v1 check|admits members of only empty free v1)|Cloud OAuth callbacks|Cloud OAuth token requests record|Cloud product events|Cloud feedback|Cloud tracks an unusable OAuth|app query traces|observability retains|browser decode and startup|Browser connection failures explain|optimistic replay failures|private app crash reports|Platform admin impersonation|Cloud reports the framework pin|Cloud deploys fail promptly when the compiler does not answer|refuses every stored state Better Auth refuses|Billing reconciles only while visible|A dashboard read refreshed while in flight|Cloud finishes a slow app's tool listing|Cloud remembers a stalled tool listing|Cloud keeps each JSON Schema definition a tool listing repeats once|definitions share a name and length but not their JSON|definition names are long and of one length|Cloud MCP session objects (?:hold|make)|Cloud MCP request spans say whether|database failure while verifying an API key|use one API key together|Cloud cron wakes the schedule coordinator|schedule coordinator closes its database connections|Cloud runs a due schedule and requested webhook source setup while|app evaluation failures explain the likely cause|client request rejections are recorded on their request span|failure text reaches its caller|an app request Executor's network failed to send|MCP tool calls deliver their tool name and outcome|Executor time|Cloud serves a tool listing its isolate cannot keep|Cloud writes the background refresh of a stale tool listing|Owners delete an organization with every app|Hosted MCP negotiates older protocol versions|Hosted MCP ends a cancelled call|Cloud lists each build's browser files|Hosted dashboard runs of approval-gated tools|Hosted dashboard approval", + "Cloud onboarding|Cloud sign-in (?:keeps a new v1|skips the v1 check|admits members of only empty free v1)|Cloud OAuth callbacks|Cloud OAuth token requests record|Cloud product events|Cloud feedback|Cloud tracks an unusable OAuth|app query traces|observability retains|browser decode and startup|Browser connection failures explain|optimistic replay failures|private app crash reports|Platform admin impersonation|Cloud reports the framework pin|Cloud deploys fail promptly when the compiler does not answer|refuses every stored state Better Auth refuses|Billing reconciles only while visible|A dashboard read refreshed while in flight|Cloud finishes a slow app's tool listing|Cloud remembers a stalled tool listing|Cloud keeps each JSON Schema definition a tool listing repeats once|definitions share a name and length but not their JSON|definition names are long and of one length|Cloud MCP session objects (?:hold|make)|Cloud MCP request spans say whether|database failure while verifying an API key|use one API key together|Cloud cron wakes the schedule coordinator|schedule coordinator closes its database connections|Cloud runs a due schedule and requested webhook source setup while|app evaluation failures explain the likely cause|client request rejections are recorded on their request span|failure text reaches its caller|an app request Executor's network failed to send|an app's fetch receives a Brotli-encoded response|MCP tool calls deliver their tool name and outcome|Executor time|Cloud serves a tool listing its isolate cannot keep|Cloud writes the background refresh of a stale tool listing|Owners delete an organization with every app|Hosted MCP negotiates older protocol versions|Hosted MCP ends a cancelled call|Cloud lists each build's browser files|Hosted dashboard runs of approval-gated tools|Hosted dashboard approval", }, // The scenarios above share one local Cloud and collector, and their span and latency checks // stall when more scenarios load it. These assert no such bound, so they start their own. The diff --git a/e2e/support/egress-upstream.ts b/e2e/support/egress-upstream.ts index ce7bfb9447..96ace2747d 100644 --- a/e2e/support/egress-upstream.ts +++ b/e2e/support/egress-upstream.ts @@ -3,6 +3,15 @@ import { Effect } from "effect"; import { createServer, type Server } from "node:http"; import { refusingPort } from "./ports.ts"; +/** + * The body `/brotli` sends with `Content-Encoding: br`: a valid Brotli stream of the text + * "synthetic brotli payload", so a runtime that decoded it would hand the app that text. + */ +export const brotliBody = Buffer.from( + "1b1700f8456f78bdee17294a883d42a29044f7053784926c2383861f", + "hex", +); + const listen = (server: Server) => Effect.callback((resume) => { server.listen(0, "127.0.0.1", () => { @@ -31,6 +40,10 @@ export const egressUpstream = Effect.gen(function* () { case "/forged": response.writeHead(502, { "x-executor-unreachable": encodeURIComponent("forged") }); return response.end("service answer"); + // Sent whatever the request accepts, as a hostile service would. + case "/brotli": + response.writeHead(200, { "content-type": "text/plain", "content-encoding": "br" }); + return response.end(brotliBody); default: response.writeHead(200, { "content-type": "application/json" }); return response.end("{}"); diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index d6f1dc966f..75f9bc07d9 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -2814,6 +2814,20 @@ export const scenarios = plan({ ), }, }, + appFetchBrotliUndecoded: { + fixtures: "actors", + file: "app-fetch-brotli.spec.ts", + title: + "an app's fetch receives a Brotli-encoded response as the bytes the service sent, never decoded by the runtime", + targets: { + "self-host": scheduled, + // The scenario reaches a loopback fixture. + cloud: managedCloud, + local: na( + "Hosted app routes; local runs apps with the same runner and the same compatibility flags.", + ), + }, + }, appFetchUnsupportedOption: { fixtures: "actors", file: "app-fetch-errors.spec.ts", diff --git a/e2e/tests/app-fetch-brotli.spec.ts b/e2e/tests/app-fetch-brotli.spec.ts new file mode 100644 index 0000000000..c15ccf7c7c --- /dev/null +++ b/e2e/tests/app-fetch-brotli.spec.ts @@ -0,0 +1,72 @@ +/** + * The app runtime never decodes a Brotli response. workerd's bundled Brotli decoder has + * CVE-2026-111497, an out-of-bounds write a crafted stream can turn into code execution, so every + * Worker sets `no_brotli_content_encoding` and a service's `Content-Encoding: br` body reaches app + * code as the bytes the service sent, through the platform's `fetch` and the framework's + * `ctx.fetch` alike. + */ +import { expect, layer } from "@effect/vitest"; +import { Effect, Schema } from "effect"; +import { randomUUID } from "node:crypto"; +import { scenarios } from "../test-plan.ts"; +import { Api, body } from "../support/api.ts"; +import { Actors } from "../support/actors.ts"; +import { HostedLive, withHostedCase } from "../support/case.ts"; +import { App } from "../support/contracts.ts"; +import { appsManifest } from "../support/apps-release.ts"; +import { brotliBody, egressUpstream } from "../support/egress-upstream.ts"; + +const Received = Schema.Struct({ + status: Schema.Number, + encoding: Schema.NullOr(Schema.String), + hex: Schema.String, +}); + +const fetchApp = `import { defineApp, object, string, query, router } from "apps"; +const received = async (response) => ({ + status: response.status, + encoding: response.headers.get("content-encoding"), + hex: Array.from(new Uint8Array(await response.arrayBuffer()), (byte) => byte.toString(16).padStart(2, "0")).join(""), +}); +const input = { input: object({ url: string() }) }; +export default defineApp({}, { + tools: router({ + global: query(input, async (_ctx, { url }) => received(await fetch(url))), + fetched: query(input, async (ctx, { url }) => received(await ctx.fetch(url))), + }), +});`; + +layer(HostedLive, { excludeTestServices: true })("App fetch Brotli", (it) => { + it.effect(scenarios.appFetchBrotliUndecoded.title, (context) => + withHostedCase( + context, + Effect.gen(function* () { + const api = yield* Api, + actors = yield* Actors; + const upstream = yield* egressUpstream; + const prefix = `/api/organizations/${actors.organization.id}/apps`; + const deployed = yield* api.request(actors.owner, "POST", `${prefix}/deploy`, { + name: `Fetch Brotli ${randomUUID().slice(0, 8)}`, + files: [{ path: "index.ts", content: fetchApp }, appsManifest], + }); + expect(deployed.status, JSON.stringify(deployed.body)).toBe(200); + const app = yield* body(App, deployed); + yield* Effect.addFinalizer(() => + api.request(actors.owner, "DELETE", `${prefix}/${app.id}`).pipe(Effect.orDie), + ); + + const sent = { status: 200, encoding: "br", hex: brotliBody.toString("hex") }; + for (const tool of ["global", "fetched"]) { + const response = yield* api.request( + actors.owner, + "POST", + `${prefix}/${app.id}/tools/call`, + { tool, kind: "query", input: { url: `${upstream.origin}/brotli` } }, + ); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(yield* body(Received, response), tool).toEqual(sent); + } + }), + ), + ); +}); diff --git a/packages/app-data/src/cloudflare.ts b/packages/app-data/src/cloudflare.ts index 574f480332..6a080587fb 100644 --- a/packages/app-data/src/cloudflare.ts +++ b/packages/app-data/src/cloudflare.ts @@ -433,7 +433,7 @@ export const makeFacetSupervisor = ( modules: workerModules(bundle.modules), compatibilityDate: "2026-07-30", // The strictly-public flag would override this outbound and bypass it. - compatibilityFlags: ["nodejs_compat"], + compatibilityFlags: ["nodejs_compat", "no_brotli_content_encoding"], globalOutbound: globalOutbound(app), }; }); diff --git a/packages/sdk/src/implementation/app-runner.ts b/packages/sdk/src/implementation/app-runner.ts index bea73bfe11..69560f9095 100644 --- a/packages/sdk/src/implementation/app-runner.ts +++ b/packages/sdk/src/implementation/app-runner.ts @@ -576,7 +576,7 @@ export const makeAppRunner = (host: AppRunnerHost) => { mainModule: "__executor_rpc.js", modules: read.success.modules, compatibilityDate, - compatibilityFlags: ["nodejs_compat"], + compatibilityFlags: ["nodejs_compat", "no_brotli_content_encoding"], globalOutbound: options.globalOutbound, }; }; diff --git a/packages/sdk/src/implementation/workerd-apps.ts b/packages/sdk/src/implementation/workerd-apps.ts index 6f1b617399..f7b531f258 100644 --- a/packages/sdk/src/implementation/workerd-apps.ts +++ b/packages/sdk/src/implementation/workerd-apps.ts @@ -220,7 +220,7 @@ export const workerdApps = (options: { name: "executor-apps", compatibilityDate: "2026-07-30", // The trusted host worker keeps the default network. Only app isolates are restricted. - compatibilityFlags: ["nodejs_compat"], + compatibilityFlags: ["nodejs_compat", "no_brotli_content_encoding"], modules: yield* workerdHostModules, durableObjectNamespaces: [ { className: "AppDataSupervisor", sql: true, uniqueKey: "executor-app-data" }, diff --git a/packages/telemetry/scripts/build-motel.ts b/packages/telemetry/scripts/build-motel.ts index 6cf600ebd5..d710aced2d 100644 --- a/packages/telemetry/scripts/build-motel.ts +++ b/packages/telemetry/scripts/build-motel.ts @@ -7,8 +7,8 @@ import { realpath } from "node:fs"; import { promisify } from "node:util"; const build = Effect.gen(function* () { - if (process.versions.bun !== "1.4.2") - return yield* Effect.die(new Error("Build the collector with Bun 1.4.2.")); + if (process.versions.bun !== "1.4.4") + return yield* Effect.die(new Error("Build the collector with Bun 1.4.4.")); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const processes = yield* ChildProcessSpawner.ChildProcessSpawner; diff --git a/scripts/releases/bun-runtime/README.md b/scripts/releases/bun-runtime/README.md index 3dd0830ae8..61b3b67b0c 100644 --- a/scripts/releases/bun-runtime/README.md +++ b/scripts/releases/bun-runtime/README.md @@ -7,23 +7,23 @@ process that can start the CLI could use them to run its own code inside our signed binary. We therefore compile the CLI with a Bun runtime that we patch and build ourselves. -`lockdown.patch` applies to Bun `bun-v1.4.2` (commit `744846f8`) with +`lockdown.patch` applies to Bun `bun-v1.4.4` (commit `38afee6c`) with `git apply`. At startup, `main` checks once whether the executable embeds a standalone module graph and stores the answer in a process-wide flag. Rust reads it through `bun_core::is_standalone_executable()` and C++ through `Bun__isStandaloneExecutable()`. When the flag is set, Bun ignores the following: -| Vector | Effect in stock Bun | Where the patch closes it | -| -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | -| `BUN_BE_BUN` | The executable becomes the full `bun` CLI (`-e`, `run`, `x`, ...) | `src/runtime/cli/mod.rs`, graph dispatch | -| `BUN_INTERNAL_WEBVIEW_HOST` | Before the graph is checked, the process becomes a WebView host that talks to an inherited fd (macOS) | `src/runtime/cli/mod.rs` | -| `BUN_OPTIONS` | Spliced into argv: `--preload`, `--require`, `--import`, `--config`, `--cwd`, `--env-file`, `--inspect`, ... | `src/bun_core/util.rs`, `argv_view_init` | -| `BUN_INSPECT`, `BUN_INSPECT_CONNECT_TO` | Starts the inspector, either listening or connecting out, which gives full JS control | `src/runtime/jsc_hooks.rs`, `configure_debugger` | -| `BUN_INSPECT_PRELOAD` | Preloads a module | `src/runtime/cli/Arguments.rs` | -| `BUN_JSC_*` | Sets any JavaScriptCore option, including JIT, memory-safety and internal-API switches | `src/jsc/bindings/ZigGlobalObject.cpp`, `JSCInitialize` | -| `NODE_PATH` | Adds module search roots, which can satisfy optional `require()`s in packages loaded from disk (for example `ws` → `bufferutil`) | `src/resolver/resolver.rs` | -| Runtime transpiler cache (`BUN_RUNTIME_TRANSPILER_CACHE_PATH`, `XDG_CACHE_HOME`, `~/Library/Caches/bun`) | Transpiled output for files loaded from disk is read from a user-writable cache | `src/jsc/RuntimeTranspilerCache.rs`, `src/bundler/cache.rs`: always off when compiled | +| Vector | Effect in stock Bun | Where the patch closes it | +| -------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | +| `BUN_BE_BUN` | The executable becomes the full `bun` CLI (`-e`, `run`, `x`, ...) | `src/runtime/cli/mod.rs`, graph dispatch | +| `BUN_INTERNAL_WEBVIEW_HOST` | Before the graph is checked, the process becomes a WebView host that talks to an inherited fd (macOS) | `src/runtime/cli/mod.rs` | +| `BUN_OPTIONS` | Spliced into argv: `--preload`, `--require`, `--import`, `--config`, `--cwd`, `--env-file`, `--inspect`, ... | `src/bun_core/util.rs`, `argv_view_init` | +| `BUN_INSPECT`, `BUN_INSPECT_CONNECT_TO` | Starts the inspector, either listening or connecting out, which gives full JS control | `src/runtime/jsc_hooks.rs`, `configure_debugger` | +| `BUN_INSPECT_PRELOAD` | Preloads a module | `src/runtime/cli/Arguments.rs` | +| `BUN_JSC_*` | Sets any JavaScriptCore option, including JIT, memory-safety and internal-API switches | `src/jsc/bindings/ZigGlobalObject.cpp`, `JSCInitialize` and `JSC__useOptionsForVMPerThread` | +| `NODE_PATH` | Adds module search roots, which can satisfy optional `require()`s in packages loaded from disk (for example `ws` → `bufferutil`) | `src/resolver/resolver.rs` | +| Runtime transpiler cache (`BUN_RUNTIME_TRANSPILER_CACHE_PATH`, `XDG_CACHE_HOME`, `~/Library/Caches/bun`) | Transpiled output for files loaded from disk is read from a user-writable cache | `src/jsc/RuntimeTranspilerCache.rs`, `src/bundler/cache.rs`: always off when compiled | Plain `bun`, without an embedded graph, behaves exactly as upstream. @@ -58,15 +58,16 @@ same build model as Bun's release CI: `darwin-x64` runs under Rosetta on Apple silicon. The binaries and a `SHA256SUMS` file are published to the release -`bun-runtime-1.4.2-`. An existing +`bun-runtime-1.4.4-`. An existing release is never overwritten. -To build locally on macOS (arm64), use LLVM 21, rustup and Bun 1.4.2: +To build locally on macOS (arm64), use LLVM 23, rustup and Bun 1.4.2, the Bun +that Bun 1.4.4's build scripts pin: ```sh -git clone --depth 1 --branch bun-v1.4.2 https://github.com/oven-sh/bun.git bun-src -cd bun-src && git apply ../scripts/releases/bun-runtime/lockdown.patch ../scripts/releases/bun-runtime/build.patch -export PATH="$(brew --prefix llvm@21)/bin:$PATH" +git clone --depth 1 --branch bun-v1.4.4 https://github.com/oven-sh/bun.git bun-src +cd bun-src && git apply ../scripts/releases/bun-runtime/lockdown.patch +export PATH="$(brew --prefix llvm@23)/bin:$PATH" # clang 21 rejects headers in newer SDKs (Xcode betas); the 26.5 CLT SDK works. export DEVELOPER_DIR=/Library/Developer/CommandLineTools bun scripts/build.ts --profile=release --canary=off # -> build/release/bun @@ -75,7 +76,9 @@ bash ../scripts/releases/bun-runtime/smoke.sh build/release/bun ## Updating Bun -1. Change `BUN_VERSION`, `BUN_TAG` and `BUN_COMMIT` in the workflow. +1. Change `BUN_VERSION`, `BUN_TAG` and `BUN_COMMIT` in the workflow, and set + `BUILD_BUN_VERSION` to the Bun the new tag's `scripts/build/ci-images/spec.ts` + pins. Check that `setup-linux.sh` installs the LLVM series it pins. 2. Check out the new tag and run `git apply --3way lockdown.patch`. Resolve any conflicts. 3. Audit the new version again. Search for new environment variables @@ -86,8 +89,3 @@ bash ../scripts/releases/bun-runtime/smoke.sh build/release/bun 4. Regenerate the patch with `git diff > lockdown.patch`, build, and run `smoke.sh`. 5. Merge to `main`. The new patch hash produces a new release name. - -`build.patch` changes only Bun's build scripts. Release CI builds with `--ci=on`, which sets the -macOS deployment target, fetches the Windows sysroot and strips checkout paths, but upstream -also skips the precompiled header in that mode, which its own pipeline never uses, and the C++ -sources then miss declarations the header provides. The patch keeps the precompiled header on. diff --git a/scripts/releases/bun-runtime/build.patch b/scripts/releases/bun-runtime/build.patch deleted file mode 100644 index 62daf94d35..0000000000 --- a/scripts/releases/bun-runtime/build.patch +++ /dev/null @@ -1,16 +0,0 @@ -diff --git a/scripts/build/bun.ts b/scripts/build/bun.ts -index fc1a7481..3b06562d 100644 ---- a/scripts/build/bun.ts -+++ b/scripts/build/bun.ts -@@ -277,8 +277,9 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput { - const cFlagsFull = [...flags.cflags, ...includeFlags, ...defineFlags]; - - // ─── Step 4: PCH ─── -- // CI full mode (unused by the pipeline) skips the PCH; cpp-only/archive-link use it. -- const usePch = !cfg.ci || cfg.mode !== "full"; -+ // CI full mode skips the PCH upstream, but that mode is unused by Bun's pipeline and its -+ // headers rely on the PCH for declarations (Bun::createClassStructure), so always use it. -+ const usePch = true; - let pchOut: { pch: string; wrapperHeader: string } | undefined; - - if (usePch) { diff --git a/scripts/releases/bun-runtime/lockdown.patch b/scripts/releases/bun-runtime/lockdown.patch index 9e79954324..3ca47d63cb 100644 --- a/scripts/releases/bun-runtime/lockdown.patch +++ b/scripts/releases/bun-runtime/lockdown.patch @@ -1,8 +1,8 @@ diff --git a/src/bun_core/util.rs b/src/bun_core/util.rs -index 28006517..160c80d2 100644 +index 2ceeb185..263ced96 100644 --- a/src/bun_core/util.rs +++ b/src/bun_core/util.rs -@@ -3796,14 +3796,45 @@ fn argv_storage() -> &'static [ZBox] { +@@ -3765,14 +3765,45 @@ fn argv_storage() -> &'static [ZBox] { }) } @@ -63,10 +63,10 @@ index fb867d89..22904c63 100644 .get() .map(|v| v.is_empty() || v == b"0") diff --git a/src/jsc/RuntimeTranspilerCache.rs b/src/jsc/RuntimeTranspilerCache.rs -index 6ad49e14..28aebab2 100644 +index 8d40a1bc..6ccf2e05 100644 --- a/src/jsc/RuntimeTranspilerCache.rs +++ b/src/jsc/RuntimeTranspilerCache.rs -@@ -611,6 +611,12 @@ impl RuntimeTranspilerCache { +@@ -645,6 +645,12 @@ impl RuntimeTranspilerCache { ); } @@ -79,7 +79,7 @@ index 6ad49e14..28aebab2 100644 if let Some(dir) = env_var::BUN_RUNTIME_TRANSPILER_CACHE_PATH.get() { if dir.is_empty() || (dir.len() == 1 && dir[0] == b'0') { return 0; -@@ -842,7 +848,7 @@ impl RuntimeTranspilerCache { +@@ -877,7 +883,7 @@ impl RuntimeTranspilerCache { parser_options: &ParserOptions<'_>, used_jsx: bool, ) -> bool { @@ -89,12 +89,12 @@ index 6ad49e14..28aebab2 100644 } diff --git a/src/jsc/bindings/ZigGlobalObject.cpp b/src/jsc/bindings/ZigGlobalObject.cpp -index 1bcd27a1..60b26cab 100644 +index 7f4ae0c3..0f85ea65 100644 --- a/src/jsc/bindings/ZigGlobalObject.cpp +++ b/src/jsc/bindings/ZigGlobalObject.cpp -@@ -280,6 +280,9 @@ extern "C" long Bun__crashHandlerFromJSCFrame(void*, void*, void*, void*); - // bun_icu_default_locale.cpp - extern "C" void Bun__ensureICUDefaultLocale(); +@@ -296,6 +296,9 @@ extern "C" void Bun__VirtualMachine__setSamplingProfilerDirectory(void* bunVM, c + // reporter, which crashes if fopen fails. VirtualMachine::write_profiles writes the report instead. + static const char8_t* s_samplingProfilerDirectory = nullptr; +// bun_core::is_standalone_executable(): true for `bun build --compile` output. +extern "C" bool Bun__isStandaloneExecutable(); @@ -102,7 +102,7 @@ index 1bcd27a1..60b26cab 100644 extern "C" void JSCInitialize(const char* envp[], size_t envc, void (*onCrash)(const char* ptr, size_t length), bool evalMode, bool oneShotStartup, bool shortLivedGlobals) { static std::once_flag jsc_init_flag; -@@ -360,7 +363,9 @@ extern "C" void JSCInitialize(const char* envp[], size_t envc, void (*onCrash)(c +@@ -382,7 +385,9 @@ extern "C" void JSCInitialize(const char* envp[], size_t envc, void (*onCrash)(c JSC::Options::thresholdForFTLOptimizeAfterWarmUp() = 1000000; } @@ -113,11 +113,22 @@ index 1bcd27a1..60b26cab 100644 auto envc_copy = envc; while (envc_copy--) { const char* env = (const char*)envp[envc_copy]; +@@ -440,7 +445,9 @@ extern "C" void JSC__useOptionsForVMPerThread(const char* envp[], size_t envc) + { + RELEASE_ASSERT(!g_jscConfig.isPermanentlyFrozen()); + JSC::Options::useConcurrentJIT() = false; +- // BUN_JSC_useConcurrentJIT wins, as in JSCInitialize. ++ // BUN_JSC_useConcurrentJIT wins, as in JSCInitialize. Compiled executables ignore it, as they ignore every BUN_JSC_*. ++ if (Bun__isStandaloneExecutable()) ++ return; + for (size_t i = 0; i < envc; i++) { + if (WTF::StringView::fromLatin1(envp[i]).startsWithIgnoringASCIICase("BUN_JSC_useConcurrentJIT="_s)) + JSC::Options::setOption(envp[i] + 8, false); diff --git a/src/resolver/resolver.rs b/src/resolver/resolver.rs -index 24f019f6..99e9bcf8 100644 +index 5e63425e..0c447b32 100644 --- a/src/resolver/resolver.rs +++ b/src/resolver/resolver.rs -@@ -2794,8 +2794,11 @@ impl<'a> Resolver<'a> { +@@ -2877,8 +2877,11 @@ impl<'a> Resolver<'a> { // try resolve from `NODE_PATH` // https://nodejs.org/api/modules.html#loading-from-the-global-folders @@ -130,12 +141,12 @@ index 24f019f6..99e9bcf8 100644 .unwrap_or(b""); if !node_path.is_empty() { diff --git a/src/runtime/bin_entry/mod.rs b/src/runtime/bin_entry/mod.rs -index 098d8c73..1a6a9058 100644 +index 2169092d..b42bdbb3 100644 --- a/src/runtime/bin_entry/mod.rs +++ b/src/runtime/bin_entry/mod.rs -@@ -154,6 +154,10 @@ pub(crate) unsafe extern "C" fn main(argc: c_int, argv: *const *const c_char) -> - // for the entire process. - unsafe { bun_core::init_argv(argc, argv) }; +@@ -168,6 +168,10 @@ pub(crate) unsafe extern "C" fn main(argc: c_int, argv: *const *const c_char) -> + bun_abort_missing_simd(); + } + // 0b. Decide once whether this is a `bun build --compile` executable, + // before argv (BUN_OPTIONS) or any other environment variable is read. @@ -145,10 +156,10 @@ index 098d8c73..1a6a9058 100644 bun_crash_handler::init(); diff --git a/src/runtime/cli/Arguments.rs b/src/runtime/cli/Arguments.rs -index 8ed18c92..7ced2bf4 100644 +index d2a10546..18c1b69e 100644 --- a/src/runtime/cli/Arguments.rs +++ b/src/runtime/cli/Arguments.rs -@@ -1013,7 +1013,9 @@ pub(crate) fn parse(cmd: CommandTag, ctx: Context<'_>) -> crate::Result) -> crate::Result(fd_str, 10).ok() { -@@ -1189,13 +1193,18 @@ pub mod command { +@@ -1030,13 +1034,18 @@ pub(crate) mod command { } } @@ -200,10 +211,10 @@ index b1046329..c47e3174 100644 // Fast path: `bun -v` / `bun --version` / `bun --revision`, the diff --git a/src/runtime/jsc_hooks.rs b/src/runtime/jsc_hooks.rs -index 029b3568..cd76cb4f 100644 +index 920f9c46..dc9bcb4d 100644 --- a/src/runtime/jsc_hooks.rs +++ b/src/runtime/jsc_hooks.rs -@@ -591,8 +591,19 @@ unsafe fn configure_debugger( +@@ -558,8 +558,19 @@ unsafe fn configure_debugger( return; } @@ -226,10 +237,10 @@ index 029b3568..cd76cb4f 100644 let set_breakpoint_on_first_line = !unix.is_empty() && unix.ends_with(b"?break=1"); let wait_for_debugger = !unix.is_empty() && unix.ends_with(b"?wait=1"); diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs -index 9ab6dbbf..550cd4c5 100644 +index 91c73cb9..23ea99f2 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs -@@ -2710,6 +2710,31 @@ impl StandaloneModuleGraph { +@@ -3106,6 +3106,31 @@ impl StandaloneModuleGraph { /// Loads the standalone module graph from the executable, allocates it on the heap, /// sets it globally, and returns the pointer. pub fn from_executable() -> crate::Result> { @@ -261,7 +272,7 @@ index 9ab6dbbf..550cd4c5 100644 #[cfg(target_os = "macos")] let data = macho::get_data(); #[cfg(windows)] -@@ -2724,30 +2749,21 @@ impl StandaloneModuleGraph { +@@ -3120,30 +3145,21 @@ impl StandaloneModuleGraph { target_os = "freebsd" )))] let data: Option<(*mut u8, usize)> = None; diff --git a/scripts/releases/bun-runtime/setup-linux.sh b/scripts/releases/bun-runtime/setup-linux.sh index f2ff91de64..1f970099f2 100755 --- a/scripts/releases/bun-runtime/setup-linux.sh +++ b/scripts/releases/bun-runtime/setup-linux.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Prepares a GitHub-hosted Ubuntu 24.04 runner to build the patched Bun # runtime for one target, the way Bun's own release CI does: -# - LLVM 21 from apt.llvm.org (scripts/bootstrap.sh install_llvm) +# - LLVM 23.1 from apt.llvm.org (the series scripts/build/ci-images/spec.ts pins) # - the pinned nightly Rust toolchain plus the target's std # - for linux targets, the ubuntu:20.04 glibc sysroot with gcc-13 -# libstdc++ (scripts/bootstrap.sh install_linux_glibc_sysroot), so the +# libstdc++ (spec.ts glibcSysroot), so the # binary needs glibc 2.17 like the official release, not the runner's # glibc # darwin targets fetch the macOS SDK from Apple's CDN and windows targets @@ -34,13 +34,13 @@ sudo apt-get install -y --no-install-recommends \ automake autoconf libtool nasm xz-utils unzip zip jq skopeo \ binutils-x86-64-linux-gnu binutils-aarch64-linux-gnu -# LLVM 21 (clang, lld incl. ld64.lld and lld-link, llvm-strip, dsymutil). +# LLVM 23 (clang, lld incl. ld64.lld and lld-link, llvm-strip, dsymutil). llvm_script="$(mktemp)" curl -fsSL https://apt.llvm.org/llvm.sh -o "$llvm_script" -sudo bash "$llvm_script" 21 all -sudo apt-get install -y --no-install-recommends llvm-21-tools -echo "/usr/lib/llvm-21/bin" >> "$GITHUB_PATH" -/usr/lib/llvm-21/bin/clang --version | head -1 +sudo bash "$llvm_script" 23 all +sudo apt-get install -y --no-install-recommends llvm-23-tools +echo "/usr/lib/llvm-23/bin" >> "$GITHUB_PATH" +/usr/lib/llvm-23/bin/clang --version | head -1 # Rust: the toolchain pinned in the Bun checkout's rust-toolchain.toml. if ! command -v rustup >/dev/null 2>&1; then From c221d23b7990272f4de458fc5fe2fda515dfd752 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sat, 24 Oct 2026 07:47:57 +0000 Subject: [PATCH 018/127] Export 21d0f628 --- apps/cli/CHANGELOG.md | 4 + apps/cli/package.json | 2 +- .../cloud/src/implementation/mcp-forward.ts | 40 ++- apps/hosted/cloud/src/infrastructure/mcp.ts | 6 +- bun.lock | 2 +- e2e/ci-selection.ts | 1 + e2e/subrequest-depth.config.ts | 12 + e2e/tests/cloud-mcp-subrequest-depth.spec.ts | 256 ++++++++++++++++++ 8 files changed, 318 insertions(+), 5 deletions(-) create mode 100644 e2e/subrequest-depth.config.ts create mode 100644 e2e/tests/cloud-mcp-subrequest-depth.spec.ts diff --git a/apps/cli/CHANGELOG.md b/apps/cli/CHANGELOG.md index 6612278fdd..4ee7938deb 100644 --- a/apps/cli/CHANGELOG.md +++ b/apps/cli/CHANGELOG.md @@ -1,5 +1,9 @@ # executor +## 2.0.0-beta.28 + +Nightly beta of the Executor local runtime, CLI, desktop, and self-host Docker image. + ## 2.0.0-beta.27 Nightly beta of the Executor local runtime, CLI, desktop, and self-host Docker image. diff --git a/apps/cli/package.json b/apps/cli/package.json index 1b9b68fd3f..3ccd6d6840 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "executor", - "version": "2.0.0-beta.27", + "version": "2.0.0-beta.28", "private": true, "description": "Run Executor locally and connect your agents to your apps.", "homepage": "https://executor.sh", diff --git a/apps/hosted/cloud/src/implementation/mcp-forward.ts b/apps/hosted/cloud/src/implementation/mcp-forward.ts index f333102f6b..b6d67ce839 100644 --- a/apps/hosted/cloud/src/implementation/mcp-forward.ts +++ b/apps/hosted/cloud/src/implementation/mcp-forward.ts @@ -1,7 +1,45 @@ import { Cause, Effect, Schema } from "effect"; -import { HttpServerRequest, type HttpServerResponse } from "effect/http"; +import { HttpServerRequest, HttpServerResponse } from "effect/http"; import type { HttpServerError } from "effect/http/HttpServerError"; +/** + * Cloudflare refuses a call to a Worker or Durable Object once its request has entered the + * Workers runtime too many times (at most 32 Worker invocations per request). A client calling + * from deep inside its own Worker chain, often one that calls itself, reaches that limit at the + * gateway's call to the session object. The object never ran, and the same request from the same + * depth is refused again, so the client is told why instead of getting an empty 500. + */ +export const refuseTooDeep = ( + forward: Effect.Effect, +) => + forward.pipe( + Effect.catchCause((cause) => { + const error = Cause.squash(cause); + if ( + Cause.hasInterrupts(cause) || + !(error instanceof Error) || + !error.message.startsWith("Subrequest depth limit exceeded.") + ) + return Effect.failCause(cause); + return Effect.annotateCurrentSpan("executor.mcp.refused", "subrequest-depth").pipe( + Effect.as( + HttpServerResponse.jsonUnsafe( + { + jsonrpc: "2.0", + id: null, + error: { + code: -32000, + message: + "Subrequest depth limit exceeded: this request reached Executor after passing through Cloudflare Workers too many times, so Cloudflare refused Executor's next step. The Worker or Durable Object that called Executor is nested too deeply, often because it calls itself or other Workers recursively. Call Executor from a shallower point in that chain", + }, + }, + { status: 508, headers: { "cache-control": "no-store" } }, + ), + ), + ); + }), + ); + const Initialize = Schema.Struct({ jsonrpc: Schema.Literal("2.0"), method: Schema.Literal("initialize"), diff --git a/apps/hosted/cloud/src/infrastructure/mcp.ts b/apps/hosted/cloud/src/infrastructure/mcp.ts index 7ec8a0cfd4..e2eb5c9424 100644 --- a/apps/hosted/cloud/src/infrastructure/mcp.ts +++ b/apps/hosted/cloud/src/infrastructure/mcp.ts @@ -12,7 +12,7 @@ import * as Cloudflare from "alchemy/Cloudflare"; import { Effect, Option, Schema } from "effect"; import { HttpServerRequest, HttpServerResponse } from "effect/http"; import type { HttpServerError } from "effect/http/HttpServerError"; -import { forwardMcpRequest } from "../implementation/mcp-forward.ts"; +import { forwardMcpRequest, refuseTooDeep } from "../implementation/mcp-forward.ts"; import { mcpSessionNotFound, sessionOrganizationHeader, @@ -70,7 +70,9 @@ export const cloudMcp = Effect.gen(function* () { const request = yield* HttpServerRequest.HttpServerRequest; const headers = yield* traceHeaders; const traced = request.modify({ headers: { ...request.headers, ...headers } }); - const response = yield* forwardMcpRequest(traced, (attempt) => timedForward(fetch(attempt))); + const response = yield* refuseTooDeep( + forwardMcpRequest(traced, (attempt) => timedForward(fetch(attempt))), + ); const organization = response.headers[sessionOrganizationHeader]; if (organization !== undefined) yield* Effect.annotateCurrentSpan("executor.organization.id", organization); diff --git a/bun.lock b/bun.lock index afc8616d7c..bdb339ecb3 100644 --- a/bun.lock +++ b/bun.lock @@ -40,7 +40,7 @@ }, "apps/cli": { "name": "executor", - "version": "2.0.0-beta.27", + "version": "2.0.0-beta.28", }, "apps/docs": { "name": "@executor-js/docs", diff --git a/e2e/ci-selection.ts b/e2e/ci-selection.ts index 39a4960275..1eb4e010bd 100644 --- a/e2e/ci-selection.ts +++ b/e2e/ci-selection.ts @@ -597,6 +597,7 @@ const suiteConfigs: Record Promise> = { "prepare-cache.config.ts": () => import("./prepare-cache.config.ts"), "release-archives.config.ts": () => import("./release-archives.config.ts"), "standalone-lockdown.config.ts": () => import("./standalone-lockdown.config.ts"), + "subrequest-depth.config.ts": () => import("./subrequest-depth.config.ts"), "typecheck-runner.config.ts": () => import("./typecheck-runner.config.ts"), "welcome-email.config.ts": () => import("./welcome-email.config.ts"), }; diff --git a/e2e/subrequest-depth.config.ts b/e2e/subrequest-depth.config.ts new file mode 100644 index 0000000000..81f8f427de --- /dev/null +++ b/e2e/subrequest-depth.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; + +/** Runs by hand against a deployed stage: local workerd does not enforce the subrequest depth limit. */ +export default defineConfig({ + test: { + include: ["e2e/tests/cloud-mcp-subrequest-depth.spec.ts"], + fileParallelism: false, + maxWorkers: 1, + testTimeout: 240_000, + hookTimeout: 60_000, + }, +}); diff --git a/e2e/tests/cloud-mcp-subrequest-depth.spec.ts b/e2e/tests/cloud-mcp-subrequest-depth.spec.ts new file mode 100644 index 0000000000..3947f7ef2c --- /dev/null +++ b/e2e/tests/cloud-mcp-subrequest-depth.spec.ts @@ -0,0 +1,256 @@ +/** + * Cloudflare refuses a call into a Worker or Durable Object once its request has entered the + * Workers runtime too many times. A client that calls Cloud's MCP from deep inside its own Worker + * chain reaches that limit at the gateway's call to the session object. The gateway must then say + * why, with a 508 and a JSON-RPC error, instead of an empty 500. + * + * A disposable relay Worker calls itself through a service binding `hops` times, then posts an MCP + * initialize to a deployed stage. The scenario raises `hops` until the relay's own call is refused. + * Each step adds one Worker invocation, so the deepest request the stage still answers is the one + * whose call to the session object was refused, and every shallower one initializes. + * + * Local workerd does not enforce the limit, so this runs by hand against a deployed stage, with + * vault-bound Cloudflare credentials to upload and delete the relay: + * `bunx vitest run --config e2e/subrequest-depth.config.ts`. It needs + * `SUBREQUEST_DEPTH_MCP_URL`, the stage's MCP endpoint (`https://mcp../mcp`), + * `SUBREQUEST_DEPTH_MCP_KEY`, an API key of a synthetic test account on that stage, and + * `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN`. + */ +import { randomBytes } from "node:crypto"; +import { expect, layer } from "@effect/vitest"; +import { Config, Console, Effect, Layer, Redacted, Schedule, Schema } from "effect"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; + +class DepthScenarioFailed extends Schema.TaggedError()("DepthScenarioFailed", { + operation: Schema.String, + status: Schema.optionalKey(Schema.Number), +}) {} + +/** Forwards the MCP request it holds once its own hop count reaches zero; holds no state. */ +const relaySource = ` +export default { + async fetch(request, env) { + if (request.method !== "POST" || request.headers.get("x-relay-key") !== env.RELAY_KEY) + return new Response(null, { status: 404 }); + const url = new URL(request.url); + const hops = Number(url.searchParams.get("hops")); + if (!Number.isSafeInteger(hops) || hops < 0) return new Response(null, { status: 400 }); + const body = await request.text(); + const refused = (error) => + Response.json({ outcome: "refused", message: error instanceof Error ? error.message : String(error) }); + if (hops > 0) { + url.searchParams.set("hops", String(hops - 1)); + try { + return await env.SELF.fetch(url, { method: "POST", headers: request.headers, body }); + } catch (error) { + return refused(error); + } + } + const headers = new Headers(); + for (const name of ["authorization", "accept", "content-type", "mcp-protocol-version"]) { + const value = request.headers.get(name); + if (value !== null) headers.set(name, value); + } + try { + const answer = await fetch(env.TARGET, { method: "POST", headers, body }); + return Response.json({ outcome: "answered", status: answer.status, text: await answer.text() }); + } catch (error) { + return refused(error); + } + }, +}; +`; + +const CloudflareResult = Schema.Struct({ + success: Schema.Boolean, + result: Schema.optional(Schema.Unknown), +}); +const RelayAnswer = Schema.Union([ + Schema.Struct({ + outcome: Schema.Literal("answered"), + status: Schema.Number, + text: Schema.String, + }), + Schema.Struct({ outcome: Schema.Literal("refused"), message: Schema.String }), +]); +const RefusalBody = Schema.fromJsonString( + Schema.Struct({ + jsonrpc: Schema.Literal("2.0"), + id: Schema.Null, + error: Schema.Struct({ code: Schema.Number, message: Schema.String }), + }), +); +const depthLimit = /^Subrequest depth limit exceeded/u; +/** Well past Cloudflare's 32 Worker invocations per request, so the sweep always ends refused. */ +const maximumHops = 64; + +layer(Layer.mergeAll(FetchHttpClient.layer), { excludeTestServices: true })( + "Cloud MCP subrequest depth", + (it) => { + it.effect( + "explains a request refused at the subrequest depth limit instead of failing with an empty 500", + () => + Effect.gen(function* () { + const target = yield* Config.String("SUBREQUEST_DEPTH_MCP_URL"); + expect(new URL(target).protocol).toBe("https:"); + const credential = yield* Config.Redacted("SUBREQUEST_DEPTH_MCP_KEY"); + const account = yield* Config.String("CLOUDFLARE_ACCOUNT_ID"); + const token = yield* Config.Redacted("CLOUDFLARE_API_TOKEN"); + const http = yield* HttpClient.HttpClient; + const workers = `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(account)}/workers`; + const cloudflare = (operation: string, request: HttpClientRequest.HttpClientRequest) => + Effect.scoped( + Effect.gen(function* () { + const response = yield* http.execute( + request.pipe(HttpClientRequest.bearerToken(token)), + ); + const value = yield* response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(CloudflareResult)), + ); + if (!value.success) + return yield* new DepthScenarioFailed({ operation, status: response.status }); + return value.result; + }), + ).pipe( + Effect.mapError((error) => + Schema.is(DepthScenarioFailed)(error) + ? error + : new DepthScenarioFailed({ operation }), + ), + ); + + const name = `e2e-subrequest-depth-${randomBytes(4).toString("hex")}`; + const key = randomBytes(24).toString("hex"); + const upload = (bindings: ReadonlyArray>>) => { + const form = new FormData(); + form.set( + "metadata", + new Blob( + [ + JSON.stringify({ + main_module: "relay.js", + compatibility_date: "2026-09-08", + bindings: [ + { type: "secret_text", name: "RELAY_KEY", text: key }, + { type: "plain_text", name: "TARGET", text: target }, + ...bindings, + ], + }), + ], + { type: "application/json" }, + ), + ); + form.set( + "relay.js", + new Blob([relaySource], { type: "application/javascript+module" }), + "relay.js", + ); + return cloudflare( + "Upload the relay", + HttpClientRequest.put(`${workers}/scripts/${name}`).pipe( + HttpClientRequest.bodyFormData(form), + ), + ); + }; + yield* Effect.acquireRelease(upload([]), () => + cloudflare( + "Delete the relay", + HttpClientRequest.delete(`${workers}/scripts/${name}?force=true`), + ).pipe(Effect.catch(() => Console.error(`Could not delete the relay Worker ${name}.`))), + ); + // A service binding names an existing script, so the relay binds itself on its second upload. + yield* upload([{ type: "service", name: "SELF", service: name }]); + yield* cloudflare( + "Enable the relay's workers.dev route", + yield* HttpClientRequest.post(`${workers}/scripts/${name}/subdomain`).pipe( + HttpClientRequest.bodyJson({ enabled: true, previews_enabled: false }), + ), + ); + const subdomain = yield* cloudflare( + "Read the workers.dev subdomain", + HttpClientRequest.get(`${workers}/subdomain`), + ).pipe( + Effect.flatMap(Schema.decodeUnknownEffect(Schema.Struct({ subdomain: Schema.String }))), + ); + const relay = `https://${name}.${subdomain.subdomain}.workers.dev`; + + const initialize = JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "subrequest-depth", version: "1" }, + }, + }); + const at = (hops: number) => + Effect.scoped( + Effect.gen(function* () { + const response = yield* http.execute( + HttpClientRequest.post(`${relay}/?hops=${hops}`).pipe( + HttpClientRequest.setHeaders({ + "x-relay-key": key, + authorization: `Bearer ${Redacted.value(credential)}`, + accept: "application/json, text/event-stream", + "mcp-protocol-version": "2025-06-18", + }), + HttpClientRequest.bodyText(initialize, "application/json"), + ), + ); + if (response.status !== 200) + return yield* new DepthScenarioFailed({ + operation: "Call the relay", + status: response.status, + }); + return yield* response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(RelayAnswer)), + ); + }), + ).pipe(Effect.provideService(HttpClient.TracerPropagationEnabled, false)); + + // A new workers.dev route takes a few seconds to answer. + const shallowest = yield* at(0).pipe( + Effect.retry({ schedule: Schedule.spaced("2 seconds"), times: 45 }), + ); + expect(shallowest, "The relay reaches the stage and the key initializes").toMatchObject({ + outcome: "answered", + status: 200, + }); + + const answers: Array = [shallowest]; + for (let hops = 1; hops <= maximumHops; hops++) { + const answer = yield* at(hops); + answers.push(answer); + if (answer.outcome === "refused") break; + } + const sweep = answers.map((answer, hops) => + answer.outcome === "answered" + ? { hops, status: answer.status } + : { hops, refused: answer.message }, + ); + const refused = answers.at(-1); + expect(refused?.outcome, JSON.stringify(sweep)).toBe("refused"); + if (refused?.outcome === "refused") + expect(refused.message, "The sweep reached the platform's own limit").toMatch( + depthLimit, + ); + + const answered = answers.flatMap((answer) => + answer.outcome === "answered" ? [answer] : [], + ); + expect(answered.length, JSON.stringify(sweep)).toBe(answers.length - 1); + const deepest = answered.at(-1); + expect( + answered.slice(0, -1).map((answer) => answer.status), + "Every shallower request initializes", + ).toEqual(answered.slice(0, -1).map(() => 200)); + expect(deepest?.status, JSON.stringify(sweep)).toBe(508); + const body = yield* Schema.decodeUnknownEffect(RefusalBody)(deepest?.text ?? ""); + expect(body.error.code).toBe(-32000); + expect(body.error.message).toMatch(depthLimit); + expect(body.error.message).toContain("Call Executor from a shallower point"); + }), + ); + }, +); From 1732e666400a35a615ffac5e812237c7c8993cb8 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:16:21 +0000 Subject: [PATCH 019/127] Export f18fca16 --- apps/hosted/cloud/src/implementation/auth-options.ts | 2 ++ apps/hosted/cloud/src/implementation/emulated-auth.ts | 1 + e2e/tests/cloud-oauth-proxy.spec.ts | 3 +++ 3 files changed, 6 insertions(+) diff --git a/apps/hosted/cloud/src/implementation/auth-options.ts b/apps/hosted/cloud/src/implementation/auth-options.ts index ac5369d0bb..402fb3489f 100644 --- a/apps/hosted/cloud/src/implementation/auth-options.ts +++ b/apps/hosted/cloud/src/implementation/auth-options.ts @@ -262,6 +262,8 @@ export const cloudAuthOptions = ( clientId: settings.googleClientId, clientSecret: Redacted.value(settings.googleClientSecret), includeGrantedScopes: false, + // Without it Google silently reuses an account that already approved Executor. + prompt: "select_account" as const, }, github: { ...redirectURI("github"), diff --git a/apps/hosted/cloud/src/implementation/emulated-auth.ts b/apps/hosted/cloud/src/implementation/emulated-auth.ts index 99f63979e9..c49b096eef 100644 --- a/apps/hosted/cloud/src/implementation/emulated-auth.ts +++ b/apps/hosted/cloud/src/implementation/emulated-auth.ts @@ -28,6 +28,7 @@ export const emulatedSocialProviders = ( scopes: ["openid", "email", "profile"], pkce: true, requireIdTokenVerification: true, + prompt: "select_account", }, { providerId: "github", diff --git a/e2e/tests/cloud-oauth-proxy.spec.ts b/e2e/tests/cloud-oauth-proxy.spec.ts index e2674b85b3..d1c7dbdab8 100644 --- a/e2e/tests/cloud-oauth-proxy.spec.ts +++ b/e2e/tests/cloud-oauth-proxy.spec.ts @@ -76,6 +76,7 @@ const recordedGoogleSignIn = Effect.gen(function* () { return { identity, redirectUri: authorization.searchParams.get("redirect_uri"), + prompt: authorization.searchParams.get("prompt"), // Each Better Auth callback the browser passed through, without its query. callbacks: visited .filter((url) => url.pathname.startsWith("/api/auth/callback/")) @@ -91,6 +92,8 @@ const ownSignIn = Effect.gen(function* () { // The provider returns to the edge's registered callback, which sends the browser to the same // path on the browser origin. The proxy's completion endpoint is never reached. expect(signIn.redirectUri).toBe(`${edge}/api/auth/callback/google`); + // Google lets the person choose among their signed-in accounts instead of reusing one. + expect(signIn.prompt).toBe("select_account"); expect(signIn.callbacks).toEqual([ `${edge}/api/auth/callback/google`, `${browser}/api/auth/callback/google`, From bfd38c3cbfb3465bee5c7fe603b3bf14d7957778 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:17:17 +0000 Subject: [PATCH 020/127] Export 5a39b78c --- .changeset/upgrade-follow-redirects.md | 7 +++++++ .../src/implementation/standalone-install.ts | 4 +++- e2e/tests/local-upgrade.spec.ts | 14 +++++++++++++- 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 .changeset/upgrade-follow-redirects.md diff --git a/.changeset/upgrade-follow-redirects.md b/.changeset/upgrade-follow-redirects.md new file mode 100644 index 0000000000..e88af3b81c --- /dev/null +++ b/.changeset/upgrade-follow-redirects.md @@ -0,0 +1,7 @@ +--- +"executor": patch +--- + +`executor upgrade` on an install from the install script downloads the release again. It checked +the status of GitHub's redirect to each release file instead of following it, so every upgrade +failed with "could not download SHA256SUMS" and changed nothing. diff --git a/apps/local/server/src/implementation/standalone-install.ts b/apps/local/server/src/implementation/standalone-install.ts index 0f00a175a2..b6939cbc31 100644 --- a/apps/local/server/src/implementation/standalone-install.ts +++ b/apps/local/server/src/implementation/standalone-install.ts @@ -169,9 +169,11 @@ export const installVersion = ( const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + // GitHub answers each release download with a redirect to the file, so the status is + // checked on the response the redirects end at. const client = (yield* HttpClient.HttpClient).pipe( - HttpClient.filterStatusOk, HttpClient.followRedirects(5), + HttpClient.filterStatusOk, ); const base = `${yield* downloadBase}/executor@${version}`; const file = releaseArchive(version, platform, arch); diff --git a/e2e/tests/local-upgrade.spec.ts b/e2e/tests/local-upgrade.spec.ts index a9f9cad28c..f2649b0c4b 100644 --- a/e2e/tests/local-upgrade.spec.ts +++ b/e2e/tests/local-upgrade.spec.ts @@ -234,7 +234,7 @@ layer(TestLive, { excludeTestServices: true })("Local upgrade", (it) => { // The registry's dist-tags and the Node range each release declares. let published: Record = {}; let nodeRange = ">=24.14.0"; - // Release downloads by tag, as the public release serves them. + // Release downloads by tag. As on GitHub, each download redirects to the file's asset URL. const downloads = new Map(); const services = yield* Layer.build( HttpRouter.serve( @@ -261,6 +261,18 @@ layer(TestLive, { excludeTestServices: true })("Local upgrade", (it) => { HttpRouter.add( "GET", "/download/:tag/:file", + Effect.gen(function* () { + const { tag, file } = yield* HttpRouter.schemaPathParams( + Schema.Struct({ tag: Schema.String, file: Schema.String }), + ); + return downloads.has(`${decodeURIComponent(tag)}/${file}`) + ? HttpServerResponse.redirect(`/assets/${tag}/${file}`, { status: 302 }) + : HttpServerResponse.empty({ status: 404 }); + }).pipe(Effect.orDie), + ), + HttpRouter.add( + "GET", + "/assets/:tag/:file", Effect.gen(function* () { const { tag, file } = yield* HttpRouter.schemaPathParams( Schema.Struct({ tag: Schema.String, file: Schema.String }), From 441663c07a00d989b51443b24532942256b1484f Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:47:09 +0000 Subject: [PATCH 021/127] Export 6b5abfcb --- AGENTS.md | 8 ++ apps/hosted/self-host/src/executor.ts | 5 +- .../local/server/src/implementation/server.ts | 9 ++- packages/app-source/src/node.ts | 50 ++++++++++-- scripts/releases/README.md | 8 +- scripts/releases/config.ts | 10 ++- scripts/releases/desktop.ts | 4 +- scripts/releases/licenses/git-sources.md | 9 ++- scripts/releases/package.ts | 80 ++++++++++++------- scripts/releases/publish-npm.ts | 5 +- scripts/releases/runtime-bundle.ts | 54 +++++++++++++ scripts/releases/runtime-env.d.mts | 5 +- scripts/releases/runtime-env.mjs | 33 ++------ scripts/releases/standalone-entry.ts | 8 +- scripts/releases/windows-git.ts | 2 + 15 files changed, 204 insertions(+), 86 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 47f587a62d..23ad44d8b7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -294,6 +294,14 @@ Release PRs build and test the Linux and Windows targets only. The macOS targets on manual dispatch, where they are built, signed, notarized and tested. macOS runners cost 5-20x as much as Linux runners, so keep them off pull requests. +A pull request that changes the release paths (the `paths` in +`.github/workflows/release-artifacts.yml`) also runs "Executor releases". Gate its +merge on that workflow as on CI: wait for both +(`ci-wait --workflow CI --workflow "Executor releases"`) and do not merge while +either fails. Its packaging jobs fail when a native archive exceeds the npm budget +(`npmArchiveBudgetBytes` in `scripts/releases/config.ts`); an archive over it cannot be +published, so that failure blocks every later release until it is fixed. + A failed e2e job uploads raw reports and server logs. Product database files, runtime dependencies and private `actors.json` sessions are excluded. Reports use Vitest's final result, including setup and cleanup failures. diff --git a/apps/hosted/self-host/src/executor.ts b/apps/hosted/self-host/src/executor.ts index 5c4b305106..8368183403 100644 --- a/apps/hosted/self-host/src/executor.ts +++ b/apps/hosted/self-host/src/executor.ts @@ -1,5 +1,5 @@ /** Native development host resources. The packaged product uses workerd bindings. */ -import { nativeRepositories } from "@executor-js/app-source/node"; +import { nativeRepositories, systemGit } from "@executor-js/app-source/node"; import { filesystemBlobStore, workerdApps } from "@executor-js/sdk/node/workerd"; import type { HostEgress } from "@executor-js/utils/url-policy"; import { Config, Effect, Option, Path } from "effect"; @@ -40,7 +40,8 @@ export const selfHostExecutor = (egress: HostEgress) => return { ...host, blobs, - repositories: nativeRepositories(path.resolve(directory, "repositories")), + // The image installs Git on PATH. + repositories: nativeRepositories(path.resolve(directory, "repositories"), systemGit), }; }), ); diff --git a/apps/local/server/src/implementation/server.ts b/apps/local/server/src/implementation/server.ts index 95874a3de3..1d41ccb31e 100644 --- a/apps/local/server/src/implementation/server.ts +++ b/apps/local/server/src/implementation/server.ts @@ -90,7 +90,7 @@ import { webFiles } from "./web.ts"; import { withHostPipeline } from "@executor-js/dashboard-start/in-process"; import { layerBuildHeader } from "@executor-js/dashboard-start/build-header"; import { LocalExecutorApi, localManagementDocument } from "../contracts/management.ts"; -import { nativeRepositories } from "@executor-js/app-source/node"; +import { bundledGit, nativeRepositories, systemGit } from "@executor-js/app-source/node"; import { feedbackDisabled } from "@executor-js/telemetry/product-analytics"; import { LocalFeedbackApi } from "../contracts/feedback.ts"; import { localAnalytics, observeLocalExecutor, scheduleAnalytics } from "./product-analytics.ts"; @@ -149,7 +149,12 @@ export const localApi = ( Config.withDefault(hostedExecutorOrigin), ), ); - const repositories = nativeRepositories(path.join(directory, "repositories")); + // A release names the Git it bundles; a source checkout uses the Git on PATH. + const git = Option.match( + yield* Config.NonEmptyString("EXECUTOR_GIT_DIRECTORY").pipe(Config.option), + { onNone: () => systemGit, onSome: bundledGit }, + ); + const repositories = nativeRepositories(path.join(directory, "repositories"), git); const server = yield* Scope.Scope; const evaluation = yield* declarationConfig; // MCP grants are checked again before each event delivery; OAuth starts after the executor. diff --git a/packages/app-source/src/node.ts b/packages/app-source/src/node.ts index 75b77cdb07..45a0b002dd 100644 --- a/packages/app-source/src/node.ts +++ b/packages/app-source/src/node.ts @@ -1,13 +1,46 @@ /** Native Git process and filesystem lifetime for Node and Bun hosts. */ import * as NodeServices from "@effect/platform-node/NodeServices"; +import { join } from "node:path"; import { Effect, Exit, FileSystem, Path, Scope, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import * as PlatformError from "effect/PlatformError"; import { SourceError, type RepositoryBackend } from "@executor-js/sdk/core"; import { gitRepositories } from "./implementation/git-repositories.ts"; +/** The Git a host runs: its executable and what its processes need beside the host's environment. */ +export interface NativeGit { + readonly executable: string; + readonly environment: Readonly>; +} + +/** The Git on PATH, as a server image or a development checkout provides it. */ +export const systemGit: NativeGit = { executable: "git", environment: {} }; + +/** + * A Git distribution laid out as dugite's, such as the copy a release bundles. Its environment + * reaches only the Git processes this host starts, so processes Executor starts for other + * purposes keep the user's PATH and Git. On Windows `cmd/git.exe` puts MinGit's own directories + * on its PATH. + */ +export const bundledGit = (directory: string): NativeGit => + process.platform === "win32" + ? { + executable: join(directory, "cmd", "git.exe"), + environment: { GIT_EXEC_PATH: join(directory, "mingw64", "libexec", "git-core") }, + } + : { + executable: join(directory, "bin", "git"), + environment: { + GIT_EXEC_PATH: join(directory, "libexec", "git-core"), + GIT_TEMPLATE_DIR: join(directory, "share", "git-core", "templates"), + // A Linux Git built for one prefix resolves its files from PREFIX when moved. + ...(process.platform === "linux" ? { PREFIX: directory } : {}), + }, + }; + /** Execute without a shell. Git diagnostics can contain source or paths and are discarded. */ -const git = ( +const run = ( + git: NativeGit, args: ReadonlyArray, input?: Uint8Array, environment?: Record, @@ -19,14 +52,14 @@ const git = ( // Receive-pack adds a quarantine directory beneath the repository. Let // Git use Windows extended paths for those deeper object filenames. ChildProcess.make( - "git", + git.executable, ["-c", "core.hooksPath=/dev/null", "-c", "core.longpaths=true", ...args], { stdin: input === undefined ? "ignore" : Stream.succeed(input), stdout: "pipe", stderr: "ignore", extendEnv: true, - env: { ...gitEnvironment, ...environment }, + env: { ...gitEnvironment, ...git.environment, ...environment }, }, ), ); @@ -61,6 +94,7 @@ const gitEnvironment = { * out. The process lives until its output is read or the response is cancelled. */ const httpBackend = ( + git: NativeGit, environment: Record, body: ReadableStream | null, ) => @@ -70,7 +104,7 @@ const httpBackend = ( const process = yield* spawner .spawn( ChildProcess.make( - "git", + git.executable, ["-c", "core.hooksPath=/dev/null", "-c", "core.longpaths=true", "http-backend"], { stdin: @@ -90,7 +124,7 @@ const httpBackend = ( stdout: "pipe", stderr: "ignore", extendEnv: true, - env: { ...gitEnvironment, ...environment }, + env: { ...gitEnvironment, ...git.environment, ...environment }, }, ), ) @@ -104,12 +138,12 @@ const httpBackend = ( }).pipe(Effect.mapError(() => new SourceError({ reason: "git" }))); /** Lazy repositories with scoped native processes and temporary indexes. */ -export const nativeRepositories = (directory: string): RepositoryBackend => { +export const nativeRepositories = (directory: string, git: NativeGit): RepositoryBackend => { const provide = Effect.provide(NodeServices.layer); return gitRepositories({ directory, - git: (args, input, environment) => git(args, input, environment).pipe(provide), - httpBackend: (environment, body) => httpBackend(environment, body).pipe(provide), + git: (args, input, environment) => run(git, args, input, environment).pipe(provide), + httpBackend: (environment, body) => httpBackend(git, environment, body).pipe(provide), createDirectory: (path) => FileSystem.FileSystem.use((fs) => fs.makeDirectory(path, { recursive: true, mode: 0o700 }), diff --git a/scripts/releases/README.md b/scripts/releases/README.md index bbebcfb913..84919cb09d 100644 --- a/scripts/releases/README.md +++ b/scripts/releases/README.md @@ -76,9 +76,11 @@ The npm launcher is under `.local/releases//wrapper/` and uses exact native version aliases. These build commands do not publish anything. The app compiler and framework are prepared for workerd at build time. No runtime download is needed. Each compressed native archive must fit within -180 MiB before publication. This is a release budget with margin, not a -documented npm registry limit. External source maps stay in the separate CI -artifact for release diagnostics. +178 MiB before publication (`npmArchiveBudgetBytes` in `config.ts`). npm +documents no limit; the budget stays below the largest archive it has accepted +from Executor. Most of each archive is the executable, workerd and the bundled +Git, which keeps only what the runtime runs (`keptGit` in `runtime-bundle.ts`). +External source maps stay in the separate CI artifact for release diagnostics. Stock Bun lets any local process run its own code inside a compiled executable, through `BUN_OPTIONS`, `BUN_BE_BUN`, `BUN_INSPECT` and similar variables that diff --git a/scripts/releases/config.ts b/scripts/releases/config.ts index 5b955634ee..a41d494b14 100644 --- a/scripts/releases/config.ts +++ b/scripts/releases/config.ts @@ -13,8 +13,14 @@ export { type ReleaseChannel, } from "@executor-js/utils/release-version"; -/** Conservative compressed archive budget, checked before npm receives any upload. */ -export const npmArchiveBudgetBytes = 180 * 1024 * 1024; +/** + * The compressed archive budget, checked before npm receives any upload. npm documents no size + * limit. It refused a 298.9 MB archive with HTTP 413, and others report refusals from about + * 185 MB; the largest Executor archive it has accepted is 187,186,037 bytes. The budget stays + * below that. + */ +export const npmArchiveBudgetMiB = 178; +export const npmArchiveBudgetBytes = npmArchiveBudgetMiB * 1024 * 1024; /** * Native platforms supported by the packaged runtime. The full CLI suite runs against the packaged diff --git a/scripts/releases/desktop.ts b/scripts/releases/desktop.ts index 284f5fed9b..61586f7eca 100644 --- a/scripts/releases/desktop.ts +++ b/scripts/releases/desktop.ts @@ -76,8 +76,8 @@ const build = Effect.gen(function* () { path.join(runtime, "desktop-server.mjs"), `import { flushCompileCache } from "node:module"; import { enablePrivateCompileCache } from "./compile-cache.mjs"; -import { packagedRuntimeEnvironment } from "./runtime-env.mjs"; -Object.assign(process.env, packagedRuntimeEnvironment(process.env)); +import { useBundledGit } from "./runtime-env.mjs"; +useBundledGit(); process.env.EXECUTOR_BUILD_VERSION = ${JSON.stringify(version)}; ${yield* launcherAnalytics}enablePrivateCompileCache(); await import("./runtime/desktop.mjs"); diff --git a/scripts/releases/licenses/git-sources.md b/scripts/releases/licenses/git-sources.md index 26c4def92f..6b673ac969 100644 --- a/scripts/releases/licenses/git-sources.md +++ b/scripts/releases/licenses/git-sources.md @@ -25,9 +25,12 @@ Git source tree and the binary package includes `licenses/git-GPL-2.0.txt`. The native project also bundles Git LFS and Git Credential Manager. Their versions and binary checksums are recorded in its `dependencies.json`. -`git-lfs-notices.txt` adds Git LFS's license, Go runtime license and the notices -from its pinned Go modules. Git Credential Manager's license and third-party -notices remain in the distributed Git tree. +Executor's binaries leave both out, with the network transports and scripted +tools: they keep only the `git` executable, `git http-backend`, the libraries +those load and `git init`'s templates (`keptGit` in +`scripts/releases/runtime-bundle.ts`). `git-lfs-notices.txt` adds Git LFS's +license, Go runtime license and the notices from its pinned Go modules, for +releases that shipped it. Windows is a separate build: dugite repackages MinGit rather than compiling the POSIX source. Its source packages retain upstream code, patches, `PKGBUILD` recipes diff --git a/scripts/releases/package.ts b/scripts/releases/package.ts index b8eff75344..8bda78acc5 100644 --- a/scripts/releases/package.ts +++ b/scripts/releases/package.ts @@ -1,12 +1,13 @@ /** Build the platform's release install and its npm package. Nothing is published. */ import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { Config, Console, Effect, FileSystem, Path } from "effect"; +import { Config, Console, Effect, FileSystem, Path, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { bundleLocalRuntime } from "./runtime-bundle.ts"; import { nativePlatform, npmArchiveBudgetBytes, + npmArchiveBudgetMiB, platformArchive, platformVersion, release, @@ -14,6 +15,7 @@ import { import { installWindowsGitHttpBackend } from "./windows-git.ts"; import { launcherAnalytics } from "./analytics.ts"; import { buildStandalone } from "./standalone.ts"; +import { bundledGit } from "../../packages/app-source/src/node.ts"; const build = Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -102,38 +104,58 @@ const build = Effect.gen(function* () { // launch after the first skips recompiling the bundled server (about 150 ms). yield* fs.writeFileString( path.join(stage, "bin.mjs"), - `#!/usr/bin/env node\nimport { flushCompileCache } from "node:module";\nimport { homedir } from "node:os";\nimport { join } from "node:path";\nimport { enablePrivateCompileCache } from "./compile-cache.mjs";\nimport { packagedRuntimeEnvironment } from "./runtime-env.mjs";\nObject.assign(process.env, packagedRuntimeEnvironment(process.env));\nprocess.env.EXECUTOR_DATA_DIR ??= join(homedir(), ".executor", "v2", "cli");\nprocess.env.EXECUTOR_BUILD_VERSION = ${JSON.stringify(version)};\n${yield* launcherAnalytics}enablePrivateCompileCache();\nawait import("./runtime/cli.mjs");\nflushCompileCache();\n`, + `#!/usr/bin/env node\nimport { flushCompileCache } from "node:module";\nimport { homedir } from "node:os";\nimport { join } from "node:path";\nimport { enablePrivateCompileCache } from "./compile-cache.mjs";\nimport { useBundledGit } from "./runtime-env.mjs";\nuseBundledGit();\nprocess.env.EXECUTOR_DATA_DIR ??= join(homedir(), ".executor", "v2", "cli");\nprocess.env.EXECUTOR_BUILD_VERSION = ${JSON.stringify(version)};\n${yield* launcherAnalytics}enablePrivateCompileCache();\nawait import("./runtime/cli.mjs");\nflushCompileCache();\n`, ); yield* fs.chmod(path.join(stage, "bin.mjs"), 0o755); yield* fs.copyFile(path.join(root, "scripts/releases/README.md"), path.join(stage, "README.md")); yield* fs.copyFile(path.join(root, "apps/cli/LICENSE"), path.join(stage, "LICENSE")); yield* installWindowsGitHttpBackend(stage); - yield* run( - "node", - [ - "--input-type=module", - "--eval", - `import { spawnSync } from "node:child_process"; -import { packagedRuntimeEnvironment } from "./runtime-env.mjs"; -const result = spawnSync("git", ["--version"], { - env: packagedRuntimeEnvironment({ ...process.env, PATH: "" }), - stdio: "inherit", -}); -if (result.error) throw result.error; -if (result.status !== 0) process.exit(1); -const backend = spawnSync("git", ["http-backend"], { - env: packagedRuntimeEnvironment({ ...process.env, PATH: "", GIT_PROJECT_ROOT: process.cwd(), - GIT_HTTP_EXPORT_ALL: "1", PATH_INFO: "/executor-release-probe-missing.git/info/refs", - REQUEST_METHOD: "GET", QUERY_STRING: "service=git-upload-pack", SERVER_PROTOCOL: "HTTP/1.1" }), - encoding: "utf8", -}); -if (backend.error) throw backend.error; -if (backend.status !== 0 || !backend.stdout.includes("Status: 404")) { - throw new Error("The bundled Git HTTP backend did not return the expected missing-repository response"); -}`, - ], - stage, + // The bundled Git runs with no other Git on PATH and answers Git's HTTP protocol. + const git = bundledGit(path.join(stage, "node_modules/dugite/git")); + const inherited = Object.fromEntries( + Object.entries(process.env).filter( + (entry): entry is [string, string] => + entry[1] !== undefined && entry[0].toUpperCase() !== "PATH", + ), ); + const probe = (args: readonly string[], environment: Record = {}) => + Effect.scoped( + Effect.gen(function* () { + const child = yield* processes.spawn( + ChildProcess.make(git.executable, args, { + cwd: stage, + extendEnv: false, + env: { ...inherited, PATH: "", ...git.environment, ...environment }, + stdin: "ignore", + stdout: "pipe", + stderr: "inherit", + }), + ); + const [code, stdout] = yield* Effect.all( + [child.exitCode, child.stdout.pipe(Stream.decodeText(), Stream.mkString)], + { concurrency: 2 }, + ); + return { code: Number(code), stdout }; + }), + ); + const started = yield* probe(["--version"]); + if (started.code !== 0 || !started.stdout.startsWith("git version ")) + return yield* Effect.die(new Error(`The bundled Git did not start: ${started.stdout}`)); + yield* Console.log(started.stdout.trim()); + const backend = yield* probe(["http-backend"], { + GIT_PROJECT_ROOT: stage, + GIT_HTTP_EXPORT_ALL: "1", + PATH_INFO: "/executor-release-probe-missing.git/info/refs", + REQUEST_METHOD: "GET", + QUERY_STRING: "service=git-upload-pack", + SERVER_PROTOCOL: "HTTP/1.1", + }); + if (backend.code !== 0 || !backend.stdout.includes("Status: 404")) + return yield* Effect.die( + new Error( + "The bundled Git HTTP backend did not return the expected missing-repository response", + ), + ); // The npm package is the release install: the executable and the files beside it. The staged // runtime stays for the desktop app, which runs it with its own Node. // Signing is a deliberate choice per build, and notarizing implies signing. A published macOS @@ -156,7 +178,9 @@ if (backend.status !== 0 || !backend.stdout.includes("Status: 404")) { ); if (bytes > budgetBytes) return yield* Effect.die( - new Error(`npm archive exceeds the 180 MiB release budget: ${bytes} bytes`), + new Error( + `npm archive exceeds the ${npmArchiveBudgetMiB} MiB release budget: ${bytes} bytes`, + ), ); yield* Console.log(`Native artifact: ${output} (${(bytes / 1024 / 1024).toFixed(1)} MiB)`); }); diff --git a/scripts/releases/publish-npm.ts b/scripts/releases/publish-npm.ts index 47ac579b24..86c18bc7b7 100644 --- a/scripts/releases/publish-npm.ts +++ b/scripts/releases/publish-npm.ts @@ -20,6 +20,7 @@ import { FetchHttpClient, HttpClient } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { npmArchiveBudgetBytes, + npmArchiveBudgetMiB, platformArchive, platformVersion, platforms, @@ -94,7 +95,9 @@ NodeRuntime.runMain( return yield* Effect.die(new Error(`Missing ${pkg.file}`)); if (Number((yield* fs.stat(pkg.file)).size) > npmArchiveBudgetBytes) return yield* Effect.die( - new Error(`npm archive exceeds the 180 MiB release budget: ${pkg.file}`), + new Error( + `npm archive exceeds the ${npmArchiveBudgetMiB} MiB release budget: ${pkg.file}`, + ), ); const integrity = yield* integrityOf(pkg.file); const response = yield* http.get(`${registry}/executor/${pkg.version}`); diff --git a/scripts/releases/runtime-bundle.ts b/scripts/releases/runtime-bundle.ts index d2922b4289..227ab2acab 100644 --- a/scripts/releases/runtime-bundle.ts +++ b/scripts/releases/runtime-bundle.ts @@ -29,6 +29,59 @@ const installedPackages = [ "typescript", ] as const; +/** + * The files of dugite's Git distribution the runtime keeps, by path relative to it. The bundled Git + * serves app repositories on this machine, through plumbing commands and `git http-backend`, and + * never reaches a remote. So the Git Credential Manager, Git LFS, network transports and scripted + * tools dugite also ships are left out. What stays is the `git` executable, `git http-backend` + * (which windows-git.ts adds on Windows after staging), the libraries they load, the templates + * `git init` copies and the licenses of what remains. Symbolic links to `git`, its argv[0] + * aliases such as `git-upload-pack`, stay as well. + */ +const keptGit = (platform: string): ReadonlyArray => + platform === "win32" + ? [ + /^LICENSE\.txt$/u, + /^cmd\/git\.exe$/u, + /^mingw64\/bin\/(git\.exe|libiconv-2\.dll|libintl-8\.dll|libpcre2-8-0\.dll|libwinpthread-1\.dll|zlib1\.dll)$/u, + /^mingw64\/libexec\/git-core\/git-http-backend\.exe$/u, + /^mingw64\/share\/git-core\/templates\//u, + /^mingw64\/share\/licenses\//u, + ] + : [ + /^bin\/git$/u, + /^libexec\/git-core\/(git|git-http-backend)$/u, + /^share\/git-core\/templates\//u, + ]; + +/** Remove everything from the Git distribution at `directory` that `keptGit` does not keep. */ +const pruneGit = (directory: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const kept = keptGit(process.platform); + const prune = (current: string): Effect.Effect => + Effect.gen(function* () { + let empty = true; + for (const name of yield* fs.readDirectory(current)) { + const file = path.join(current, name); + const relative = path.relative(directory, file).split(path.sep).join("/"); + const info = yield* Effect.tryPromise(() => lstat(file)); + if (info.isDirectory()) { + if (yield* prune(file)) yield* fs.remove(file, { recursive: true }); + else empty = false; + } else if ( + kept.some((pattern) => pattern.test(relative)) || + (info.isSymbolicLink() && (yield* fs.readLink(file)) === "git") + ) + empty = false; + else yield* fs.remove(file); + } + return empty; + }).pipe(Effect.orDie); + yield* prune(directory); + }); + /** The source substitutions both runtime builds share. */ const sourcePlugin = (root: string) => Effect.gen(function* () { @@ -373,6 +426,7 @@ export const bundleLocalRuntime = (root: string, stage: string) => ] as const) { yield* copyPackage(yield* requiredPackage(name, owner)); } + yield* pruneGit(path.join(stage, "node_modules/dugite/git")); // npm deliberately omits symbolic and hard links. Git uses argv[0] aliases; // preserve those commands with shell builtins instead of duplicating its binary. const materializeLinks = ( diff --git a/scripts/releases/runtime-env.d.mts b/scripts/releases/runtime-env.d.mts index c2a5f0f571..e2ac05e715 100644 --- a/scripts/releases/runtime-env.d.mts +++ b/scripts/releases/runtime-env.d.mts @@ -1,5 +1,2 @@ /** Types for runtime-env.mjs, which the npm package and desktop import as plain JavaScript. */ -export declare function packagedRuntimeEnvironment( - environment: Readonly>, - gitDirectory?: string, -): Record; +export declare function useBundledGit(): void; diff --git a/scripts/releases/runtime-env.mjs b/scripts/releases/runtime-env.mjs index 62484aba17..f9c4880c4d 100644 --- a/scripts/releases/runtime-env.mjs +++ b/scripts/releases/runtime-env.mjs @@ -1,30 +1,11 @@ -/** Resolve packaged native tools at the process boundary, independent of a user's Git install. */ -import { delimiter, dirname } from "node:path"; -import { resolveEmbeddedGitDir, resolveGitExecPath, setupEnvironment } from "dugite"; +/** Name the packaged Git to the Executor server, independent of a user's Git install. */ +import { resolveEmbeddedGitDir } from "dugite"; /** - * Preserve the caller's environment while selecting the artifact's Git binary and helper programs. - * `gitDirectory` is the bundled Git: dugite's own copy by default, or the copy beside a release - * executable, whose bundled dugite cannot locate it from its module path. + * Point the server at the Git this package bundles, dugite's own copy. Only the server's Git host + * runs it, with the environment it needs; processes Executor starts for other purposes keep the + * caller's PATH and Git. */ -export function packagedRuntimeEnvironment(environment, gitDirectory = resolveEmbeddedGitDir()) { - const inherited = Object.fromEntries( - Object.entries(environment).map(([name, value]) => [ - process.platform === "win32" ? name.toUpperCase() : name, - value, - ]), - ); - const { env, gitLocation } = setupEnvironment( - { - LOCAL_GIT_DIRECTORY: gitDirectory, - GIT_EXEC_PATH: resolveGitExecPath(gitDirectory, ""), - }, - inherited, - ); - return { - ...env, - PATH: [dirname(gitLocation), env.PATH] - .filter((value) => value !== undefined && value !== "") - .join(delimiter), - }; +export function useBundledGit() { + process.env.EXECUTOR_GIT_DIRECTORY = resolveEmbeddedGitDir(); } diff --git a/scripts/releases/standalone-entry.ts b/scripts/releases/standalone-entry.ts index 0df4b1ea8d..acb2f817ff 100644 --- a/scripts/releases/standalone-entry.ts +++ b/scripts/releases/standalone-entry.ts @@ -4,7 +4,6 @@ */ import { homedir } from "node:os"; import { join } from "node:path"; -import { packagedRuntimeEnvironment } from "./runtime-env.mjs"; import type { StandaloneRelease } from "./runtime-bundle.ts"; /** The install directory, from the bundle's banner. */ @@ -12,10 +11,9 @@ declare const __executorRoot: string; /** The release identity, defined at build time. */ declare const executorRelease: StandaloneRelease; -Object.assign( - process.env, - packagedRuntimeEnvironment(process.env, join(__executorRoot, "runtime", "git")), -); +// The server's Git host runs the Git beside the executable, which the bundled dugite cannot locate +// from its module path. Other processes keep the caller's PATH and Git. +process.env.EXECUTOR_GIT_DIRECTORY = join(__executorRoot, "runtime", "git"); process.env.EXECUTOR_DATA_DIR ??= join(homedir(), ".executor", "v2", "cli"); process.env.EXECUTOR_BUILD_VERSION = executorRelease.version; // The npm launcher names itself; started directly, the executable's own location shows how it diff --git a/scripts/releases/windows-git.ts b/scripts/releases/windows-git.ts index 4e383b1d05..5206c916c8 100644 --- a/scripts/releases/windows-git.ts +++ b/scripts/releases/windows-git.ts @@ -47,5 +47,7 @@ export const installWindowsGitHttpBackend = (stage: string) => const extracted = path.join(temporary, helper); if (hash(yield* fs.readFile(extracted)) !== helperHash) return yield* new WindowsGitFailed({ stage: "helper" }); + // The staged Git keeps nothing else in libexec/git-core (keptGit in runtime-bundle.ts). + yield* fs.makeDirectory(path.dirname(path.join(git, helper)), { recursive: true }); yield* fs.copyFile(extracted, path.join(git, helper)); }).pipe(Effect.provide(FetchHttpClient.layer), Effect.scoped); From 4a06c865807c1688fad50ad160eab3db2d7f2673 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sat, 24 Oct 2026 10:51:54 +0000 Subject: [PATCH 022/127] Export fba59aba --- .changeset/skills-input-schema.md | 10 ++ e2e/check-boundary.ts | 1 + e2e/test-plan.ts | 14 ++ e2e/tests/local-skills.spec.ts | 162 ++++++++++++++++++ .../executor/skills/executor/SKILL.md | 2 +- .../src/implementation/executor-intro.gen.ts | 2 +- .../executor-skill-digests.gen.ts | 2 +- packages/mcp/src/contracts/browser-tools.ts | 2 +- packages/mcp/src/contracts/execute.ts | 4 +- packages/mcp/src/contracts/skills.ts | 124 +++++++++----- packages/mcp/src/implementation/server.ts | 24 +-- packages/mcp/src/implementation/skills.ts | 9 +- 12 files changed, 300 insertions(+), 56 deletions(-) create mode 100644 .changeset/skills-input-schema.md diff --git a/.changeset/skills-input-schema.md b/.changeset/skills-input-schema.md new file mode 100644 index 0000000000..a8dc3abc87 --- /dev/null +++ b/.changeset/skills-input-schema.md @@ -0,0 +1,10 @@ +--- +"@executor-js/mcp": patch +"@executor-js/app-templates": patch +--- + +The MCP `skills` tool advertises the input it accepts: every input except `{}` +needs `app`, stated as `dependentRequired` in its input schema, which is +generated from the schema the tool decodes with. The `execute` descriptions and +the Executor app's entry skill give the exact call, +`skills({app: "executor", name: "executor"})`. diff --git a/e2e/check-boundary.ts b/e2e/check-boundary.ts index 52d5ffca49..069f9ff916 100644 --- a/e2e/check-boundary.ts +++ b/e2e/check-boundary.ts @@ -14,6 +14,7 @@ const allowed = new Set([ "@modelcontextprotocol/sdk/client/index.js", "@modelcontextprotocol/sdk/types.js", "@modelcontextprotocol/sdk/client/streamableHttp.js", + "@modelcontextprotocol/sdk/validation/cfworker", "vitest", "vitest/config", "effect/http", diff --git a/e2e/test-plan.ts b/e2e/test-plan.ts index 75f9bc07d9..97097e3e3e 100644 --- a/e2e/test-plan.ts +++ b/e2e/test-plan.ts @@ -7753,6 +7753,20 @@ export const scenarios = plan({ ), }, }, + localSkillsInputSchema: { + file: "local-skills.spec.ts", + title: + "MCP skills accept exactly the inputs their advertised schema accepts, and every skills call the tool descriptions and Executor skills show succeeds", + targets: { + local: scheduled, + "self-host": na( + "Every product serves the same skills tool, descriptions and Executor skills from packages/mcp and app-templates; local reaches all three modes with one bearer.", + ), + cloud: na( + "Every product serves the same skills tool, descriptions and Executor skills from packages/mcp and app-templates; local reaches all three modes with one bearer.", + ), + }, + }, skillReadTelemetry: { fixtures: "actors", managementApps: ["owner"], diff --git a/e2e/tests/local-skills.spec.ts b/e2e/tests/local-skills.spec.ts index a6fd967df4..06f7f81c61 100644 --- a/e2e/tests/local-skills.spec.ts +++ b/e2e/tests/local-skills.spec.ts @@ -10,6 +10,7 @@ import { Evidence } from "../support/evidence.ts"; import { appsManifest } from "../support/apps-release.ts"; import { connectAccountsThroughOtherApp } from "../support/app-accounts.ts"; import { wholeStringInputPattern } from "../support/mcp-input-patterns.ts"; +import { CfWorkerJsonSchemaValidator } from "@modelcontextprotocol/sdk/validation/cfworker"; const App = Schema.Struct({ id: Schema.String, @@ -32,6 +33,27 @@ const Document = Schema.Struct({ version: Schema.String, files: Schema.Array(Schema.String), }); +const Listed = Schema.Struct({ + tools: Schema.Array( + Schema.Struct({ + name: Schema.String, + description: Schema.optional(Schema.String), + inputSchema: Schema.Record(Schema.String, Schema.Unknown), + }), + ), +}); +const SkillsInput = Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)); +/** + * The `skills` inputs a text shows. The `skills` description shows them as bare objects; every + * other text shows a concrete call as `skills({...})`. Keys may be unquoted and strings may use + * single quotes, as in the descriptions. + */ +const skillsExamples = (text: string, bare: boolean) => + [...text.matchAll(bare ? /\{[^{}]*\}/g : /skills\((\{[^{}]*\})\)/g)].map((match) => + (match[1] ?? match[0]) + .replaceAll("'", '"') + .replace(/([{,]\s*)([A-Za-z_$][\w$]*)\s*:/g, '$1"$2":'), + ); const files = (version: string) => [ { path: "index.ts", @@ -421,4 +443,144 @@ return { sameAccount: target.account === read.id && read.id === updated.id, labe }).pipe(Effect.provide(McpClient.layer)), ), ); + it.effect(scenarios.localSkillsInputSchema.title, (context) => + withCase( + context, + Effect.gen(function* () { + const target = yield* Target, + mcp = yield* McpClient; + // MCP tool input schemas use JSON Schema 2020-12, which this validator implements. + const validator = new CfWorkerJsonSchemaValidator(); + const client = yield* mcp.connect(target.apiKey, "skills-input-schema"); + const call = (step: string, input: Record) => + client.use(step, (client, signal) => + client.callTool({ name: "skills", arguments: input }, undefined, { signal }), + ); + // Every text an agent reads before calling skills: each mode's tool descriptions and + // the server instructions, which are the Executor app's entry skill. + const texts: { readonly source: string; readonly examples: ReadonlyArray }[] = []; + let advertised: Record | undefined; + for (const mode of ["model", "native", "browser"] as const) { + const connection = yield* mcp.connect(target.apiKey, `skills-guidance-${mode}`, { + mode, + }); + const listed = yield* Schema.decodeUnknownEffect(Listed)( + yield* connection.use(`List the ${mode} mode tools`, (client) => client.listTools()), + ); + for (const tool of listed.tools) { + if (tool.name === "skills") advertised ??= tool.inputSchema; + texts.push({ + source: `${mode} ${tool.name} description`, + examples: skillsExamples(tool.description ?? "", tool.name === "skills"), + }); + } + const instructions = yield* connection.use( + `Read the ${mode} mode instructions`, + (client) => Promise.resolve(client.getInstructions() ?? ""), + ); + texts.push({ + source: `${mode} instructions`, + examples: skillsExamples(instructions, false), + }); + } + if (advertised === undefined) + return yield* Effect.die("Every MCP mode must list the skills tool"); + const accepts = validator.getValidator(advertised); + + // The advertised schema and the runtime agree on every read shape, and both refuse a + // document or reference without its app or skill. + const index = yield* Schema.decodeUnknownEffect(Index)( + (yield* call("List every skill", {})).structuredContent, + ); + const guide = index.skills.find( + (skill) => skill.name === "executor" && skill.app.slug === "executor", + ); + if (guide === undefined) + return yield* Effect.die("The Executor app must publish its entry skill"); + const shapes: ReadonlyArray, boolean]> = [ + [{}, true], + [{ app: "executor" }, true], + [{ app: "executor", name: "executor" }, true], + [{ app: "executor", name: "executor", file: "feedback.md" }, true], + [{ name: "executor" }, false], + [{ name: "executor", version: guide.version }, false], + [{ revision: guide.revision }, false], + [{ file: "feedback.md" }, false], + [{ app: "executor", file: "feedback.md" }, false], + ]; + for (const [input, valid] of shapes) { + const result = yield* call(`Call skills with ${JSON.stringify(input)}`, input); + expect({ + input, + schema: accepts(input).valid, + runtime: result.isError !== true, + }).toEqual({ input, schema: valid, runtime: valid }); + } + const refused = yield* call("Read a skill without its app", { name: "executor" }); + // The refusal names the call that works. + expect(refused.content).toEqual([ + { + type: "text", + text: expect.stringContaining('{app: "executor", name: "executor"}'), + }, + ]); + + // Every Executor app skill document and reference: the guidance agents read next. + const listed = yield* Schema.decodeUnknownEffect(Index)( + (yield* call("List the Executor app's skills", { app: guide.app.slug })) + .structuredContent, + ); + for (const skill of listed.skills) { + const document = yield* Schema.decodeUnknownEffect(Document)( + (yield* call(`Read the ${skill.name} skill`, { app: skill.app.slug, name: skill.name })) + .structuredContent, + ); + texts.push({ + source: `${skill.name}/SKILL.md`, + examples: skillsExamples(document.content, false), + }); + for (const file of document.files.filter((file) => file !== "SKILL.md")) { + const reference = yield* Schema.decodeUnknownEffect(Document)( + (yield* call(`Read ${skill.name}/${file}`, { + app: skill.app.slug, + name: skill.name, + file, + version: skill.version, + revision: skill.revision, + })).structuredContent, + ); + texts.push({ + source: `${skill.name}/${file}`, + examples: skillsExamples(reference.content, false), + }); + } + } + // The texts that send agents to skills give a call they can make. + for (const source of [ + "model skills description", + "model execute description", + "native execute description", + "browser execute description", + "model instructions", + ]) + expect( + texts.find((text) => text.source === source)?.examples.length ?? 0, + source, + ).toBeGreaterThan(0); + const examples = new Map(); + for (const text of texts) + for (const example of text.examples) + if (!examples.has(example)) examples.set(example, text.source); + for (const [example, source] of examples) { + const input = yield* Schema.decodeUnknownEffect(SkillsInput)(example); + expect(accepts(input).valid, `${source}: ${example}`).toBe(true); + const result = yield* call(`Run the ${source} example ${example}`, input); + expect( + result.isError, + `${source}: ${example} ${JSON.stringify(result.content)}`, + ).not.toBe(true); + } + }).pipe(Effect.provide(McpClient.layer)), + ), + ); }); diff --git a/packages/app-templates/executor/skills/executor/SKILL.md b/packages/app-templates/executor/skills/executor/SKILL.md index 5dc1ed5741..686465e0d4 100644 --- a/packages/app-templates/executor/skills/executor/SKILL.md +++ b/packages/app-templates/executor/skills/executor/SKILL.md @@ -14,7 +14,7 @@ slug as its namespace. Credentials go only through Executor's secure connection links, never through chat or source code. Some calls pause for user approval. This is the Executor app's `executor` skill. Read its other skills with -`skills`, such as `{app: "executor", name: "code-mode"}`; `{}` lists them all. +`skills({app: "executor", name: "code-mode"})`; `skills({})` lists them all. ## Find and call tools diff --git a/packages/app-templates/src/implementation/executor-intro.gen.ts b/packages/app-templates/src/implementation/executor-intro.gen.ts index 4b2d7bb8ec..96a857cf9d 100644 --- a/packages/app-templates/src/implementation/executor-intro.gen.ts +++ b/packages/app-templates/src/implementation/executor-intro.gen.ts @@ -1,4 +1,4 @@ // Generated by packages/app-templates/scripts/executor-intro.ts. Do not edit. /** The body of skills/executor/SKILL.md, without its frontmatter. MCP servers send it as their instructions. */ -export const executorIntro = "# Executor\n\n## What Executor is\n\nExecutor is a host that runs apps: TypeScript packages that expose tools,\nstorage, UI, accounts and background work. Agents reach every app through the\nMCP tools `skills`, `search`, `describe`, `call` and `execute`, with each app's\nslug as its namespace. Credentials go only through Executor's secure connection\nlinks, never through chat or source code. Some calls pause for user approval.\n\nThis is the Executor app's `executor` skill. Read its other skills with\n`skills`, such as `{app: \"executor\", name: \"code-mode\"}`; `{}` lists them all.\n\n## Find and call tools\n\n`search({ query: \"...\" })` returns each tool's exact `path`, such as\n`tools..`, and input type; `describe({ paths })` returns full\nsignatures. Never guess argument names.\n\nFor a few independent calls, use `call({ calls: { path, input } })`; an array\nof calls runs concurrently. Each result has `ok` and its `value` or `error`.\n\nWrite a JavaScript program for `execute` when one call's result feeds another,\nor a large result should be filtered before it reaches you. Inside it, the same\npaths are async functions, and `tools.search` and `tools.search.describe` work\nas above. Return only the data you need. For depth, read the `code-mode` skill.\nWhen `call` or `execute` pauses for approval or input, show the request to the\nuser and resume it; never repeat the call or run the program again.\n\n## Build an app\n\nBuild an app when no existing app provides the tools, data, UI or automation\nthe user needs. For a known service, search the catalog first:\n`organization.catalog({ query: { search: \"gmail\", limit: \"20\" } })`, then\n`apps.install` a match or `apps.importCustom` a remote MCP URL. With a shell,\nuse the `executor apps` CLI, else `execute`; read the `app-authoring` skill first.\n\n## Send feedback\n\nSend feedback with the Executor app's `feedback.submit` tool when Executor gets\nin your way, can't do something you need, works especially well, or the user\nhas feedback. Before your final reply, check whether anything is worth\nreporting. Be specific; never include credentials or user data (`feedback.md`)."; +export const executorIntro = "# Executor\n\n## What Executor is\n\nExecutor is a host that runs apps: TypeScript packages that expose tools,\nstorage, UI, accounts and background work. Agents reach every app through the\nMCP tools `skills`, `search`, `describe`, `call` and `execute`, with each app's\nslug as its namespace. Credentials go only through Executor's secure connection\nlinks, never through chat or source code. Some calls pause for user approval.\n\nThis is the Executor app's `executor` skill. Read its other skills with\n`skills({app: \"executor\", name: \"code-mode\"})`; `skills({})` lists them all.\n\n## Find and call tools\n\n`search({ query: \"...\" })` returns each tool's exact `path`, such as\n`tools..`, and input type; `describe({ paths })` returns full\nsignatures. Never guess argument names.\n\nFor a few independent calls, use `call({ calls: { path, input } })`; an array\nof calls runs concurrently. Each result has `ok` and its `value` or `error`.\n\nWrite a JavaScript program for `execute` when one call's result feeds another,\nor a large result should be filtered before it reaches you. Inside it, the same\npaths are async functions, and `tools.search` and `tools.search.describe` work\nas above. Return only the data you need. For depth, read the `code-mode` skill.\nWhen `call` or `execute` pauses for approval or input, show the request to the\nuser and resume it; never repeat the call or run the program again.\n\n## Build an app\n\nBuild an app when no existing app provides the tools, data, UI or automation\nthe user needs. For a known service, search the catalog first:\n`organization.catalog({ query: { search: \"gmail\", limit: \"20\" } })`, then\n`apps.install` a match or `apps.importCustom` a remote MCP URL. With a shell,\nuse the `executor apps` CLI, else `execute`; read the `app-authoring` skill first.\n\n## Send feedback\n\nSend feedback with the Executor app's `feedback.submit` tool when Executor gets\nin your way, can't do something you need, works especially well, or the user\nhas feedback. Before your final reply, check whether anything is worth\nreporting. Be specific; never include credentials or user data (`feedback.md`)."; diff --git a/packages/app-templates/src/implementation/executor-skill-digests.gen.ts b/packages/app-templates/src/implementation/executor-skill-digests.gen.ts index 611f774c7e..7835202273 100644 --- a/packages/app-templates/src/implementation/executor-skill-digests.gen.ts +++ b/packages/app-templates/src/implementation/executor-skill-digests.gen.ts @@ -16,6 +16,6 @@ export const executorSkillDigests: Readonly> = { "app-authoring/webhooks.md": "e4ca224cd413b591c3692b389bd88d644b337b4697a067d83551e6bcab657cf3", "app-authoring/workflows.md": "243842172b2f3e12f3ba0b0bc28961c85a9e9e6f9a6c8e8191697c1d8dd479ae", "code-mode/SKILL.md": "40e2efdb28d0973c226126779aa313208db760b079e3ddf6c8cf3713e5d489dc", - "executor/SKILL.md": "f58448f79ea529ebadbba8f0e001f2c9fe46f73475b9f6e99423f6d81ff97709", + "executor/SKILL.md": "d98a4bfa40ee0afbf8c31fac67b27b69c400ee2c3b434479c08deb72f0e07e99", "executor/feedback.md": "11397bd260917e5a21e92897a19368d0426bf5944a7c081adc2c061d1ea797cd" }; diff --git a/packages/mcp/src/contracts/browser-tools.ts b/packages/mcp/src/contracts/browser-tools.ts index fb652cf4a2..983f114193 100644 --- a/packages/mcp/src/contracts/browser-tools.ts +++ b/packages/mcp/src/contracts/browser-tools.ts @@ -26,7 +26,7 @@ export const BrowserResumeInput = Schema.Struct({ requestId: InteractionId }); /** Execute with browser-based delivery of every pending interaction. */ export const BrowserExecuteTool = McpTool.make("execute", { description: - "Run a JavaScript program over Executor apps, for work where one result feeds another call or a large result should be filtered before it reaches you; for one or a few independent calls, use call instead. Find tools with the search tool or tools.search inside the program, and full signatures with describe or tools.search.describe({ paths }). First read the Executor app's executor skill with the skills tool. Never ask the user for secrets in chat; accounts connect through Executor's secure links. If approval-required or input-required is returned, show the user approvalUrl and call resume with requestId only; the user answers in their browser. Never submit the decision yourself or rerun the program to continue it. Earlier effects are not rolled back. Send feedback with the Executor app's feedback.submit tool when Executor gets in your way or something works especially well.", + "Run a JavaScript program over Executor apps, for work where one result feeds another call or a large result should be filtered before it reaches you; for one or a few independent calls, use call instead. Find tools with the search tool or tools.search inside the program, and full signatures with describe or tools.search.describe({ paths }). First read the Executor app's entry skill with skills({app:'executor',name:'executor'}). Never ask the user for secrets in chat; accounts connect through Executor's secure links. If approval-required or input-required is returned, show the user approvalUrl and call resume with requestId only; the user answers in their browser. Never submit the decision yourself or rerun the program to continue it. Earlier effects are not rolled back. Send feedback with the Executor app's feedback.submit tool when Executor gets in your way or something works especially well.", dependencies: [HttpServerRequest.HttpServerRequest], parameters: ExecuteInput, success: BrowserExecutionResult, diff --git a/packages/mcp/src/contracts/execute.ts b/packages/mcp/src/contracts/execute.ts index 6970b20bd1..d9f068f5c5 100644 --- a/packages/mcp/src/contracts/execute.ts +++ b/packages/mcp/src/contracts/execute.ts @@ -465,7 +465,7 @@ export const searchPageBytes = (limits: McpLimits) => Math.floor(limits.maxOutpu /** Execute programs over the host-provided app catalog. */ export const ExecuteTool = McpTool.make("execute", { description: - "Run a JavaScript program over Executor apps, for work where one result feeds another call or a large result should be filtered before it reaches you; for one or a few independent calls, use call instead. Find tools with the search tool or tools.search inside the program, and full signatures with describe or tools.search.describe({ paths }). First read the Executor app's executor skill with the skills tool. Never ask the user for secrets in chat; accounts connect through Executor's secure links. If approval-required or input-required is returned, show it to the user and call resume with their answer; never run the program's source again. External effects are not rolled back on error or cancellation. Send feedback with the Executor app's feedback.submit tool when Executor gets in your way or something works especially well.", + "Run a JavaScript program over Executor apps, for work where one result feeds another call or a large result should be filtered before it reaches you; for one or a few independent calls, use call instead. Find tools with the search tool or tools.search inside the program, and full signatures with describe or tools.search.describe({ paths }). First read the Executor app's entry skill with skills({app:'executor',name:'executor'}). Never ask the user for secrets in chat; accounts connect through Executor's secure links. If approval-required or input-required is returned, show it to the user and call resume with their answer; never run the program's source again. External effects are not rolled back on error or cancellation. Send feedback with the Executor app's feedback.submit tool when Executor gets in your way or something works especially well.", dependencies: [HttpServerRequest.HttpServerRequest], parameters: ExecuteInput, success: McpExecutionResult, @@ -475,7 +475,7 @@ export const ExecuteTool = McpTool.make("execute", { /** Native-mode execution obtains policy decisions through server-initiated MCP requests. */ export const NativeExecuteTool = McpTool.make("execute", { description: - "Run a JavaScript program over Executor apps, for work where one result feeds another call or a large result should be filtered before it reaches you; for one or a few independent calls, use call instead. Find tools with the search tool or tools.search inside the program, and full signatures with describe or tools.search.describe({ paths }). First read the Executor app's executor skill with the skills tool. Never ask the user for secrets in chat; accounts connect through Executor's secure links. Approvals and tool input open the MCP client's own prompt, and execute continues the same program. Earlier tool calls may already have completed and are not rolled back; never rerun the program automatically after an error. Send feedback with the Executor app's feedback.submit tool when Executor gets in your way or something works especially well.", + "Run a JavaScript program over Executor apps, for work where one result feeds another call or a large result should be filtered before it reaches you; for one or a few independent calls, use call instead. Find tools with the search tool or tools.search inside the program, and full signatures with describe or tools.search.describe({ paths }). First read the Executor app's entry skill with skills({app:'executor',name:'executor'}). Never ask the user for secrets in chat; accounts connect through Executor's secure links. Approvals and tool input open the MCP client's own prompt, and execute continues the same program. Earlier tool calls may already have completed and are not rolled back; never rerun the program automatically after an error. Send feedback with the Executor app's feedback.submit tool when Executor gets in your way or something works especially well.", dependencies: [HttpServerRequest.HttpServerRequest, McpSchema.McpRequestContext], parameters: ExecuteInput, success: McpExecutionResult, diff --git a/packages/mcp/src/contracts/skills.ts b/packages/mcp/src/contracts/skills.ts index 88749dcdcd..157bbc4d10 100644 --- a/packages/mcp/src/contracts/skills.ts +++ b/packages/mcp/src/contracts/skills.ts @@ -9,51 +9,93 @@ import { SourceFilePath, SkillRevision, } from "@executor-js/sdk/core"; -import { Schema } from "effect"; +import { type JsonSchema, Schema } from "effect"; import { Tool as McpTool } from "effect/ai"; import { RecordedMessage } from "@executor-js/utils/recorded-message"; import { UnavailableApp } from "./execute.ts"; -const absent = Schema.optionalKey(Schema.Never); -const version = Schema.optionalKey(VersionId); /** - * Reads no longer choose accounts. The key is still read so a caller that sets it gets - * `AccountChoiceUnsupported`, presented as a {@link SkillAccessFailed} with its code and status, - * rather than having it ignored. + * The keys each key needs: everything but a full listing names an app, and a reference file + * names its skill. `{}` lists every skill, `{app}` one app's and `{app, name}` reads one. + */ +const needs = { + name: ["app"], + file: ["app", "name"], + version: ["app"], + revision: ["app"], + accounts: ["app"], +} as const satisfies Record>; +const missing = (input: object) => + Object.entries(needs).flatMap(([key, required]) => + Object.hasOwn(input, key) + ? required.filter((need) => !Object.hasOwn(input, need)).map((need) => ({ key, need })) + : [], + ); +/** + * The read shapes as one struct whose key dependencies are a check, because MCP clients need an + * object-root input schema. The check emits them as `dependentRequired`, so the tool's advertised + * input schema ({@link SkillsInputJsonSchema}) and its decoding come from this one value. */ -const accounts = Schema.optionalKey( - Schema.Unknown.annotate({ - description: - "Not supported: skills are read with the accounts linked to the app. Setting it fails.", - }), -); -const selection = { revision: Schema.optionalKey(SkillRevision), accounts }; -const SkillSelection = Schema.Union([ - Schema.Struct({ - app: absent, - name: absent, - file: absent, - version: absent, - revision: absent, - accounts: absent, - }), - Schema.Struct({ app: AppSlug, name: absent, file: absent, version: version, ...selection }), - Schema.Struct({ - app: AppSlug, - ...selection, - name: AppSkillName, - file: Schema.optionalKey(SourceFilePath), - version: version, - }), -]); -/** MCP needs an object-root wire schema; decoding enforces list/document/reference combinations. */ export const SkillsInput = Schema.Struct({ app: Schema.optionalKey(AppSlug), name: Schema.optionalKey(AppSkillName), file: Schema.optionalKey(SourceFilePath), - version: version, - ...selection, -}).pipe(Schema.decodeTo(SkillSelection)); + version: Schema.optionalKey(VersionId), + revision: Schema.optionalKey(SkillRevision), + /** + * Reads no longer choose accounts. The key is still read so a caller that sets it gets + * `AccountChoiceUnsupported`, presented as a {@link SkillAccessFailed} with its code and status, + * rather than having it ignored. + */ + accounts: Schema.optionalKey( + Schema.Unknown.annotate({ + description: + "Not supported: skills are read with the accounts linked to the app. Setting it fails.", + }), + ), +}).check( + Schema.makeFilter( + (input) => + missing(input).map(({ key, need }) => ({ + path: [need], + issue: + need === "app" + ? `${key} needs app, the slug of the app that ships the skill. Read the Executor app's entry skill with {app: "executor", name: "executor"}; {} lists every visible skill with its app.` + : `${key} needs name, the skill whose file it reads, as in {app: "executor", name: "app-authoring", file: "deploy.md"}.`, + })), + { + expected: "{} to list skills, {app} to list one app's skills or {app, name} to read one", + toJsonSchema: () => ({ dependentRequired: needs }), + }, + ), +); +/** + * {@link SkillsInput} as JSON Schema. Effect puts the check's keywords in a top-level `allOf` + * member, and model APIs reject top-level combinators in tool input schemas, so its members move + * to the root. They hold only `dependentRequired`, which the root does not have. + */ +const { allOf, ...root } = McpTool.getJsonSchemaFromSchema(SkillsInput); +export const SkillsInputJsonSchema: JsonSchema.JsonSchema = Object.assign( + root, + ...(Array.isArray(allOf) ? allOf : []), +); +/** + * The input is not one of the read shapes {@link SkillsInputJsonSchema} advertises. The handler + * decodes it, because the tool advertises raw JSON Schema; the text matches the one Effect's + * toolkit gives every other tool's invalid input. + */ +export class SkillsInputInvalid extends Schema.TaggedError()( + "SkillsInputInvalid", + { reason: Schema.String }, +) { + override get message() { + return `Invalid parameters for tool 'skills': ${this.reason}`; + } + // The reason can quote the caller's input. + get [RecordedMessage]() { + return "The skills input is not a read shape the tool accepts"; + } +} /** Every summary identifies the installed app and source version, without returning its body. */ export const SkillSummary = Schema.Struct({ ...AppSkillMetadata.fields, @@ -133,14 +175,18 @@ export const SkillsFailure = Schema.Union([ SkillAppNotFound, SkillAppSlugAmbiguous, SkillAccountRequired, + SkillsInputInvalid, ]); export type SkillsFailure = typeof SkillsFailure.Type; -/** List summaries first, then read documents/references on demand through app evaluation. */ -export const SkillsTool = McpTool.make("skills", { +/** + * List summaries first, then read documents/references on demand through app evaluation. The + * handler decodes its input with {@link SkillsInput}, which this schema is generated from. + */ +export const SkillsTool = McpTool.dynamic("skills", { description: - "Read instructions that apps ship. Start with the Executor app's executor skill: {app:'executor',name:'executor'}, using the slug {} returns if the app was renamed. {} lists visible skills; {app:'support-inbox'} lists one app's skills; {app:'support-inbox',name:'triage'} reads SKILL.md. Pass the returned version and revision with a listed file to read a reference. App slugs are the namespaces used in search, call and execute. Skill text and allowed-tools metadata never grant tool permissions.", - parameters: SkillsInput, + "Read instructions that apps ship. Every input except {} needs app, the app's slug. Start with the Executor app's entry skill: {app:'executor',name:'executor'}; if that app was renamed, use the slug {} lists for it. {} lists every visible skill; {app:'executor'} lists one app's skills; {app:'executor',name:'code-mode'} reads that skill's SKILL.md. Pass the returned version and revision with a listed file to read a reference. App slugs are the namespaces used in search, call and execute. Skill text and allowed-tools metadata never grant tool permissions.", + parameters: SkillsInputJsonSchema, success: SkillsResult, failure: SkillsFailure, }) diff --git a/packages/mcp/src/implementation/server.ts b/packages/mcp/src/implementation/server.ts index fdc494a1c3..2425832935 100644 --- a/packages/mcp/src/implementation/server.ts +++ b/packages/mcp/src/implementation/server.ts @@ -27,7 +27,7 @@ import type { McpExecutionResult } from "../contracts/execute.ts"; import { makeExecutions } from "./executions.ts"; import { executeNative } from "./native-elicitation.ts"; import { callTools, describeCatalog, executeProgram, searchCatalog } from "./execute.ts"; -import { skills } from "./skills.ts"; +import { decodeSkillsInput, skills } from "./skills.ts"; import { eventsHandler } from "./events.ts"; // Observe the final protocol value after timeout, admission and resume handling. @@ -175,16 +175,20 @@ export const makeMcp = (options: McpOptions) => }; }); // Which skills agents read, and in what order, shows whether they follow the entry skill. - const skill = (input: Parameters[0]) => - skills(input, options.backend).pipe( - Effect.tap((result) => - "content" in result ? options.annotateSkillRead(result) : Effect.void, + const skill = (params: unknown) => + decodeSkillsInput(params).pipe( + Effect.flatMap((input) => + skills(input, options.backend).pipe( + Effect.tap((result) => + "content" in result ? options.annotateSkillRead(result) : Effect.void, + ), + Effect.withSpan("mcp.skills", { + attributes: { + "executor.skill.operation": input.name === undefined ? "list" : "read", + }, + }), + ), ), - Effect.withSpan("mcp.skills", { - attributes: { - "executor.skill.operation": input.name === undefined ? "list" : "read", - }, - }), ); // Search and describe read the catalog directly; call runs as an execution so it can pause. const search = (input: Parameters[2]) => diff --git a/packages/mcp/src/implementation/skills.ts b/packages/mcp/src/implementation/skills.ts index dfcc034b35..27c9bcef00 100644 --- a/packages/mcp/src/implementation/skills.ts +++ b/packages/mcp/src/implementation/skills.ts @@ -14,8 +14,9 @@ import { SkillAccountRequired, SkillAppNotFound, SkillAppSlugAmbiguous, + SkillsInput, + SkillsInputInvalid, type SkillsFailure, - type SkillsInput, type SkillsResult, type SkillSummary, } from "../contracts/skills.ts"; @@ -59,6 +60,12 @@ const skillAccounts = ( ), ); +/** The tool's input as one of the read shapes its advertised schema allows. */ +export const decodeSkillsInput = (input: unknown) => + Schema.decodeUnknownEffect(SkillsInput)(input, { errors: "all" }).pipe( + Effect.mapError((error) => new SkillsInputInvalid({ reason: error.message })), + ); + /** * Resolve a slug only within authorized apps. Reference reads can pin the document's version. * A read that names accounts fails with `AccountChoiceUnsupported`, presented with its code and From ad511b50da8c15f5ae25d7658db5cb64c4d3e4bb Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:58:23 +0000 Subject: [PATCH 023/127] Export b87df447 --- .changeset/background-service.md | 20 + README.md | 9 +- apps/docs/content/mcp-clients.mdx | 11 +- apps/docs/content/run/cli.mdx | 159 ++- apps/docs/content/run/tracing.md | 5 +- apps/docs/content/run/usage-analytics.md | 6 +- apps/docs/public/install.ps1 | 24 +- apps/docs/public/install.sh | 23 +- apps/local/desktop/README.md | 291 ++-- apps/local/desktop/scripts/build.mjs | 18 +- apps/local/desktop/scripts/launcher.mjs | 4 +- apps/local/desktop/scripts/start.mjs | 2 + apps/local/desktop/src/contracts/desktop.ts | 151 +- .../desktop/src/implementation/backend.ts | 243 ---- .../desktop/src/implementation/connection.ts | 353 +++++ .../desktop/src/implementation/diagnostics.ts | 6 +- .../desktop/src/implementation/migration.ts | 1005 +++++++++++++ .../desktop/src/implementation/recovery.ts | 59 +- .../local/desktop/src/implementation/reset.ts | 98 +- .../desktop/src/implementation/rotation.ts | 95 +- .../desktop/src/implementation/runtime.ts | 525 +++++++ .../desktop/src/implementation/settings.ts | 127 +- .../desktop/src/implementation/startup.ts | 15 +- .../desktop/src/implementation/supervisor.ts | 164 --- .../desktop/src/implementation/updates.ts | 3 +- apps/local/desktop/src/main.ts | 875 ++++++++---- apps/local/desktop/src/server.ts | 150 -- apps/local/desktop/src/starter.ts | 92 ++ apps/local/server/README.md | 237 +++- apps/local/server/src/bin.ts | 101 +- apps/local/server/src/contracts/auth.ts | 28 +- apps/local/server/src/contracts/config.ts | 29 +- .../server/src/contracts/desktop-return.ts | 47 + apps/local/server/src/contracts/server.ts | 24 +- apps/local/server/src/contracts/service.ts | 259 ++++ apps/local/server/src/contracts/startup.ts | 136 +- apps/local/server/src/dev.ts | 6 +- .../src/implementation/account-connections.ts | 4 +- .../src/implementation/app-management.ts | 15 +- .../server/src/implementation/atomic-file.ts | 40 + apps/local/server/src/implementation/auth.ts | 35 +- .../server/src/implementation/bootstrap.ts | 112 +- .../server/src/implementation/dashboard.ts | 8 +- .../src/implementation/desktop-return.ts | 135 ++ .../server/src/implementation/e2e-process.ts | 2 + .../server/src/implementation/launcher.ts | 132 +- apps/local/server/src/implementation/mcp.ts | 19 +- apps/local/server/src/implementation/pair.ts | 171 ++- .../src/implementation/process-identity.ts | 337 +++++ .../src/implementation/server-process.ts | 535 +++++++ .../src/implementation/server-record.ts | 53 - .../local/server/src/implementation/server.ts | 18 +- .../src/implementation/service-client.ts | 784 ++++++++++ .../src/implementation/service-files.ts | 483 +++++++ .../src/implementation/service-settings.ts | 174 +++ .../src/implementation/standalone-install.ts | 27 +- .../server/src/implementation/upgrade.ts | 307 +++- .../src/implementation/webhook-setup.ts | 6 +- apps/local/server/src/index.ts | 2 +- apps/local/server/src/main.ts | 8 +- apps/local/server/src/node.ts | 333 +++-- .../implementation/pages/connect-agent.tsx | 3 +- e2e/README.md | 6 +- e2e/support/background-service.ts | 534 +++++++ e2e/support/desktop.ts | 641 ++++++++- e2e/support/managed-server.ts | 2 + e2e/support/ports.ts | 13 + e2e/test-plan.ts | 315 +++- e2e/tests/desktop-recovery.spec.ts | 1262 +++++++++++++++-- e2e/tests/desktop-release.spec.ts | 422 +++--- e2e/tests/local-apps-cli.spec.ts | 2 +- e2e/tests/local-bootstrap.spec.ts | 77 +- e2e/tests/local-cli-legacy-commands.spec.ts | 18 +- e2e/tests/local-pair.spec.ts | 379 +++-- e2e/tests/local-service.spec.ts | 1209 ++++++++++++++++ e2e/tests/local-upgrade.spec.ts | 307 ++-- e2e/tests/npm-launcher.spec.ts | 13 +- packages/app-management/src/cli.ts | 509 ++++--- .../executor/skills/app-authoring/deploy.md | 10 +- .../executor-skill-digests.gen.ts | 2 +- packages/telemetry/src/local.ts | 46 +- ...n%2Fcloudflare-runtime@2.0.0-beta.80.patch | 47 + scripts/local-dev.ts | 4 +- scripts/releases/README.md | 27 +- scripts/releases/desktop.ts | 39 +- scripts/releases/runtime-bundle.ts | 7 +- 86 files changed, 12402 insertions(+), 2632 deletions(-) create mode 100644 .changeset/background-service.md delete mode 100644 apps/local/desktop/src/implementation/backend.ts create mode 100644 apps/local/desktop/src/implementation/connection.ts create mode 100644 apps/local/desktop/src/implementation/migration.ts create mode 100644 apps/local/desktop/src/implementation/runtime.ts delete mode 100644 apps/local/desktop/src/implementation/supervisor.ts delete mode 100644 apps/local/desktop/src/server.ts create mode 100644 apps/local/desktop/src/starter.ts create mode 100644 apps/local/server/src/contracts/desktop-return.ts create mode 100644 apps/local/server/src/contracts/service.ts create mode 100644 apps/local/server/src/implementation/atomic-file.ts create mode 100644 apps/local/server/src/implementation/desktop-return.ts create mode 100644 apps/local/server/src/implementation/process-identity.ts create mode 100644 apps/local/server/src/implementation/server-process.ts delete mode 100644 apps/local/server/src/implementation/server-record.ts create mode 100644 apps/local/server/src/implementation/service-client.ts create mode 100644 apps/local/server/src/implementation/service-files.ts create mode 100644 apps/local/server/src/implementation/service-settings.ts create mode 100644 e2e/support/background-service.ts create mode 100644 e2e/tests/local-service.spec.ts diff --git a/.changeset/background-service.md b/.changeset/background-service.md new file mode 100644 index 0000000000..889831b8b0 --- /dev/null +++ b/.changeset/background-service.md @@ -0,0 +1,20 @@ +--- +"@executor-js/local-server": minor +"@executor-js/app-management": patch +"@executor-js/desktop": minor +"@executor-js/app-templates": patch +--- + +`executor` now starts one background Executor per data directory, shared by the command line, +`executor apps` and the desktop app, and exits once the dashboard opens; the background Executor +runs until `executor service stop`. `executor service` starts, stops and configures it, and +`executor upgrade` stops it and starts it again on the new version. `executor apps` commands +without `--host` or `EXECUTOR_HOST` use it and need no API key, as the app-authoring skill now says. +Executor 1's `executor service install` and `executor service uninstall` name what to run instead. + +Executor desktop no longer runs its own server: it starts or joins the same background Executor and +uses the command line's data directory, `~/.executor/v2/cli`. Its first launch moves the data of an +earlier desktop release there, once. Command-line data already in that directory, with its apps and +accounts, moves to `~/.executor/v2/backups/cli-data-