Merge upstream v2 (54bfc1a59) into the fork

.changeset/local-tool-schemas-hydrate.md

@@ -0,0 +1,5 @@
+---
+"@executor-js/local-web": patch
+---
+
+The local Tools tab sends the selected tool's schemas with the server-rendered page, so the page hydrates without a markup mismatch.

apps/hosted/cloud/alchemy.github.ts

@@ -19,7 +19,25 @@ import {
   missingV1MembershipCheckSince,
   missingV1WorkosKey,
 } from "./src/infrastructure/v1-membership-settings.ts";
-import { type V1ImportSwitch, v1ImportSetting } from "./src/infrastructure/v1-import-settings.ts";
+import {
+  type V1ImportSwitch,
+  V1FirstPartyClients,
+  invalidV1FirstPartyClients,
+  missingV1FirstPartyClient,
+  invalidV1ImportAllowedOrgs,
+  parseV1ImportAllowedOrgs,
+  v1ImportAllowedOrgsName,
+  missingV1ImportWriter,
+  missingV1McpForwardSecret,
+  v1ImportWriterName,
+  v1McpForwardSecretName,
+  v1ImportOperatorTokenMinLength,
+  missingV1ImportOperatorToken,
+  v1FirstPartyClientsName,
+  v1ImportSetting,
+  v1ImportWritesNeedImport,
+  v1ImportWritesSetting,
+} from "./src/infrastructure/v1-import-settings.ts";
 import {
   firstPartyGitHubSettings,
   firstPartyGoogleSettings,
@@ -294,6 +312,68 @@ export default Alchemy.Stack(
       value: v1Import,
       environment: production,
     }).pipe(retain());
+    /**
+     * Imports that write v2 need their own switch, off unless `.env.ci.op` sets it. While the
+     * import is on, operators start runs with `V1_IMPORT_OPERATOR_TOKEN`, so this stack refuses
+     * to apply without it.
+     */
+    const v1ImportWrites: V1ImportSwitch = yield* v1ImportWritesSetting;
+    // The Worker refuses this pair when it starts; refuse it here, before it is stored.
+    if (v1ImportWrites === "on" && v1Import !== "on")
+      return yield* Effect.die(new Error(v1ImportWritesNeedImport));
+    yield* GitHub.Variable("production-CLOUD_V1_IMPORT_WRITES", {
+      ...target,
+      name: "CLOUD_V1_IMPORT_WRITES",
+      value: v1ImportWrites,
+      environment: production,
+    }).pipe(retain());
+    /**
+     * The v1 organizations production may import (`v1-import-settings.ts`): with writes on, an
+     * import names its organizations and each must be on this list. `.env.ci.op` sets it; unset
+     * or empty imports none, and stores nothing, as GitHub refuses an empty variable. It is not
+     * retained: emptying the list removes the variable, so production imports none again.
+     */
+    const allowedOrgs = yield* Config.String(v1ImportAllowedOrgsName).pipe(Config.withDefault(""));
+    if (parseV1ImportAllowedOrgs(allowedOrgs) === undefined)
+      return yield* Effect.die(new Error(invalidV1ImportAllowedOrgs));
+    if (allowedOrgs.trim() !== "")
+      yield* GitHub.Variable(`production-${v1ImportAllowedOrgsName}`, {
+        ...target,
+        name: v1ImportAllowedOrgsName,
+        value: allowedOrgs,
+        environment: production,
+      });
+    /**
+     * The URL of v1's `v1_import_writer` role (`moved-flag.ts`), which freezes and cuts over each
+     * imported organization. Imports that write need it, so this stack refuses writes without it.
+     */
+    const writer = yield* nonEmptySecret(v1ImportWriterName);
+    if (v1ImportWrites === "on" && Option.isNone(writer))
+      return yield* Effect.die(new Error(missingV1ImportWriter));
+    if (Option.isSome(writer))
+      yield* GitHub.Secret(`production-${v1ImportWriterName}`, {
+        ...target,
+        name: v1ImportWriterName,
+        value: writer.value,
+        environment: production,
+      }).pipe(retain());
+    if (v1Import === "on") {
+      const operatorToken = yield* Config.Redacted("V1_IMPORT_OPERATOR_TOKEN").pipe(
+        Config.option,
+        Effect.map(Option.filter((value) => Redacted.value(value) !== "")),
+      );
+      if (
+        Option.isNone(operatorToken) ||
+        Redacted.value(operatorToken.value).length < v1ImportOperatorTokenMinLength
+      )
+        return yield* Effect.die(new Error(missingV1ImportOperatorToken));
+      yield* GitHub.Secret("production-V1_IMPORT_OPERATOR_TOKEN", {
+        ...target,
+        name: "V1_IMPORT_OPERATOR_TOKEN",
+        value: operatorToken.value,
+        environment: production,
+      }).pipe(retain());
+    }
 
     /**
      * The v1 sign-in check: v1's WorkOS API key, the name of v1's PlanetScale database (the
@@ -381,6 +461,58 @@ export default Alchemy.Stack(
       }).pipe(retain()),
     );
 
+    /**
+     * v1's first-party client IDs (`v1-import-settings.ts`), public values the import compares with
+     * each v2 client's. While the import is on, every first-party client production offers needs
+     * v1's ID, or its connections are all reported as mismatches.
+     */
+    const v1Clients = yield* Config.String(v1FirstPartyClientsName).pipe(
+      Config.option,
+      Effect.map(Option.filter((value) => value !== "")),
+    );
+    const v1ClientIds = Option.isNone(v1Clients)
+      ? {}
+      : yield* Schema.decodeUnknownEffect(Schema.fromJsonString(V1FirstPartyClients))(
+          v1Clients.value,
+        ).pipe(
+          Effect.mapError(() => new Error(invalidV1FirstPartyClients)),
+          Effect.orDie,
+        );
+    if (v1Import === "on") {
+      const offered = [
+        "github",
+        ...(Option.isSome(googleId) ? ["google"] : []),
+        ...(Option.isSome(slackId) ? ["slack"] : []),
+      ];
+      const missing = offered.find((client) => !Object.hasOwn(v1ClientIds, client));
+      if (missing !== undefined)
+        return yield* Effect.die(new Error(missingV1FirstPartyClient(missing)));
+    }
+    if (Option.isSome(v1Clients))
+      yield* GitHub.Variable(`production-${v1FirstPartyClientsName}`, {
+        ...target,
+        name: v1FirstPartyClientsName,
+        value: v1Clients.value,
+        environment: production,
+      }).pipe(retain());
+
+    /**
+     * The secret v1 signs the MCP requests it forwards with (`v1-mcp-forward.ts`), the same value
+     * as v1's `V2_MCP_FORWARD_SECRET`. Without it production accepts no forwarded request, which v1
+     * answers as unavailable for a moved workspace, so imports that write, which cut workspaces
+     * over, need it.
+     */
+    const v1Forward = yield* nonEmptySecret(v1McpForwardSecretName);
+    if (v1ImportWrites === "on" && Option.isNone(v1Forward))
+      return yield* Effect.die(new Error(missingV1McpForwardSecret));
+    if (Option.isSome(v1Forward))
+      yield* GitHub.Secret(`production-${v1McpForwardSecretName}`, {
+        ...target,
+        name: v1McpForwardSecretName,
+        value: v1Forward.value,
+        environment: production,
+      }).pipe(retain());
+
     // Rulesets are unavailable on private repositories under the GitHub Free plan, so
     // `CI_RULESET_ENFORCEMENT=disabled` skips the resource instead of asking GitHub for it.
     const ruleset =

apps/hosted/cloud/package.json

@@ -49,8 +49,7 @@
     "js-yaml": "5.4.3",
     "kysely": "0.29.6",
     "pg": "8.16.3",
-    "postal-mime": "4.0.2",
-    "tldts": "7.4.13"
+    "postal-mime": "4.0.2"
   },
   "devDependencies": {
     "@cloudflare/workers-types": "5.20260729.1",

apps/hosted/cloud/scripts/check-v1-import-access.ts

@@ -0,0 +1,172 @@
+/**
+ * The v1 import reads v1's real data on production (`CLOUD_V1_IMPORT=on`) and, for an operator's
+ * dry run, from local Cloud dev (`CLOUD_V1_IMPORT_SOURCE=live`). Neither may write v2 until v1's
+ * freeze exists. This check evaluates `v1ImportAccess` for every combination of stage, origin,
+ * emulators and the three switches, with the stage given as Alchemy's `Stage` service and as
+ * `ALCHEMY_STAGE`. It fails when:
+ *
+ * - a real v1 source (production or live) is not marked `realV1`, which refuses imports that write;
+ * - the live source runs anywhere but a loopback origin outside production and test stages, or
+ *   with writes allowed;
+ * - production's source runs on any other stage;
+ * - the emulated Cloud the E2E runner uses stops allowing imports that write;
+ * - an import over v1's real data runs without naming its organizations, or for one the allow-list
+ *   (`CLOUD_V1_IMPORT_ALLOWED_ORGS`) does not list, or the allow-list reads wrongly.
+ */
+import { Stage } from "alchemy/Stage";
+import { ConfigProvider, Effect, Exit, Schema } from "effect";
+import { V1OrganizationId } from "../src/contracts/v1-import.ts";
+import type { V1MovedFlag } from "../src/implementation/v1-import/moved-flag.ts";
+import { V1ImportWrites, importRefusal } from "../src/implementation/v1-import/writes.ts";
+import { productionStage } from "../src/infrastructure/stage.ts";
+import {
+  parseV1ImportAllowedOrgs,
+  v1ImportAccess,
+} from "../src/infrastructure/v1-import-settings.ts";
+
+const stages = [productionStage, "dev", "dev-checkout", "staging", "test-e2e-check", "test-pr-1"];
+const loopbackOrigins = ["http://localhost:4411", "http://127.0.0.1:4411", "http://[::1]:4411"];
+const origins = [...loopbackOrigins, "https://executor.sh", "https://cloud.example.com"];
+const switches = [undefined, "off", "on"] as const;
+const sources = [undefined, "emulated", "live"] as const;
+
+interface Case {
+  readonly stage: string;
+  readonly stageSource: "Stage service" | "ALCHEMY_STAGE";
+  readonly origin: string;
+  readonly emulated: boolean;
+  readonly enabled: (typeof switches)[number];
+  readonly writes: (typeof switches)[number];
+  readonly source: (typeof sources)[number];
+}
+
+const describe = (c: Case) =>
+  `stage ${c.stage} (${c.stageSource}), BETTER_AUTH_URL=${c.origin}, ` +
+  `emulators ${c.emulated ? "on" : "off"}, CLOUD_V1_IMPORT=${c.enabled ?? "<unset>"}, ` +
+  `CLOUD_V1_IMPORT_WRITES=${c.writes ?? "<unset>"}, CLOUD_V1_IMPORT_SOURCE=${c.source ?? "<unset>"}`;
+
+const evaluate = (c: Case) => {
+  const env: Record<string, string> = {
+    BETTER_AUTH_URL: c.origin,
+    ...(c.stageSource === "ALCHEMY_STAGE" ? { ALCHEMY_STAGE: c.stage } : {}),
+    ...(c.enabled === undefined ? {} : { CLOUD_V1_IMPORT: c.enabled }),
+    ...(c.writes === undefined ? {} : { CLOUD_V1_IMPORT_WRITES: c.writes }),
+    ...(c.source === undefined ? {} : { CLOUD_V1_IMPORT_SOURCE: c.source }),
+  };
+  const access = v1ImportAccess(c.emulated).pipe(
+    Effect.provideService(ConfigProvider.ConfigProvider, ConfigProvider.fromEnv({ env })),
+  );
+  return Effect.runSyncExit(
+    c.stageSource === "Stage service" ? access.pipe(Effect.provideService(Stage, c.stage)) : access,
+  );
+};
+
+const cases = stages.flatMap((stage) =>
+  (["Stage service", "ALCHEMY_STAGE"] as const).flatMap((stageSource) =>
+    origins.flatMap((origin) =>
+      [false, true].flatMap((emulated) =>
+        switches.flatMap((enabled) =>
+          switches.flatMap((writes) =>
+            sources.map((source): Case => ({
+              stage,
+              stageSource,
+              origin,
+              emulated,
+              enabled,
+              writes,
+              source,
+            })),
+          ),
+        ),
+      ),
+    ),
+  ),
+);
+
+const failures: string[] = [];
+let live = 0;
+for (const c of cases) {
+  const exit = evaluate(c);
+  if (Exit.isFailure(exit)) continue;
+  const access = exit.value;
+  const fail = (rule: string) => failures.push(`${describe(c)}: ${rule}`);
+  if ((access.source === "production" || access.source === "live") && !access.realV1)
+    fail("a real v1 source does not refuse imports that write");
+  if (access.source === "production" && c.stage !== productionStage)
+    fail("production's v1 source runs on another stage");
+  if (access.source === "live") {
+    live++;
+    if (c.stage === productionStage || c.stage.startsWith("test-"))
+      fail("the live source runs on a deployed stage");
+    if (!loopbackOrigins.includes(c.origin)) fail("the live source runs on a public origin");
+    if (access.writes) fail("the live source allows imports that write");
+  }
+  if (c.source === "live" && access.source !== "live") fail("the live source was ignored");
+}
+
+// The live source must still be reachable where it is meant to run.
+if (live === 0) failures.push("No combination allows the live source on local Cloud dev.");
+// The E2E runner's managed Cloud: emulated, loopback, no switches, imports that write allowed.
+const e2e = evaluate({
+  stage: "dev",
+  stageSource: "ALCHEMY_STAGE",
+  origin: "http://127.0.0.1:4411",
+  emulated: true,
+  enabled: undefined,
+  writes: undefined,
+  source: undefined,
+});
+if (!Exit.isSuccess(e2e) || e2e.value.source !== "emulated" || !e2e.value.writes)
+  failures.push("Emulated Cloud no longer allows imports that write.");
+
+// Which organizations an import may write. The flag is never written here.
+const flag: V1MovedFlag = { set: () => Effect.void };
+const id = Schema.decodeUnknownSync(V1OrganizationId);
+const [listedOrg, otherOrg] = [id("org_listed1"), id("org_other1")];
+const listed = V1ImportWrites.Listed({ organizations: new Set([listedOrg]), flag });
+const writeRules: ReadonlyArray<readonly [string, string | undefined, boolean]> = [
+  ["a real import that pages through v1", importRefusal(listed, undefined), false],
+  ["a real import that names no organization", importRefusal(listed, []), false],
+  ["a real import of a listed organization", importRefusal(listed, [listedOrg]), true],
+  ["a real import of an unlisted organization", importRefusal(listed, [otherOrg]), false],
+  ["a real import that adds an unlisted one", importRefusal(listed, [listedOrg, otherOrg]), false],
+  [
+    "a real import with an empty allow-list",
+    importRefusal(V1ImportWrites.Listed({ organizations: new Set(), flag }), [listedOrg]),
+    false,
+  ],
+  ["an emulated import that pages", importRefusal(V1ImportWrites.Any({ flag }), undefined), true],
+  [
+    "a deployment that refuses imports",
+    importRefusal(V1ImportWrites.Refused({ reason: "off" }), [listedOrg]),
+    false,
+  ],
+];
+for (const [rule, refusal, allowed] of writeRules)
+  if ((refusal === undefined) !== allowed)
+    failures.push(`${rule} is ${refusal === undefined ? "allowed" : "refused"}`);
+const lists: ReadonlyArray<readonly [string, readonly string[] | undefined]> = [
+  ["", []],
+  [" ", []],
+  ["org_a1", ["org_a1"]],
+  [" org_a1 , org_b2,", ["org_a1", "org_b2"]],
+  ["org_a1,not-an-org", undefined],
+  ["org_a1;org_b2", undefined],
+];
+for (const [value, expected] of lists) {
+  const parsed = parseV1ImportAllowedOrgs(value);
+  const got = parsed === undefined ? undefined : [...parsed].toSorted();
+  if (JSON.stringify(got) !== JSON.stringify(expected))
+    failures.push(
+      `CLOUD_V1_IMPORT_ALLOWED_ORGS=${JSON.stringify(value)} reads as ${JSON.stringify(got)}`,
+    );
+}
+
+if (failures.length > 0) {
+  console.error(`v1 import access rules failed in ${failures.length} combinations:`);
+  for (const failure of failures.slice(0, 20)) console.error(`- ${failure}`);
+  process.exit(1);
+}
+console.log(
+  `v1 import access: ${cases.length} combinations checked, ${live} run the live source; ${writeRules.length} write rules and ${lists.length} allow-lists checked.`,
+);

apps/hosted/cloud/scripts/local-dev.ts

@@ -20,6 +20,7 @@ import {
   localDevelopmentFiles,
   localDevelopmentStage,
 } from "./local-development.ts";
+import { v1LiveSettings } from "../src/infrastructure/v1-import-settings.ts";
 
 class LocalDevelopmentFailed extends Schema.TaggedError<LocalDevelopmentFailed>()(
   "LocalDevelopmentFailed",
@@ -60,6 +61,18 @@ const inherited = [
   "NODE_EXTRA_CA_CERTS",
 ] as const;
 
+/** Passed through only for a live v1 dry run; see `cloudV1LiveSource`. */
+const liveV1DryRun = [
+  "CLOUD_V1_IMPORT_SOURCE",
+  ...Object.values(v1LiveSettings),
+  "V1_IMPORT_OPERATOR_TOKEN",
+  "V1_FIRST_PARTY_CLIENT_IDS",
+  ...["GITHUB", "GOOGLE", "SLACK"].flatMap((client) => [
+    `EXECUTOR_OAUTH_${client}_CLIENT_ID`,
+    `EXECUTOR_OAUTH_${client}_CLIENT_SECRET`,
+  ]),
+] as const;
+
 const main = Effect.scoped(
   Effect.gen(function* () {
     const fs = yield* FileSystem.FileSystem;
@@ -179,6 +192,14 @@ const main = Effect.scoped(
       V1_MEMBERSHIP_CHECK_SINCE: new Date(yield* Clock.currentTimeMillis).toISOString(),
     });
 
+    // A live v1 dry run (`CLOUD_V1_IMPORT_SOURCE=live`) reads v1's production data with the
+    // operator's credentials and production's first-party client IDs, bound for this process only.
+    if (process.env.CLOUD_V1_IMPORT_SOURCE === "live")
+      for (const key of liveV1DryRun) {
+        const value = process.env[key];
+        if (value !== undefined) env[key] = value;
+      }
+
     yield* fs.writeFileString(
       localDevelopmentFiles.session,
       JSON.stringify(LocalSession.make({ pid: process.pid, origin, apiPort, databasePort })),