From 9a9e2232a72886a5ccc1ab85524daac688cc7f24 Mon Sep 17 00:00:00 2001 From: Maria Melo Date: Thu, 24 Sep 2026 17:55:41 +0100 Subject: [PATCH 1/2] Add campaign experiment tools for the upcoming API client. Agents need create, run, and reporting actions once @iterable/api 0.13.0 is published. --- TOOLS.md | 13 +- package.json | 2 +- src/tool-filter.ts | 13 ++ src/tools/experiments.ts | 68 ++++++ tests/unit/experiments-tools.test.ts | 296 +++++++++++++++++++++++++++ tests/unit/tools.test.ts | 11 +- 6 files changed, 399 insertions(+), 4 deletions(-) create mode 100644 tests/unit/experiments-tools.test.ts diff --git a/TOOLS.md b/TOOLS.md index 14645b1..2d5c530 100644 --- a/TOOLS.md +++ b/TOOLS.md @@ -1,4 +1,4 @@ -# Available Iterable MCP Tools (109 tools) +# Available Iterable MCP Tools (118 tools) **Legend:** - 🔒 = Requires enabling user PII access @@ -50,11 +50,20 @@ - **track_bulk_events** 🔒✏️✉️: Track multiple events in a single request for better performance - **track_event** 🔒✏️✉️: Track a custom event for a user -## Experiments (4 tools) +## Experiments (13 tools) +- **cancel_experiment** ✏️: Cancel a running or winner_found campaign experiment without declaring a winner. +- **copy_experiment_variant** ✏️: Copy a template into a new variant on a draft or ready campaign experiment. +- **create_experiment** ✏️: Create a draft campaign experiment. The campaign template becomes the control. +- **declare_experiment_winner** ✏️✉️: Declare a winning variant and end a running or winner_found campaign experiment. +- **delete_experiment** ✏️: Delete a draft, ready, or errored campaign experiment. - **get_experiment**: Get detailed information about a specific experiment by ID, including variants summary and constraints - **get_experiment_metrics**: Get experiment metrics for A/B testing analysis (currently supports email experiments only) +- **get_experiment_totals**: Get lifetime send and conversion totals for a campaign experiment. Holdout is omitted, missing counts are 0, and lift and confidence are not included. +- **get_experiment_trends**: Get the performance time series for a campaign experiment. Provide both startDateTime and endDateTime, or omit both to use the run window. The span is at most 31 days; a longer default window is clipped to the last 31 days. - **get_experiment_variants**: Get variant content for an experiment, including subject lines, preheaders, HTML source, and plain text - **list_experiments**: List experiments with optional filtering by campaign, status, and date range. Supports pagination. +- **start_experiment** ✏️✉️: Start a draft or ready campaign experiment. +- **update_experiment_settings** ✏️: Update settings on a draft or ready campaign experiment. Omit a field to leave it unchanged. ## Journeys (2 tools) - **get_journeys**: Get journeys (workflows) with optional pagination and state filtering diff --git a/package.json b/package.json index a4b42aa..99bb0e9 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,7 @@ }, "dependencies": { "@alcyone-labs/zod-to-json-schema": "4.0.10", - "@iterable/api": "0.12.0", + "@iterable/api": "0.13.0", "@modelcontextprotocol/sdk": "1.18.1", "@primno/dpapi": "2.0.1", "@types/json-schema": "7.0.15", diff --git a/src/tool-filter.ts b/src/tool-filter.ts index 8376dc3..b706853 100644 --- a/src/tool-filter.ts +++ b/src/tool-filter.ts @@ -13,13 +13,18 @@ export const NON_PII_TOOLS: Set = new Set([ "archive_campaigns", "bulk_delete_catalog_items", "cancel_campaign", + "cancel_experiment", + "copy_experiment_variant", "create_blast_campaign", + "create_experiment", "create_triggered_campaign", "create_catalog", "create_list", "create_snippet", "deactivate_triggered_campaign", + "declare_experiment_winner", "delete_catalog", + "delete_experiment", "delete_catalog_item", "delete_list", "delete_snippet", @@ -36,6 +41,8 @@ export const NON_PII_TOOLS: Set = new Set([ "get_email_template", "get_experiment", "get_experiment_metrics", + "get_experiment_totals", + "get_experiment_trends", "get_experiment_variants", "get_inapp_template", "get_journeys", @@ -57,11 +64,13 @@ export const NON_PII_TOOLS: Set = new Set([ "replace_catalog_item", "schedule_campaign", "send_campaign", + "start_experiment", "trigger_campaign", "update_catalog_field_mappings", "partial_update_catalog_items", "replace_catalog_items", "update_email_template", + "update_experiment_settings", "update_inapp_template", "update_push_template", "update_sms_template", @@ -87,6 +96,8 @@ export const READ_ONLY_TOOLS: Set = new Set([ "get_embedded_messages", "get_experiment", "get_experiment_metrics", + "get_experiment_totals", + "get_experiment_trends", "get_experiment_variants", "get_export_files", "get_export_jobs", @@ -125,6 +136,8 @@ export const SEND_TOOLS: Set = new Set([ "send_campaign", "trigger_campaign", "schedule_campaign", + "start_experiment", + "declare_experiment_winner", // Triggered campaigns can cause sends upon activation; block unless explicitly allowed "activate_triggered_campaign", // Journey triggers enqueue users which may send diff --git a/src/tools/experiments.ts b/src/tools/experiments.ts index b2d1271..d82abc7 100644 --- a/src/tools/experiments.ts +++ b/src/tools/experiments.ts @@ -4,10 +4,17 @@ import type { IterableClient } from "@iterable/api"; import { + CopyExperimentVariantParamsSchema, + CreateExperimentParamsSchema, + DeclareExperimentWinnerParamsSchema, + ExperimentIdParamsSchema, GetExperimentMetricsParamsSchema, GetExperimentParamsSchema, + GetExperimentTotalsParamsSchema, + GetExperimentTrendsParamsSchema, GetExperimentVariantsParamsSchema, ListExperimentsParamsSchema, + UpdateExperimentSettingsParamsSchema, } from "@iterable/api"; import type { Tool } from "@modelcontextprotocol/sdk/types.js"; @@ -43,5 +50,66 @@ export function createExperimentTools(client: IterableClient): Tool[] { schema: GetExperimentMetricsParamsSchema, execute: (params) => client.getExperimentMetrics(params), }), + createTool({ + name: "get_experiment_totals", + description: + "Get lifetime send and conversion totals for a campaign experiment. Holdout is omitted, missing counts are 0, and lift and confidence are not included.", + schema: GetExperimentTotalsParamsSchema, + execute: (params) => client.getExperimentTotals(params), + }), + createTool({ + name: "get_experiment_trends", + description: + "Get the performance time series for a campaign experiment. Provide both startDateTime and endDateTime, or omit both to use the run window. The span is at most 31 days; a longer default window is clipped to the last 31 days.", + schema: GetExperimentTrendsParamsSchema, + execute: (params) => client.getExperimentTrends(params), + }), + createTool({ + name: "create_experiment", + description: + "Create a draft campaign experiment. The campaign template becomes the control.", + schema: CreateExperimentParamsSchema, + execute: (params) => client.createExperiment(params), + }), + createTool({ + name: "copy_experiment_variant", + description: + "Copy a template into a new variant on a draft or ready campaign experiment.", + schema: CopyExperimentVariantParamsSchema, + execute: (params) => client.copyExperimentVariant(params), + }), + createTool({ + name: "update_experiment_settings", + description: + "Update settings on a draft or ready campaign experiment. Omit a field to leave it unchanged.", + schema: UpdateExperimentSettingsParamsSchema, + execute: (params) => client.updateExperimentSettings(params), + }), + createTool({ + name: "start_experiment", + description: "Start a draft or ready campaign experiment.", + schema: ExperimentIdParamsSchema, + execute: (params) => client.startExperiment(params), + }), + createTool({ + name: "cancel_experiment", + description: + "Cancel a running or winner_found campaign experiment without declaring a winner.", + schema: ExperimentIdParamsSchema, + execute: (params) => client.cancelExperiment(params), + }), + createTool({ + name: "declare_experiment_winner", + description: + "Declare a winning variant and end a running or winner_found campaign experiment.", + schema: DeclareExperimentWinnerParamsSchema, + execute: (params) => client.declareExperimentWinner(params), + }), + createTool({ + name: "delete_experiment", + description: "Delete a draft, ready, or errored campaign experiment.", + schema: ExperimentIdParamsSchema, + execute: (params) => client.deleteExperiment(params), + }), ]; } diff --git a/tests/unit/experiments-tools.test.ts b/tests/unit/experiments-tools.test.ts new file mode 100644 index 0000000..cc63972 --- /dev/null +++ b/tests/unit/experiments-tools.test.ts @@ -0,0 +1,296 @@ +import { createIterableError, IterableClient } from "@iterable/api"; +import { beforeEach, describe, expect, it, jest } from "@jest/globals"; +import { McpError } from "@modelcontextprotocol/sdk/types.js"; + +import { createExperimentTools } from "../../src/tools/experiments.js"; + +const asyncMock = () => jest.fn<(...args: unknown[]) => Promise>(); + +const mockAxiosInstance = { + get: asyncMock(), + post: asyncMock(), + put: asyncMock(), + delete: asyncMock(), + patch: asyncMock(), +}; + +type ToolResult = { + content: Array<{ type: string; text: string }>; +}; + +const experiment = { + id: 884102, + status: "draft", + channelType: "email", + experimentType: "SubjectLine", + allocationMode: "even_split", + sizing: { + holdoutPercentage: null, + perVariantPercentage: null, + }, + constraints: null, + meta: { + name: "Welcome Experiment", + conversionMetrics: ["opens"], + campaignId: 129500, + projectId: 1, + orgId: 2, + }, + variants: [ + { + id: 0, + name: "Control", + value: { templateId: 55 }, + currentPercentage: 50, + isWinner: false, + isControl: true, + }, + ], +}; + +const totals = { + id: 884102, + status: "running", + variants: [ + { + id: 0, + name: "Control", + isControl: true, + isWinner: false, + metrics: { sends: 5000, emailOpen: 1250, purchase: 0 }, + }, + ], +}; + +function toolHandler(client: IterableClient, name: string) { + const tool = createExperimentTools(client).find((item) => item.name === name); + if (!tool || !("handler" in tool) || typeof tool.handler !== "function") { + throw new Error(`missing tool ${name}`); + } + return tool.handler as (args: unknown) => Promise; +} + +function restError(status: number, error: string) { + return createIterableError({ + response: { status, data: { error } }, + config: { url: "/api/experiments/884102" }, + }); +} + +async function errorBody(client: IterableClient, name: string, args: unknown) { + const result = await toolHandler(client, name)(args); + return JSON.parse(result.content[0]?.text ?? "{}") as { + statusCode: number; + rawResponse: { error: string }; + }; +} + +describe("experiment tools", () => { + let client: IterableClient; + + beforeEach(() => { + jest.clearAllMocks(); + client = new IterableClient( + { + apiKey: "test-key", + baseUrl: "https://api.iterable.com", + timeout: 30000, + }, + mockAxiosInstance as never + ); + }); + + it("returns lifetime totals", async () => { + mockAxiosInstance.get.mockResolvedValue({ data: totals }); + + const result = await toolHandler( + client, + "get_experiment_totals" + )({ experimentId: 884102 }); + + expect(mockAxiosInstance.get).toHaveBeenCalledWith( + "/api/experiments/884102/totals" + ); + expect(JSON.parse(result.content[0]?.text ?? "{}")).toEqual(totals); + }); + + it.each([ + { status: 400, error: "Journey experiments are not supported" }, + { status: 404, error: "Experiment 884102 not found" }, + ])("returns totals status $status for $error", async ({ status, error }) => { + mockAxiosInstance.get.mockRejectedValue(restError(status, error)); + + const body = await errorBody(client, "get_experiment_totals", { + experimentId: 884102, + }); + + expect(body.statusCode).toBe(status); + expect(body.rawResponse).toEqual({ error }); + }); + + it("returns a trends series", async () => { + const trends = { + id: 884102, + status: "running", + interval: "day", + startDateTime: "2024-01-01T00:00:00.000Z", + endDateTime: "2024-01-07T00:00:00.000Z", + variants: [], + }; + mockAxiosInstance.get.mockResolvedValue({ data: trends }); + + const result = await toolHandler( + client, + "get_experiment_trends" + )({ + experimentId: 884102, + startDateTime: "2024-01-01T00:00:00.000Z", + endDateTime: "2024-01-07T00:00:00.000Z", + }); + + expect(JSON.parse(result.content[0]?.text ?? "{}")).toEqual(trends); + }); + + it("rejects a trends request that sets only one datetime", async () => { + await expect( + toolHandler( + client, + "get_experiment_trends" + )({ + experimentId: 884102, + startDateTime: "2024-01-01T00:00:00.000Z", + }) + ).rejects.toBeInstanceOf(McpError); + expect(mockAxiosInstance.get).not.toHaveBeenCalled(); + }); + + it.each([ + { status: 400, error: "Journey experiments are not supported" }, + { status: 400, error: "Date range cannot exceed 31 days" }, + { status: 400, error: "Date range is outside the experiment run window" }, + { status: 404, error: "Experiment 884102 not found" }, + ])("returns trends status $status for $error", async ({ status, error }) => { + mockAxiosInstance.get.mockRejectedValue(restError(status, error)); + + const body = await errorBody(client, "get_experiment_trends", { + experimentId: 884102, + }); + + expect(body.statusCode).toBe(status); + expect(body.rawResponse.error).toBe(error); + }); + + it.each([ + { + name: "create_experiment", + method: "post", + toolArgs: { campaignId: 129500, experimentType: "SubjectLine" }, + url: "/api/experiments", + body: { campaignId: 129500, experimentType: "SubjectLine" }, + }, + { + name: "copy_experiment_variant", + method: "post", + toolArgs: { experimentId: 884102, copyFromTemplateId: 55 }, + url: "/api/experiments/884102/variants", + body: { copyFromTemplateId: 55 }, + }, + { + name: "update_experiment_settings", + method: "patch", + toolArgs: { experimentId: 884102, evenlySplitVariations: true }, + url: "/api/experiments/884102/settings", + body: { evenlySplitVariations: true }, + }, + { + name: "declare_experiment_winner", + method: "post", + toolArgs: { experimentId: 884102, variantId: 2 }, + url: "/api/experiments/884102/winner", + body: { variantId: 2 }, + }, + { + name: "start_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + url: "/api/experiments/884102/start", + body: {}, + }, + { + name: "cancel_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + url: "/api/experiments/884102/cancel", + body: {}, + }, + { + name: "delete_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + url: "/api/experiments/884102/delete", + body: {}, + }, + ] as const)( + "$name sends $url", + async ({ name, method, toolArgs, url, body }) => { + mockAxiosInstance[method].mockResolvedValue({ data: experiment }); + + const result = await toolHandler(client, name)(toolArgs); + + expect(mockAxiosInstance[method]).toHaveBeenCalledWith(url, body); + expect(JSON.parse(result.content[0]?.text ?? "{}").id).toBe(884102); + } + ); + + it.each([ + { + name: "create_experiment", + method: "post", + toolArgs: { campaignId: 129500, experimentType: "SubjectLine" }, + }, + { + name: "copy_experiment_variant", + method: "post", + toolArgs: { experimentId: 884102, copyFromTemplateId: 55 }, + }, + { + name: "update_experiment_settings", + method: "patch", + toolArgs: { experimentId: 884102, holdoutSettings: null }, + }, + { + name: "start_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + }, + { + name: "cancel_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + }, + { + name: "declare_experiment_winner", + method: "post", + toolArgs: { experimentId: 884102, variantId: 2 }, + }, + { + name: "delete_experiment", + method: "post", + toolArgs: { experimentId: 884102 }, + }, + ] as const)( + "$name returns 400, 404, and 409", + async ({ name, method, toolArgs }) => { + for (const [status, error] of [ + [400, "Journey experiments are not supported"], + [404, "Experiment 884102 not found"], + [409, "Experiment cannot be changed in its current status"], + ] as const) { + mockAxiosInstance[method].mockRejectedValue(restError(status, error)); + const body = await errorBody(client, name, toolArgs); + expect(body.statusCode).toBe(status); + expect(body.rawResponse.error).toBe(error); + } + } + ); +}); diff --git a/tests/unit/tools.test.ts b/tests/unit/tools.test.ts index f008b92..1b83772 100644 --- a/tests/unit/tools.test.ts +++ b/tests/unit/tools.test.ts @@ -15,19 +15,24 @@ const EXPECTED_TOOLS = [ "bulk_update_users", "cancel_campaign", "cancel_email", + "cancel_experiment", "cancel_export_job", "cancel_in_app", "cancel_push", "cancel_sms", "cancel_web_push", "cancel_whatsapp", + "copy_experiment_variant", "create_blast_campaign", + "create_experiment", "create_triggered_campaign", "create_catalog", "create_list", "create_snippet", "deactivate_triggered_campaign", + "declare_experiment_winner", "delete_catalog", + "delete_experiment", "delete_catalog_item", "delete_list", "delete_snippet", @@ -48,6 +53,8 @@ const EXPECTED_TOOLS = [ "get_embedded_messages", "get_experiment", "get_experiment_metrics", + "get_experiment_totals", + "get_experiment_trends", "get_experiment_variants", "get_export_files", "get_export_jobs", @@ -89,6 +96,7 @@ const EXPECTED_TOOLS = [ "send_sms_template_proof", "send_web_push", "send_whatsapp", + "start_experiment", "start_export_job", "subscribe_to_list", "subscribe_user_by_email", @@ -105,6 +113,7 @@ const EXPECTED_TOOLS = [ "replace_catalog_items", "update_email", "update_email_template", + "update_experiment_settings", "update_inapp_template", "update_push_template", "update_sms_template", @@ -177,7 +186,7 @@ describe("Tool Modules", () => { // Should have a reasonable number of tools (at least 107, allowing for growth) expect(allTools.length).toBeGreaterThanOrEqual(107); - expect(allTools.length).toBeLessThan(115); // Sanity check + expect(allTools.length).toBeLessThan(130); // Sanity check }); it("should have tools from all categories", () => { From 4448f72a9344f872b5e48b46bf93f709ed23582e Mon Sep 17 00:00:00 2001 From: Maria Melo Date: Wed, 30 Sep 2026 13:18:13 +0100 Subject: [PATCH 2/2] pnpm-lock --- pnpm-lock.yaml | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4544a74..dd13d55 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: 4.0.10 version: 4.0.10(zod@4.3.6) '@iterable/api': - specifier: 0.12.0 - version: 0.12.0(typescript@6.0.2) + specifier: 0.13.0 + version: 0.13.0(typescript@6.0.2) '@modelcontextprotocol/sdk': specifier: 1.18.1 version: 1.18.1 @@ -654,8 +654,8 @@ packages: resolution: {integrity: sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==} engines: {node: '>=8'} - '@iterable/api@0.12.0': - resolution: {integrity: sha512-EW4Uwbr1wWJ7sh5pi39b/U2aQvdQy62psu0ZMVqoO3j5lG1VegbOXD6FHM9FNuLRQAq8TuE5Gy9q/BKCIQHFrg==} + '@iterable/api@0.13.0': + resolution: {integrity: sha512-iF6m56WlDzSH1CCdFgANIk30cNpNVYvpOiPer8xr5AKhgVxV0NNsTByV3b7lqGw5NcDJPMAC8bjeIfArySeEwQ==} engines: {node: '>=18.0.0'} '@jest/console@30.2.0': @@ -1037,6 +1037,7 @@ packages: '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + deprecated: Potential CWE-502 - Update to 1.3.1 or higher '@unrs/resolver-binding-android-arm-eabi@1.11.1': resolution: {integrity: sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==} @@ -1534,6 +1535,7 @@ packages: eslint@9.39.1: resolution: {integrity: sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true peerDependencies: jiti: '*' @@ -1775,10 +1777,6 @@ packages: resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} engines: {node: '>= 0.4'} - hasown@2.0.2: - resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} - engines: {node: '>= 0.4'} - hasown@2.0.4: resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} engines: {node: '>= 0.4'} @@ -3264,7 +3262,7 @@ snapshots: '@istanbuljs/schema@0.1.3': {} - '@iterable/api@0.12.0(typescript@6.0.2)': + '@iterable/api@0.13.0(typescript@6.0.2)': dependencies: '@t3-oss/env-core': 0.13.11(typescript@6.0.2)(zod@4.3.6) axios: 1.13.6 @@ -4486,7 +4484,7 @@ snapshots: get-proto: 1.0.1 gopd: 1.2.0 has-symbols: 1.1.0 - hasown: 2.0.2 + hasown: 2.0.4 math-intrinsics: 1.1.0 get-package-type@0.1.0: {} @@ -4549,10 +4547,6 @@ snapshots: dependencies: has-symbols: 1.1.0 - hasown@2.0.2: - dependencies: - function-bind: 1.1.2 - hasown@2.0.4: dependencies: function-bind: 1.1.2