From 77f5ae54ae48a5d4f879e4c48c56dc674aad8aff Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Mon, 12 Oct 2026 14:26:02 +0200 Subject: [PATCH 01/21] feat(apps): add app control tools for simulators and Android devices Add MCP tools that list, launch, stop, and install apps, open deep links, change app permissions, and send test push notifications on iOS simulators and Android devices through simctl and adb. Panel-only tools read app icons, Android labels, declared URL schemes, and permission state. Unexpected command failures reach Sentry with a fixed message so command output with bundle IDs, URLs, and serials stays local. --- docs/telemetry.md | 11 + src/server/app-control.ts | 569 ++++++++++++++++++++++++++++++++++++++ src/server/plugin.ts | 2 + src/shared/app-control.ts | 33 +++ tests/app-control.test.ts | 217 +++++++++++++++ tests/telemetry.test.ts | 36 +++ 6 files changed, 868 insertions(+) create mode 100644 src/server/app-control.ts create mode 100644 src/shared/app-control.ts create mode 100644 tests/app-control.test.ts diff --git a/docs/telemetry.md b/docs/telemetry.md index dc55b43..888d079 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -163,6 +163,17 @@ so their duration is not attributed to the next surface or device. No bundle IDs PIDs, device IDs, app lists, or query output are sent. Existing CPU batch-processing coverage and native Baguette crash/resource telemetry are preserved. +Since 0.1.147, app control tools (`mobile_list_apps`, `mobile_launch_app`, +`mobile_stop_app`, `mobile_install_app`, `mobile_open_url`, `mobile_app_icons`, +`mobile_app_links`, `mobile_app_permissions`, `mobile_set_app_permission`, and +`mobile_send_push`) use the existing sampled MCP traces. Unexpected `simctl`, `adb`, +`aapt2`, or `plutil` failures are reported under `app_control.` with a fixed +message, because command output can contain bundle IDs, URLs, device serials, and +local paths. Expected outcomes such as a missing app, an unhandled URL, an +unrequested permission, or a stopped device count only through +`server.error.outcome`. Extracted app icons and Android labels stay in the server's +memory cache and are returned only to the panel. + Saved chart cards use the `recording` surface and view. Existing readiness, interaction and frame-pacing coverage is preserved. `ui.recording.process` and `ui.recording.derive` measure result validation and chart/summary processing; diff --git a/src/server/app-control.ts b/src/server/app-control.ts new file mode 100644 index 0000000..d009b45 --- /dev/null +++ b/src/server/app-control.ts @@ -0,0 +1,569 @@ +import { execFile } from "node:child_process"; +import { access, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { homedir, tmpdir } from "node:os"; +import { isAbsolute, join } from "node:path"; +import { z } from "zod"; +import { registerAppTool } from "@modelcontextprotocol/ext-apps/server"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { adbPath } from "./native-logs.ts"; +import { androidIdSchema } from "./serve-emu.ts"; +import { readDeviceApps } from "./device-apps/sources.ts"; +import { runningDeviceApps } from "./device-apps/apps.ts"; +import { foregroundAndroidPackage } from "./device-apps/foreground.ts"; +import { captureServerError } from "./telemetry.ts"; +import { errorMessage, udidSchema } from "../shared/protocol.ts"; +import { ExpectedOperationError, expectedOutcome } from "../shared/error-reporting.ts"; +import { appPermissions, appTargetShape, appUrlSchema, bundleIdSchema, permissionStates, platformPermissions } from "../shared/app-control.ts"; +import type { AppIcon, AppPermission, AppPermissionEntry, InstalledApp, InstalledApps, PermissionState } from "../shared/app-control.ts"; + +export type AppTarget = { platform: "ios" | "android"; deviceId: string }; +type CommandOptions = { timeout: number; input?: string; maxBuffer?: number }; +export type CommandRunner = (file: string, args: string[], options: CommandOptions) => Promise<{ stdout: string; stderr: string }>; +export type BinaryRunner = (file: string, args: string[], options: { timeout: number }) => Promise; + +const runCommand: CommandRunner = (file, args, { timeout, input, maxBuffer = 8 * 1024 * 1024 }) => new Promise((resolve, reject) => { + const child = execFile(file, args, { encoding: "utf8", timeout, maxBuffer }, (error, stdout, stderr) => { + if (error) reject(Object.assign(error, { stdout, stderr })); + else resolve({ stdout, stderr }); + }); + if (input !== undefined) child.stdin?.end(input); +}); + +const runBinary: BinaryRunner = (file, args, { timeout }) => new Promise((resolve, reject) => { + execFile(file, args, { encoding: "buffer", timeout, maxBuffer: 64 * 1024 * 1024 }, (error, stdout) => { + if (error) reject(error); + else resolve(stdout); + }); +}); + +async function mapLimit(items: readonly T[], limit: number, work: (item: T) => Promise) { + let next = 0; + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => { + while (next < items.length) await work(items[next++]); + })); +} + +let aapt2: Promise | undefined; +// aapt2 ships with the Android SDK build tools; use the newest installed version. +function findAapt2(): Promise { + return aapt2 ??= (async () => { + for (const root of [process.env.ANDROID_HOME, process.env.ANDROID_SDK_ROOT, join(homedir(), "Library/Android/sdk")]) { + if (!root) continue; + const versions = await readdir(join(root, "build-tools")).catch(() => []); + versions.sort((a, b) => b.localeCompare(a, undefined, { numeric: true })); + for (const version of versions) { + const path = join(root, "build-tools", version, "aapt2"); + if (await access(path).then(() => true, () => false)) return path; + } + } + return null; + })(); +} + +function commandOutput(error: unknown): string { + if (error === null || typeof error !== "object") return String(error); + const stderr = "stderr" in error && typeof error.stderr === "string" ? error.stderr : ""; + const stdout = "stdout" in error && typeof error.stdout === "string" ? error.stdout : ""; + return `${stderr}\n${stdout}`.trim(); +} + +const unavailable = /current state: Shutdown|not booted|Invalid device|device offline|device unauthorized|device '.*' not found|no devices\/emulators found/i; + +// Keep only the readable simctl or adb lines; "Command failed: ..." repeats every argument. +function commandError(error: unknown, action: string): Error { + if (expectedOutcome(error) !== undefined) return error as Error; + const output = commandOutput(error); + const message = output.split("\n").map(line => line.trim()).filter(line => line && !/^Underlying error|^An error was encountered/.test(line)).slice(0, 3).join(" "); + const text = `${action} failed${message ? `: ${message}` : "."}`; + if (unavailable.test(output)) return new ExpectedOperationError("device_unavailable", text); + if (error !== null && typeof error === "object" && "code" in error && error.code === "ENOENT") return new ExpectedOperationError("unsupported_platform", `${action} needs ${process.platform === "darwin" ? "Xcode or the Android SDK" : "macOS and Xcode"}.`); + return new Error(text, { cause: error }); +} + +function shellQuote(value: string) { return `'${value.replaceAll("'", "'\\''")}'`; } + +export function parseSimulatorApps(json: string): InstalledApp[] { + const record = z.record(z.string(), z.object({ ApplicationType: z.string().optional(), CFBundleIdentifier: z.string().optional(), CFBundleDisplayName: z.string().optional(), CFBundleName: z.string().optional() }).loose()).parse(JSON.parse(json)); + const apps: InstalledApp[] = []; + for (const [key, app] of Object.entries(record)) { + if (app.ApplicationType !== "User") continue; + const bundleId = bundleIdSchema.safeParse(app.CFBundleIdentifier ?? key); + if (bundleId.success) apps.push({ bundleId: bundleId.data, name: app.CFBundleDisplayName || app.CFBundleName || bundleId.data }); + } + return apps.sort((a, b) => a.name.localeCompare(b.name)); +} + +export function parseSimulatorAppPaths(json: string): Map { + const record = z.record(z.string(), z.object({ CFBundleIdentifier: z.string().optional(), Path: z.string().optional() }).loose()).parse(JSON.parse(json)); + return new Map(Object.entries(record).flatMap(([key, app]) => app.Path ? [[app.CFBundleIdentifier ?? key, app.Path] as const] : [])); +} + +export function parseAndroidBadging(output: string): { icon: string | null; label: string | null } { + const label = /^application-label:'([^']*)'/m.exec(output) ?? /^application: label='([^']*)'/m.exec(output); + const icon = /^application: .*\bicon='([^']+)'/m.exec(output); + return { icon: icon?.[1] ?? null, label: label?.[1] || null }; +} + +const densities: Record = { ldpi: 1, mdpi: 2, tvdpi: 3, hdpi: 4, xhdpi: 5, xxhdpi: 6, xxxhdpi: 7 }; +// Adaptive launcher icons are XML; the same resource usually keeps raster versions for older Android releases. +export function pickAndroidIconFile(resources: string, iconPath: string): string | null { + const blocks: { config: string; path: string }[][] = []; + let current: { config: string; path: string }[] | undefined; + for (const line of resources.split("\n")) { + if (/^\s+resource 0x[0-9a-f]+ /.test(line)) { current = []; blocks.push(current); continue; } + if (/^\s*(?:type|Package) /.test(line)) { current = undefined; continue; } + const file = /^\s+\(([^)]*)\) \(file\) (\S+)/.exec(line); + if (current && file) current.push({ config: file[1], path: file[2] }); + } + const entries = blocks.find(block => block.some(entry => entry.path === iconPath)) ?? [{ config: "", path: iconPath }]; + const rank = (config: string) => densities[/\b(ldpi|mdpi|tvdpi|hdpi|xhdpi|xxhdpi|xxxhdpi)\b/.exec(config)?.[1] ?? ""] ?? 0; + const rasters = entries.filter(entry => /\.(?:png|webp)$/i.test(entry.path)).sort((a, b) => rank(b.config) - rank(a.config)); + return rasters[0]?.path ?? null; +} + +export function parseAndroidPackages(output: string): InstalledApp[] { + const apps: InstalledApp[] = []; + for (const line of output.split("\n")) { + const match = /^package:([A-Za-z0-9._-]+)\s*$/.exec(line); + if (match) apps.push({ bundleId: match[1], name: match[1] }); + } + return apps.sort((a, b) => a.bundleId.localeCompare(b.bundleId)); +} + +export function parseLauncherActivity(output: string, bundleId: string): string | null { + const lines = output.split("\n").map(line => line.trim()).filter(Boolean); + const component = lines.at(-1); + return component && component.startsWith(`${bundleId}/`) && /^[A-Za-z0-9._-]+\/[A-Za-z0-9._$]+$/.test(component) ? component : null; +} + +// Sign-in, payment, and social SDKs register redirect schemes that are not app screens. +const sdkScheme = /^fb\d+$|^com\.googleusercontent\.apps\.|auth|^stripe|^link-popup$|^msal|^db-|^twitterkit-|^pdk\d|^vk\d|^tiktok/i; +const maxLinks = 5; + +// Own short schemes first, then web links, then bundle-ID-style schemes that SDKs and dev clients add. +export function rankAppLinks(links: string[]): string[] { + const rank = (link: string) => /^https?:/.test(link) ? 1 : /^[^:]*[.+][^:]*:/.test(link) ? 2 : 0; + const unique = [...new Set(links)].filter(link => !sdkScheme.test(link.slice(0, link.indexOf(":")))); + return unique.map((link, index) => ({ link, index })).sort((a, b) => rank(a.link) - rank(b.link) || a.index - b.index).slice(0, maxLinks).map(entry => entry.link); +} + +export function parseIosUrlSchemes(json: string): string[] { + const types = z.array(z.object({ CFBundleURLSchemes: z.array(z.string()).optional() }).loose()).safeParse(JSON.parse(json)); + if (!types.success) return []; + return rankAppLinks(types.data.flatMap(type => type.CFBundleURLSchemes ?? []).filter(scheme => /^[A-Za-z][A-Za-z0-9+.-]*$/.test(scheme)).map(scheme => `${scheme}://`)); +} + +export function parseAndroidAppLinks(output: string, bundleId: string): string[] { + const table = output.slice(output.indexOf("Activity Resolver Table:"), output.indexOf("Receiver Resolver Table:") >= 0 ? output.indexOf("Receiver Resolver Table:") : undefined); + type Filter = { own: boolean; view: boolean; browsable: boolean; schemes: string[]; hosts: string[]; paths: string[] }; + const filters = new Map(); + let current: Filter | undefined; + for (const line of table.split("\n")) { + const header = /^\s+[0-9a-f]+ ([A-Za-z0-9._]+)\/(\S+) filter ([0-9a-f]+)\s*$/.exec(line); + if (header) { + // Filters repeat once per scheme they declare; read each one once. + current = filters.has(header[3]) ? undefined : { own: header[1] === bundleId && (header[2].startsWith(".") || header[2].startsWith(`${bundleId}.`)), view: false, browsable: false, schemes: [], hosts: [], paths: [] }; + if (current) filters.set(header[3], current); + continue; + } + if (!current) continue; + const field = /^\s+(Action|Category|Scheme|Authority|Path): "([^"]*)"/.exec(line); + if (!field) continue; + if (field[1] === "Action" && field[2] === "android.intent.action.VIEW") current.view = true; + else if (field[1] === "Category" && field[2] === "android.intent.category.BROWSABLE") current.browsable = true; + else if (field[1] === "Scheme") current.schemes.push(field[2]); + else if (field[1] === "Authority" && /^[A-Za-z0-9.-]+$/.test(field[2])) current.hosts.push(field[2]); + else if (field[1] === "Path") { + const path = /^PatternMatcher\{(?:LITERAL|PREFIX): (\/[^}*]*)\}$/.exec(field[2]); + if (path && path[1] !== "/") current.paths.push(path[1]); + } + } + const links: string[] = []; + for (const filter of filters.values()) { + if (!filter.own || !filter.view || !filter.browsable) continue; + for (const scheme of filter.schemes) { + if (!/^[A-Za-z][A-Za-z0-9+.-]*$/.test(scheme)) continue; + if (filter.hosts.length === 0) { if (!/^https?$/.test(scheme)) links.push(`${scheme}://`); continue; } + for (const host of [...new Set(filter.hosts)].slice(0, 2)) links.push(`${scheme}://${host}${filter.paths[0] ?? "/"}`); + } + } + // Web links are listed for both http and https; offer the secure one. + return rankAppLinks(links.filter(link => !link.startsWith("http://") || !links.includes(link.replace(/^http:/, "https:")))); +} + +type RuntimePermission = { granted: boolean; userFixed: boolean }; +export function parseAndroidRuntimePermissions(output: string): Map { + if (/^Unable to find package/m.test(output)) throw new ExpectedOperationError("invalid_input", "The app is not installed on this device."); + const permissions = new Map(); + let section = false; + for (const line of output.split("\n")) { + if (/^\s*runtime permissions:\s*$/.test(line)) { section = true; continue; } + if (section === false) continue; + const match = /^\s+([A-Za-z0-9_.]+): granted=(true|false)(?:, flags=\[([^\]]*)\])?/.exec(line); + // Only the first user's runtime permission block describes the foreground user. + if (match === null) { if (permissions.size > 0) break; section = false; continue; } + permissions.set(match[1], { granted: match[2] === "true", userFixed: /\bUSER_FIXED\b/.test(match[3] ?? "") }); + } + return permissions; +} + +const androidGroups: Record = { + location: ["ACCESS_FINE_LOCATION", "ACCESS_COARSE_LOCATION"], + photos: ["READ_MEDIA_IMAGES", "READ_MEDIA_VIDEO", "READ_MEDIA_VISUAL_USER_SELECTED", "READ_EXTERNAL_STORAGE"], + camera: ["CAMERA"], + microphone: ["RECORD_AUDIO"], + notifications: ["POST_NOTIFICATIONS"], + contacts: ["READ_CONTACTS", "WRITE_CONTACTS"], + calendar: ["READ_CALENDAR", "WRITE_CALENDAR"], + reminders: [], + motion: [], +}; + +export function androidPermissionStates(runtime: Map): AppPermissionEntry[] { + const entries: AppPermissionEntry[] = []; + for (const permission of platformPermissions.android) { + const states = androidGroups[permission].flatMap(name => runtime.get(`android.permission.${name}`) ?? []); + if (states.length === 0) continue; + const state: PermissionState = states.some(entry => entry.granted) ? "allow" : states.some(entry => entry.userFixed) ? "deny" : "ask"; + entries.push({ permission, state }); + } + return entries; +} + +// Baguette cannot read the foreground app on every simulator runtime; callers then fall back to running apps. +async function simulatorForeground(deviceId: string): Promise { + return (await readDeviceApps(deviceId, undefined, "ios")).foregroundApp?.bundleId ?? null; +} + +export class AppControl { + private readonly run: CommandRunner; + private readonly adb: () => Promise; + private readonly foreground: (target: AppTarget) => Promise; + private readonly running: (target: AppTarget) => Promise<{ bundleId: string; pid: number }[]>; + private readonly readBinary: BinaryRunner; + private readonly aapt2: () => Promise; + // Keyed by bundle path and Info.plist time on iOS, and by APK path on Android, so reinstalls refresh. + private readonly iconCache = new Map(); + + constructor(options: { run?: CommandRunner; readBinary?: BinaryRunner; aapt2?: () => Promise; adb?: () => Promise; foreground?: (target: AppTarget) => Promise; running?: (target: AppTarget) => Promise<{ bundleId: string; pid: number }[]> } = {}) { + this.running = options.running ?? (target => runningDeviceApps(target.deviceId, undefined, target.platform)); + this.run = options.run ?? runCommand; + this.readBinary = options.readBinary ?? runBinary; + this.aapt2 = options.aapt2 ?? findAapt2; + this.adb = options.adb ?? adbPath; + this.foreground = options.foreground ?? (target => target.platform === "android" ? foregroundAndroidPackage(target.deviceId) : simulatorForeground(target.deviceId)); + } + + private target(target: AppTarget): AppTarget { + if (target.platform === "android") { + if (androidIdSchema.safeParse(target.deviceId).success === false || target.deviceId.startsWith("avd:")) throw new ExpectedOperationError("invalid_input", "Choose a running Android device serial."); + return target; + } + if (udidSchema.safeParse(target.deviceId).success === false) throw new ExpectedOperationError("invalid_input", "App tools support iOS simulators and Android devices. Pass an iOS simulator UDID."); + if (process.platform !== "darwin") throw new ExpectedOperationError("unsupported_platform", "iOS simulators require macOS and Xcode."); + return target; + } + + private async simctl(action: string, args: string[], options: Partial = {}) { + try { return await this.run("xcrun", ["simctl", ...args], { timeout: 30000, ...options }); } + catch (error) { throw commandError(error, action); } + } + + private async shell(action: string, deviceId: string, command: string, timeout = 15000) { + const adb = await this.adb(); + // am and pm report rejected intents and permissions on stdout, sometimes with exit status 0. + const rejected = (output: string) => { + const failure = /^(?:Error: |Exception occurred while executing)[^\n]*/m.exec(output) ?? /^Error[^\n]*/m.exec(output); + return failure ? new ExpectedOperationError("invalid_input", `${action} failed: ${failure[0]}`) : undefined; + }; + let result: { stdout: string; stderr: string }; + try { result = await this.run(adb, ["-s", deviceId, "shell", command], { timeout }); } + catch (error) { throw rejected(commandOutput(error)) ?? commandError(error, action); } + const failure = rejected(result.stdout); + if (failure) throw failure; + return result.stdout; + } + + async apps(input: AppTarget): Promise { + const target = this.target(input); + const foreground = this.foreground(target).catch(() => null); + const running = this.running(target).catch(() => []); + let apps: InstalledApp[]; + if (target.platform === "ios") { + apps = parseSimulatorApps(await this.simulatorListing(target.deviceId)); + } else apps = parseAndroidPackages(await this.shell("Listing apps", target.deviceId, "pm list packages -3")); + const bundleId = await foreground; + const installed = new Set(apps.map(app => app.bundleId)); + // Higher PIDs started later, so the first entry is the most recently launched app. + const runningBundleIds = [...new Set((await running).sort((a, b) => b.pid - a.pid).map(app => app.bundleId))].filter(id => installed.has(id)); + return { apps, foregroundBundleId: bundleId !== null && installed.has(bundleId) ? bundleId : null, runningBundleIds }; + } + + private async simulatorListing(deviceId: string) { + const listing = await this.simctl("Listing apps", ["listapps", deviceId]); + try { return (await this.run("plutil", ["-convert", "json", "-o", "-", "-"], { timeout: 10000, input: listing.stdout })).stdout; } + catch (error) { throw commandError(error, "Reading the app list"); } + } + + // Icons are best effort: an app without a readable icon keeps its initial in the panel. + async icons(input: AppTarget, bundleIds?: string[]): Promise<{ apps: Record }> { + const target = this.target(input); + const apps: Record = {}; + const collect = async (ids: readonly string[], read: (bundleId: string) => Promise) => mapLimit(ids, 3, async bundleId => { + const icon = await read(bundleId).catch((): AppIcon => ({})); + if (icon.icon || icon.name) apps[bundleId] = icon; + }); + if (target.platform === "ios") { + const listing = await this.simulatorListing(target.deviceId); + const paths = parseSimulatorAppPaths(listing); + await collect(bundleIds ?? parseSimulatorApps(listing).map(app => app.bundleId), async bundleId => { + const path = paths.get(bundleId); + return path ? this.simulatorIcon(path) : {}; + }); + return { apps }; + } + const aapt2 = await this.aapt2(); + if (!aapt2) return { apps }; + const ids = bundleIds ?? parseAndroidPackages(await this.shell("Listing apps", target.deviceId, "pm list packages -3")).map(app => app.bundleId); + await collect(ids, bundleId => this.androidIcon(target.deviceId, bundleId, aapt2)); + return { apps }; + } + + private async cached(key: string, read: () => Promise) { + const hit = this.iconCache.get(key); + if (hit) return hit; + const icon = await read(); + if (this.iconCache.size >= 200) this.iconCache.delete(this.iconCache.keys().next().value!); + this.iconCache.set(key, icon); + return icon; + } + + private async pngDataUrl(source: string, directory: string) { + const output = join(directory, "icon-64.png"); + await this.run("sips", ["-s", "format", "png", "-Z", "64", source, "--out", output], { timeout: 10000 }); + return `data:image/png;base64,${(await readFile(output)).toString("base64")}`; + } + + private async simulatorIcon(bundlePath: string): Promise { + const plist = join(bundlePath, "Info.plist"); + const info = await stat(plist); + return this.cached(`ios:${bundlePath}:${info.mtimeMs}`, async () => { + const icons = await this.run("plutil", ["-extract", "CFBundleIcons", "json", "-o", "-", plist], { timeout: 10000 }).catch(() => undefined); + const names = z.object({ CFBundlePrimaryIcon: z.object({ CFBundleIconFiles: z.array(z.string()).optional() }).loose().optional() }).loose() + .safeParse(icons ? JSON.parse(icons.stdout) : {}).data?.CFBundlePrimaryIcon?.CFBundleIconFiles ?? []; + if (names.length === 0) return {}; + const files = (await readdir(bundlePath)).filter(file => file.endsWith(".png") && names.some(name => file.startsWith(name))); + const sized = await Promise.all(files.map(async file => ({ file, size: (await stat(join(bundlePath, file))).size }))); + const largest = sized.sort((a, b) => b.size - a.size)[0]; + if (!largest) return {}; + const directory = await mkdtemp(join(tmpdir(), "mobile-dev-icon-")); + try { return { icon: await this.pngDataUrl(join(bundlePath, largest.file), directory) }; } + finally { await rm(directory, { recursive: true, force: true }); } + }); + } + + // Reads only the manifest, resource table, and icon from the device instead of pulling the whole APK. + private async androidIcon(deviceId: string, bundleId: string, aapt2: string): Promise { + const paths = await this.shell("Finding the app package", deviceId, `pm path ${bundleId}`); + const apk = paths.split("\n").map(line => line.trim().replace(/^package:/, "")).find(line => line.endsWith("base.apk")) ?? paths.trim().replace(/^package:/, "").split("\n")[0]; + if (!apk || !/^\/[\w./~=+-]+\.apk$/.test(apk)) return {}; + return this.cached(`android:${deviceId}:${apk}`, async () => { + const adb = await this.adb(); + const entry = (name: string) => this.readBinary(adb, ["-s", deviceId, "exec-out", "unzip", "-p", apk, name], { timeout: 20000 }); + const directory = await mkdtemp(join(tmpdir(), "mobile-dev-icon-")); + try { + const [manifest, resources] = await Promise.all([entry("AndroidManifest.xml"), entry("resources.arsc")]); + await Promise.all([writeFile(join(directory, "AndroidManifest.xml"), manifest), writeFile(join(directory, "resources.arsc"), resources)]); + const partial = join(directory, "partial.apk"); + await this.run("zip", ["-q", "-0", "-j", partial, join(directory, "AndroidManifest.xml"), join(directory, "resources.arsc")], { timeout: 10000 }); + const badging = parseAndroidBadging((await this.run(aapt2, ["dump", "badging", partial], { timeout: 15000 })).stdout); + const name = badging.label ?? undefined; + if (!badging.icon) return { name }; + const table = await this.run(aapt2, ["dump", "resources", partial], { timeout: 15000, maxBuffer: 64 * 1024 * 1024 }); + const file = pickAndroidIconFile(table.stdout, badging.icon); + if (!file) return { name }; + const source = join(directory, `icon${file.slice(file.lastIndexOf("."))}`); + await writeFile(source, await entry(file)); + return { name, icon: await this.pngDataUrl(source, directory) }; + } finally { await rm(directory, { recursive: true, force: true }); } + }); + } + + async launch(input: AppTarget, bundleId: string) { + const target = this.target(input); + if (target.platform === "ios") { + try { await this.simctl("Launching the app", ["launch", "--terminate-running-process", target.deviceId, bundleId]); } + catch (error) { + if (/FBSOpenApplicationServiceErrorDomain/.test(commandOutput((error as Error).cause))) throw new ExpectedOperationError("invalid_input", `${bundleId} is not installed or could not open.`); + throw error; + } + return { launched: bundleId }; + } + // Some emulators only resolve launcher activities when direct-boot matching is explicit. + const resolved = await this.shell("Finding the launch activity", target.deviceId, `cmd package resolve-activity --brief --query-flags 0x000C0000 -a android.intent.action.MAIN -c android.intent.category.LAUNCHER ${bundleId}`); + const component = parseLauncherActivity(resolved, bundleId); + if (component === null) throw new ExpectedOperationError("invalid_input", `${bundleId} is not installed or has no launcher activity.`); + await this.shell("Launching the app", target.deviceId, `am force-stop ${bundleId} && am start -W -n ${component}`, 30000); + return { launched: bundleId }; + } + + async stop(input: AppTarget, bundleId: string) { + const target = this.target(input); + if (target.platform === "android") await this.shell("Stopping the app", target.deviceId, `am force-stop ${bundleId}`); + else { + try { await this.simctl("Stopping the app", ["terminate", target.deviceId, bundleId]); } + catch (error) { if (!/found nothing to terminate/.test(commandOutput((error as Error).cause))) throw error; } + } + return { stopped: bundleId }; + } + + async install(input: AppTarget, bundlePath: string) { + const target = this.target(input); + const path = bundlePath.replace(/(.)\/+$/, "$1"); + const extension = target.platform === "ios" ? ".app" : ".apk"; + const info = isAbsolute(path) && path.endsWith(extension) ? await stat(path).catch(() => undefined) : undefined; + if (info === undefined || (target.platform === "ios" ? !info.isDirectory() : !info.isFile())) { + throw new ExpectedOperationError("invalid_input", `Pass the absolute path of a built ${extension} ${target.platform === "ios" ? "bundle for the simulator" : "file"}.`); + } + if (target.platform === "android") { + const adb = await this.adb(); + try { await this.run(adb, ["-s", target.deviceId, "install", "-r", path], { timeout: 300000 }); } + catch (error) { throw commandError(error, "Installing the app"); } + return { installed: true }; + } + await this.simctl("Installing the app", ["install", target.deviceId, path], { timeout: 300000 }); + const plist = await this.run("plutil", ["-extract", "CFBundleIdentifier", "raw", "-o", "-", `${path}/Info.plist`], { timeout: 10000 }).catch(() => undefined); + const bundleId = bundleIdSchema.safeParse(plist?.stdout.trim()); + return { installed: true, ...(bundleId.success ? { bundleId: bundleId.data } : {}) }; + } + + async openUrl(input: AppTarget, url: string) { + const target = this.target(input); + if (target.platform === "android") { + await this.shell("Opening the URL", target.deviceId, `am start -W -a android.intent.action.VIEW -d ${shellQuote(url)}`, 30000); + return { opened: url }; + } + try { await this.simctl("Opening the URL", ["openurl", target.deviceId, url]); } + catch (error) { + if (/LSApplicationWorkspaceErrorDomain/.test(commandOutput((error as Error).cause))) throw new ExpectedOperationError("invalid_input", "No installed app handles this URL."); + throw error; + } + return { opened: url }; + } + + async links(input: AppTarget, bundleId: string): Promise<{ links: string[] }> { + const target = this.target(input); + if (target.platform === "android") return { links: parseAndroidAppLinks(await this.shell("Reading app links", target.deviceId, `dumpsys package ${bundleId}`), bundleId) }; + const container = await this.simctl("Reading app links", ["get_app_container", target.deviceId, bundleId, "app"]).catch(error => { + throw expectedOutcome(error) === undefined ? new ExpectedOperationError("invalid_input", `${bundleId} is not installed on this simulator.`) : error; + }); + // Apps without URL types make plutil exit with "No value at that key path". + const types = await this.run("plutil", ["-extract", "CFBundleURLTypes", "json", "-o", "-", `${container.stdout.trim()}/Info.plist`], { timeout: 10000 }).catch(() => undefined); + return { links: types ? parseIosUrlSchemes(types.stdout) : [] }; + } + + private async androidRuntime(deviceId: string, bundleId: string) { + return parseAndroidRuntimePermissions(await this.shell("Reading app permissions", deviceId, `dumpsys package ${bundleId}`)); + } + + async permissions(input: AppTarget, bundleId: string): Promise<{ permissions: AppPermissionEntry[] }> { + const target = this.target(input); + // The simulator does not expose privacy decisions through simctl. + if (target.platform === "ios") return { permissions: platformPermissions.ios.map(permission => ({ permission, state: null })) }; + return { permissions: androidPermissionStates(await this.androidRuntime(target.deviceId, bundleId)) }; + } + + async setPermission(input: AppTarget, bundleId: string, permission: AppPermission | "all", state: PermissionState) { + const target = this.target(input); + if (permission !== "all" && !platformPermissions[target.platform].includes(permission)) { + throw new ExpectedOperationError("invalid_input", `${target.platform === "ios" ? "iOS simulators do" : "Android does"} not support the ${permission} permission here.`); + } + if (target.platform === "ios") { + const action = { allow: "grant", deny: "revoke", ask: "reset" }[state]; + await this.simctl("Changing the permission", ["privacy", target.deviceId, action, permission, bundleId]); + return { permissions: permission === "all" ? platformPermissions.ios.map(entry => ({ permission: entry, state })) : [{ permission, state }] }; + } + const runtime = await this.androidRuntime(target.deviceId, bundleId); + const groups = permission === "all" ? platformPermissions.android : [permission]; + const names = groups.flatMap(group => androidGroups[group].map(name => `android.permission.${name}`)).filter(name => runtime.has(name)); + if (names.length === 0) throw new ExpectedOperationError("invalid_input", permission === "all" ? "The app does not request any supported runtime permissions." : `The app does not request the ${permission} permission.`); + // USER_FIXED makes a denial final, matching "Don't allow"; clearing it makes Android ask again. + const commands = names.flatMap(name => state === "allow" + ? [`pm clear-permission-flags ${bundleId} ${name} user-set user-fixed`, `pm grant ${bundleId} ${name}`] + : [`pm revoke ${bundleId} ${name}`, `pm ${state === "deny" ? "set" : "clear"}-permission-flags ${bundleId} ${name} user-set user-fixed`]); + await this.shell("Changing the permission", target.deviceId, commands.join(" && ")); + return { permissions: androidPermissionStates(await this.androidRuntime(target.deviceId, bundleId)) }; + } + + async push(input: AppTarget, bundleId: string, payload: Record) { + const target = this.target(input); + if (target.platform === "android") throw new ExpectedOperationError("unsupported_platform", "Test push notifications are available on iOS simulators. Android needs Firebase Cloud Messaging."); + const json = JSON.stringify(payload); + if (Buffer.byteLength(json) > 4096) throw new ExpectedOperationError("invalid_input", "Push payloads must be 4 KB or smaller."); + try { await this.simctl("Sending the notification", ["push", target.deviceId, bundleId, "-"], { input: json }); } + catch (error) { + if (/UNErrorDomain/.test(commandOutput((error as Error).cause))) throw new ExpectedOperationError("invalid_input", `${bundleId} is not installed or cannot receive notifications.`); + throw error; + } + return { sent: true }; + } +} + +const bundleId = bundleIdSchema.describe("App bundle ID or Android package name."); +const payloadSchema = z.record(z.string(), z.unknown()).refine(value => typeof value.aps === "object" && value.aps !== null, "Payload needs an aps object."); + +export function registerAppControlTools(server: McpServer, control = new AppControl()) { + const tool = (name: string, action: string, config: { title: string; description: string; inputSchema: z.ZodRawShape; readOnly?: boolean; destructive?: boolean; model?: boolean }, + handler: (input: T) => Promise) => { + registerAppTool(server, name, { + title: config.title, description: config.description, inputSchema: { ...appTargetShape, ...config.inputSchema }, + annotations: { readOnlyHint: config.readOnly ?? false, destructiveHint: config.destructive ?? false, openWorldHint: false }, + _meta: { ui: { visibility: config.model === false ? ["app"] : ["app", "model"] } }, + }, async input => { + try { + const result = await handler(input as T); + return { content: [{ type: "text" as const, text: JSON.stringify(result) }], structuredContent: result as Record }; + } catch (error) { + // Command output can contain app IDs, URLs, and device serials, so Sentry receives only the action. + if (expectedOutcome(error) === undefined) captureServerError(new Error(`App ${action} command failed.`), `app_control.${action}`); + return { isError: true, content: [{ type: "text" as const, text: errorMessage(error) }] }; + } + }); + }; + const target = "Selected iOS simulator or Android device."; + tool("mobile_list_apps", "list", { title: "List installed apps", readOnly: true, inputSchema: {}, + description: `List user-installed apps on the ${target} Returns bundle IDs, names, the foreground app, and running apps, most recent first.` }, + input => control.apps(input)); + tool("mobile_launch_app", "launch", { title: "Launch app", inputSchema: { bundleId }, + description: "Launch an installed app, restarting it if it is already running. Does not build; use the project's build tools first." }, + input => control.launch(input, input.bundleId)); + tool("mobile_stop_app", "stop", { title: "Stop app", inputSchema: { bundleId }, + description: "Terminate a running app on the device." }, + input => control.stop(input, input.bundleId)); + tool("mobile_install_app", "install", { title: "Install app", destructive: true, + inputSchema: { path: z.string().min(1).max(4096).describe("Absolute path to a simulator .app bundle or an .apk.") }, + description: "Install or replace a built app on the device. Build it with the project's tools first." }, + input => control.install(input, input.path)); + tool("mobile_open_url", "open_url", { title: "Open URL", + inputSchema: { url: appUrlSchema.describe("Deep link or web URL, such as myapp://checkout.") }, + description: "Open a deep link or URL on the device to jump straight to a screen." }, + input => control.openUrl(input, input.url)); + tool("mobile_app_icons", "icons", { title: "Read app icons", readOnly: true, model: false, + inputSchema: { bundleIds: z.array(bundleIdSchema).max(200).optional() }, + description: "Read small PNG icons for installed apps, plus Android app labels." }, + input => control.icons(input, input.bundleIds)); + tool("mobile_app_links", "links", { title: "Read app URL schemes", readOnly: true, model: false, inputSchema: { bundleId }, + description: "Read the URL schemes and Android link hosts an app declares, without SDK redirect schemes." }, + input => control.links(input, input.bundleId)); + tool("mobile_app_permissions", "permissions", { title: "Read app permissions", readOnly: true, model: false, inputSchema: { bundleId }, + description: "Read supported app permissions. Android reports current states; iOS simulators report null." }, + input => control.permissions(input, input.bundleId)); + tool("mobile_set_app_permission", "permission", { title: "Set app permission", + inputSchema: { bundleId, permission: z.enum([...appPermissions, "all"]), state: z.enum(permissionStates).describe("allow grants, deny blocks, ask resets so the app prompts again.") }, + description: "Allow, deny, or reset an app permission without the system prompt. iOS simulators lack camera and notifications; Android lacks reminders and motion. Changes may restart the app." }, + input => control.setPermission(input, input.bundleId, input.permission, input.state)); + tool }>("mobile_send_push", "push", { title: "Send test push", + inputSchema: { bundleId, payload: payloadSchema.describe("APNs payload with an aps object, such as {\"aps\":{\"alert\":\"Hi\"}}.") }, + description: "Send a test push notification to an app on an iOS simulator." }, + input => control.push(input, input.bundleId, input.payload)); +} diff --git a/src/server/plugin.ts b/src/server/plugin.ts index 4bf0fd3..9a9abb4 100644 --- a/src/server/plugin.ts +++ b/src/server/plugin.ts @@ -15,6 +15,7 @@ import { registerAppResource, registerAppTool, RESOURCE_MIME_TYPE } from "@model import { OpenAIExtensions } from "@openai/mcp-extensions/server"; import { z } from "zod"; import { registerDeviceSettingsTools } from "./device-settings.ts"; +import { registerAppControlTools } from "./app-control.ts"; import { readBezel } from "./bezel.ts"; import { ServeEmu } from "./serve-emu.ts"; import { registerAndroidTools } from "./android-tools.ts"; @@ -143,6 +144,7 @@ export async function createPlugin(html: string | (() => Promise), b await Promise.all([closeCpu, closeFps]); }); registerDeviceSettingsTools(server, baguette, android); + registerAppControlTools(server); const automaticRepairs = new Map }>(); async function panelInputStatus(udid: string) { diff --git a/src/shared/app-control.ts b/src/shared/app-control.ts new file mode 100644 index 0000000..030db26 --- /dev/null +++ b/src/shared/app-control.ts @@ -0,0 +1,33 @@ +import { z } from "zod"; + +export const appPermissions = ["location", "photos", "camera", "microphone", "notifications", "contacts", "calendar", "reminders", "motion"] as const; +export type AppPermission = typeof appPermissions[number]; +// simctl privacy has no camera or notification services; Android has no reminders or motion runtime permissions. +export const platformPermissions: Record<"ios" | "android", readonly AppPermission[]> = { + ios: ["location", "photos", "microphone", "contacts", "calendar", "reminders", "motion"], + android: ["location", "photos", "camera", "microphone", "notifications", "contacts", "calendar"], +}; +export const permissionStates = ["allow", "deny", "ask"] as const; +export type PermissionState = typeof permissionStates[number]; + +export const bundleIdSchema = z.string().min(1).max(255).regex(/^[A-Za-z0-9_][A-Za-z0-9._-]*$/); +export const appUrlSchema = z.string().min(3).max(2048).regex(/^[A-Za-z][A-Za-z0-9+.-]*:\S+$/, "Expected a URL with a scheme, such as myapp://checkout."); +export const appTargetShape = { + platform: z.enum(["ios", "android"]), + deviceId: z.string().min(1).max(256), +}; + +const installedApp = z.object({ bundleId: bundleIdSchema, name: z.string() }); +export const installedAppsSchema = z.object({ apps: z.array(installedApp), foregroundBundleId: bundleIdSchema.nullable(), runningBundleIds: z.array(bundleIdSchema).default([]) }); +export type InstalledApp = z.infer; +export type InstalledApps = z.infer; + +const permissionEntry = z.object({ permission: z.enum(appPermissions), state: z.enum(permissionStates).nullable() }); +export const appPermissionsSchema = z.object({ permissions: z.array(permissionEntry) }); +export type AppPermissionEntry = z.infer; + +export const appLinksSchema = z.object({ links: z.array(z.string().max(2048)) }); + +const appIcon = z.object({ icon: z.string().startsWith("data:image/png;base64,").max(256 * 1024).optional(), name: z.string().max(256).optional() }); +export const appIconsSchema = z.object({ apps: z.record(z.string(), appIcon) }); +export type AppIcon = z.infer; diff --git a/tests/app-control.test.ts b/tests/app-control.test.ts new file mode 100644 index 0000000..0168369 --- /dev/null +++ b/tests/app-control.test.ts @@ -0,0 +1,217 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, mkdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { AppControl, androidPermissionStates, parseAndroidAppLinks, parseAndroidBadging, pickAndroidIconFile, parseAndroidRuntimePermissions, parseIosUrlSchemes, parseLauncherActivity, parseSimulatorApps } from "../src/server/app-control.ts"; +import type { CommandRunner } from "../src/server/app-control.ts"; +import { expectedOutcome } from "../src/shared/error-reporting.ts"; + +const ios = { platform: "ios", deviceId: "B5C969F6-58A4-4C31-AB12-FB9E56D681DE" } as const; +const android = { platform: "android", deviceId: "emulator-5554" } as const; +const dumpsys = `Packages: + requested permissions: + android.permission.CAMERA + install permissions: + android.permission.INTERNET: granted=true + User 0: ceDataInode=1 + runtime permissions: + android.permission.CAMERA: granted=false, flags=[ USER_SET|USER_FIXED|USER_SENSITIVE_WHEN_GRANTED] + android.permission.ACCESS_FINE_LOCATION: granted=true, flags=[ USER_SET] + android.permission.ACCESS_COARSE_LOCATION: granted=false, flags=[ ] + android.permission.RECORD_AUDIO: granted=false, flags=[ USER_SENSITIVE_WHEN_GRANTED] + enabledComponents: + User 10: ceDataInode=2 + runtime permissions: + android.permission.CAMERA: granted=true, flags=[ ] +`; + +function failure(stderr: string, code = 1) { return Object.assign(new Error(`Command failed: ${stderr}`), { code, stderr, stdout: "" }); } +function fixture(respond: (file: string, args: string[], input?: string) => string | Error = () => "") { + const calls: { file: string; args: string[]; input?: string }[] = []; + const run: CommandRunner = async (file, args, options) => { + calls.push({ file, args, ...(options.input === undefined ? {} : { input: options.input }) }); + const result = respond(file, args, options.input); + if (result instanceof Error) throw result; + return { stdout: result, stderr: "" }; + }; + const control = new AppControl({ run, adb: async () => "adb", foreground: async target => target.platform === "ios" ? "com.acme.shop" : "com.acme.android", + running: async () => [{ bundleId: "com.other", pid: 300 }, { bundleId: "com.acme.shop", pid: 100 }, { bundleId: "com.acme.android", pid: 200 }] }); + return { control, calls }; +} + +test("simulator listings keep user apps with display names", () => { + const apps = parseSimulatorApps(JSON.stringify({ + "com.apple.mobilesafari": { ApplicationType: "System", CFBundleIdentifier: "com.apple.mobilesafari" }, + "com.acme.shop": { ApplicationType: "User", CFBundleIdentifier: "com.acme.shop", CFBundleDisplayName: "Acme Shop", CFBundleName: "Shop" }, + "dev.zeta": { ApplicationType: "User", CFBundleName: "Zeta" }, + })); + assert.deepEqual(apps, [{ bundleId: "com.acme.shop", name: "Acme Shop" }, { bundleId: "dev.zeta", name: "Zeta" }]); +}); + +test("Android launcher and permission output parse into stable values", () => { + assert.equal(parseLauncherActivity("priority=0 match=0x108000\ncom.acme/.MainActivity\n", "com.acme"), "com.acme/.MainActivity"); + assert.equal(parseLauncherActivity("No activity found\n", "com.acme"), null); + assert.equal(parseLauncherActivity("com.other/.Main\n", "com.acme"), null); + const runtime = parseAndroidRuntimePermissions(dumpsys); + assert.equal(runtime.size, 4, "only the first user's runtime block is read"); + assert.deepEqual(androidPermissionStates(runtime), [ + { permission: "location", state: "allow" }, { permission: "camera", state: "deny" }, { permission: "microphone", state: "ask" }, + ]); + assert.throws(() => parseAndroidRuntimePermissions("Unable to find package: com.nope\n"), (error: unknown) => expectedOutcome(error) === "invalid_input"); +}); + +test("iOS simulator actions use simctl and report the foreground app", async () => { + const { control, calls } = fixture((file, args) => { + if (args[1] === "listapps") return "{ }"; + if (file === "plutil") return JSON.stringify({ "com.acme.shop": { ApplicationType: "User", CFBundleIdentifier: "com.acme.shop", CFBundleName: "Shop" } }); + if (args[1] === "terminate") return failure("Simulator device failed to terminate com.acme.shop.\nfound nothing to terminate", 3); + return ""; + }); + assert.deepEqual(await control.apps(ios), { apps: [{ bundleId: "com.acme.shop", name: "Shop" }], foregroundBundleId: "com.acme.shop", runningBundleIds: ["com.acme.shop"] }); + assert.equal(calls[1].input, "{ }", "plutil converts the simctl plist from stdin"); + await control.launch(ios, "com.acme.shop"); + await control.stop(ios, "com.acme.shop"); + await control.openUrl(ios, "myapp://checkout?step=2&x='1'"); + assert.deepEqual(await control.setPermission(ios, "com.acme.shop", "photos", "deny"), { permissions: [{ permission: "photos", state: "deny" }] }); + await control.push(ios, "com.acme.shop", { aps: { alert: "Hi" } }); + assert.deepEqual(calls.slice(2).map(call => call.args), [ + ["simctl", "launch", "--terminate-running-process", ios.deviceId, "com.acme.shop"], + ["simctl", "terminate", ios.deviceId, "com.acme.shop"], + ["simctl", "openurl", ios.deviceId, "myapp://checkout?step=2&x='1'"], + ["simctl", "privacy", ios.deviceId, "revoke", "photos", "com.acme.shop"], + ["simctl", "push", ios.deviceId, "com.acme.shop", "-"], + ]); + assert.equal(calls.at(-1)?.input, '{"aps":{"alert":"Hi"}}'); +}); + +test("Android actions resolve launchers, quote URLs, and only touch requested permissions", async () => { + const { control, calls } = fixture((_file, args) => { + const command = args[3] ?? ""; + if (command.startsWith("pm list packages")) return "package:com.acme.android\npackage:com.other\n"; + if (command.startsWith("cmd package resolve-activity")) return "priority=0\ncom.acme.android/.MainActivity\n"; + if (command.startsWith("dumpsys package")) return dumpsys; + return "Status: ok\n"; + }); + assert.equal((await control.apps(android)).foregroundBundleId, "com.acme.android"); + assert.deepEqual((await control.apps(android)).runningBundleIds, ["com.other", "com.acme.android"], "running apps are newest first and installed"); + await control.launch(android, "com.acme.android"); + await control.openUrl(android, "myapp://checkout?step=2&x='1'"); + await control.setPermission(android, "com.acme.android", "location", "ask"); + await control.setPermission(android, "com.acme.android", "camera", "allow"); + const shell = calls.map(call => call.args[3]); + assert.ok(shell.includes("am force-stop com.acme.android && am start -W -n com.acme.android/.MainActivity")); + assert.ok(shell.includes(`am start -W -a android.intent.action.VIEW -d 'myapp://checkout?step=2&x='\\''1'\\'''`)); + assert.ok(shell.includes([ + "pm revoke com.acme.android android.permission.ACCESS_FINE_LOCATION", "pm clear-permission-flags com.acme.android android.permission.ACCESS_FINE_LOCATION user-set user-fixed", + "pm revoke com.acme.android android.permission.ACCESS_COARSE_LOCATION", "pm clear-permission-flags com.acme.android android.permission.ACCESS_COARSE_LOCATION user-set user-fixed", + ].join(" && "))); + assert.ok(shell.includes("pm clear-permission-flags com.acme.android android.permission.CAMERA user-set user-fixed && pm grant com.acme.android android.permission.CAMERA")); + await assert.rejects(control.setPermission(android, "com.acme.android", "contacts", "allow"), /does not request the contacts permission/); + await assert.rejects(control.push(android, "com.acme.android", { aps: {} }), (error: unknown) => expectedOutcome(error) === "unsupported_platform"); +}); + +test("command failures become readable, expected outcomes where the user can fix them", async () => { + const { control } = fixture((_file, args) => { + if (args[1] === "launch") return failure("An error was encountered processing the command (domain=FBSOpenApplicationServiceErrorDomain, code=4):\nSimulator device failed to launch com.nope.", 4); + if (args[1] === "openurl") return failure("Unable to lookup in current state: Shutdown", 149); + if (args[1] === "privacy") return failure("Something unexpected", 1); + if ((args[3] ?? "").includes("am start")) return "Starting: Intent\nError type 3\nError: Activity class {com.acme/.Main} does not exist.\n"; + if ((args[3] ?? "").startsWith("cmd package resolve-activity")) return "com.acme/.Main\n"; + return ""; + }); + await assert.rejects(control.launch(ios, "com.nope"), (error: unknown) => expectedOutcome(error) === "invalid_input"); + await assert.rejects(control.openUrl(ios, "myapp://x"), (error: unknown) => expectedOutcome(error) === "device_unavailable"); + await assert.rejects(control.setPermission(ios, "com.acme", "photos", "allow"), (error: unknown) => { + assert.equal(expectedOutcome(error), undefined); + assert.equal((error as Error).message, "Changing the permission failed: Something unexpected"); + return true; + }); + await assert.rejects(control.launch(android, "com.acme"), /Error: Activity class \{com\.acme\/\.Main\} does not exist/); + await assert.rejects(control.apps({ platform: "ios", deviceId: "00008110-001A2B3C4D5E801E" }), (error: unknown) => expectedOutcome(error) === "invalid_input"); + await assert.rejects(control.setPermission(ios, "com.acme", "camera", "allow"), /iOS simulators do not support the camera permission/); +}); + +test("installs require a built bundle of the platform's type", async () => { + const directory = await mkdtemp(join(tmpdir(), "app-control-")); + const bundle = join(directory, "Acme.app"); + await mkdir(bundle); + const { control, calls } = fixture((file, args) => file === "plutil" && args[0] === "-extract" ? "com.acme.shop\n" : ""); + assert.deepEqual(await control.install(ios, `${bundle}/`), { installed: true, bundleId: "com.acme.shop" }); + assert.deepEqual(calls[0].args, ["simctl", "install", ios.deviceId, bundle]); + await assert.rejects(control.install(ios, "Acme.app"), (error: unknown) => expectedOutcome(error) === "invalid_input"); + await assert.rejects(control.install(android, bundle), /absolute path of a built \.apk file/); +}); + +test("app link suggestions keep the app's own schemes and drop SDK redirects", async () => { + assert.deepEqual(parseIosUrlSchemes(JSON.stringify([ + { CFBundleURLName: "com.acme.shop", CFBundleURLSchemes: ["com.acme.shop", "acme", "fb1696089354000816", "expauth"] }, + { CFBundleURLSchemes: ["com.googleusercontent.apps.603386649315-abc", "exp+acme-shop"] }, + ])), ["acme://", "com.acme.shop://", "exp+acme-shop://"]); + assert.deepEqual(parseIosUrlSchemes("{}"), []); + const dumpsys = `Activity Resolver Table: + Schemes: + acme: + f06257 com.acme/.MainActivity filter a9a9144 + Action: "android.intent.action.VIEW" + Category: "android.intent.category.DEFAULT" + Category: "android.intent.category.BROWSABLE" + Scheme: "acme" + stripe: + 72eccf3 com.acme/com.stripe.android.RedirectActivity filter 8517db0 + Action: "android.intent.action.VIEW" + Category: "android.intent.category.BROWSABLE" + Scheme: "acmepay" + http: + dc9d03 com.acme/com.acme.LinkActivity filter 6b24ab9 + Action: "android.intent.action.VIEW" + Category: "android.intent.category.BROWSABLE" + Scheme: "https" + Scheme: "http" + Authority: "acme.dev": -1 + Path: "PatternMatcher{PREFIX: /p}" + https: + dc9d03 com.acme/com.acme.LinkActivity filter 6b24ab9 + Action: "android.intent.action.VIEW" + Scheme: "https" + launcher: + 11aa22 com.acme/.MainActivity filter 33bb44 + Action: "android.intent.action.MAIN" + Scheme: "internal" + +Receiver Resolver Table: + acme: + 44cc55 com.acme/.Receiver filter 55dd66 + Action: "android.intent.action.VIEW" + Category: "android.intent.category.BROWSABLE" + Scheme: "receiver" +`; + assert.deepEqual(parseAndroidAppLinks(dumpsys, "com.acme"), ["acme://", "https://acme.dev/p"]); + const { control, calls } = fixture((file, args) => { + if (args[1] === "get_app_container") return "/tmp/Acme.app\n"; + if (file === "plutil") return JSON.stringify([{ CFBundleURLSchemes: ["acme"] }]); + return ""; + }); + assert.deepEqual(await control.links(ios, "com.acme.shop"), { links: ["acme://"] }); + assert.deepEqual(calls.map(call => call.args.at(-1)), ["app", "/tmp/Acme.app/Info.plist"]); +}); + +test("Android icons resolve adaptive launcher icons to their sharpest raster version", () => { + assert.deepEqual(parseAndroidBadging("package: name='com.acme'\napplication-label:'Acme Shop'\napplication: label='Acme Shop' icon='res/mipmap-anydpi-v26/ic_launcher.xml'\n"), + { icon: "res/mipmap-anydpi-v26/ic_launcher.xml", label: "Acme Shop" }); + assert.deepEqual(parseAndroidBadging("package: name='com.acme'\n"), { icon: null, label: null }); + const resources = `Package name=com.acme id=7f + type mipmap id=10 entryCount=2 + resource 0x7f100000 mipmap/ic_launcher + (mdpi) (file) res/mipmap-mdpi-v4/ic_launcher.webp type=PNG + (xxxhdpi) (file) res/mipmap-xxxhdpi-v4/ic_launcher.webp type=PNG + (xhdpi) (file) res/mipmap-xhdpi-v4/ic_launcher.webp type=PNG + (anydpi-v26) (file) res/mipmap-anydpi-v26/ic_launcher.xml type=XML + resource 0x7f100001 mipmap/ic_launcher_round + (xxxhdpi) (file) res/mipmap-xxxhdpi-v4/ic_launcher_round.webp type=PNG + type plurals id=11 entryCount=1 +`; + assert.equal(pickAndroidIconFile(resources, "res/mipmap-anydpi-v26/ic_launcher.xml"), "res/mipmap-xxxhdpi-v4/ic_launcher.webp"); + assert.equal(pickAndroidIconFile(resources, "res/legacy.png"), "res/legacy.png", "an unlisted raster icon is used directly"); + assert.equal(pickAndroidIconFile(resources, "res/vector.xml"), null); +}); diff --git a/tests/telemetry.test.ts b/tests/telemetry.test.ts index a3a77db..49895dc 100644 --- a/tests/telemetry.test.ts +++ b/tests/telemetry.test.ts @@ -18,6 +18,7 @@ import { registerDeviceAppsTool } from "../src/server/device-apps/tools.ts"; import { directoryBytes } from "../src/server/storage-metrics.ts"; import { SimulatorUnavailableError } from "../src/server/simulator-unavailable.ts"; import { registerDeviceChoiceTools } from "../src/server/device-choice-tools.ts"; +import { AppControl, registerAppControlTools } from "../src/server/app-control.ts"; import type { OpenAIFormResult } from "@openai/mcp-extensions/server"; import { adapterClient } from "./agent-device-fixtures.ts"; @@ -792,3 +793,38 @@ test("cancellation during stream response preparation closes the session and ret contains(encoded, "stream.cancel_cleanup"); contains(encoded, "Unexpected stream cleanup failure"); }); + +test("app control reports unexpected command failures without command output and skips expected outcomes", async t => { + const envelopes: Envelope[] = []; + Sentry.init({ + dsn: "https://public@example.com/1", defaultIntegrations: false, environment: "release", beforeSend: scrubErrorEvent, + transport: () => ({ async send(envelope) { envelopes.push(envelope); return { statusCode: 200 }; }, async flush() { return true; } }), + }); + const run = async (_file: string, args: string[]) => { + const command = args.join(" "); + if (command.includes("openurl")) throw Object.assign(new Error(`Command failed: xcrun simctl ${command}`), { code: 1, stderr: "PRIVATE_FAILURE for myapp://PRIVATE_ROUTE", stdout: "" }); + throw Object.assign(new Error("Command failed"), { code: 4, stderr: "FBSOpenApplicationServiceErrorDomain PRIVATE_BUNDLE", stdout: "" }); + }; + const server = new McpServer({ name: "app-control-telemetry", version: "1" }); + instrumentMcpServer(server); + registerAppControlTools(server, new AppControl({ run, adb: async () => "adb", foreground: async () => null })); + const client = new Client({ name: "test", version: "1" }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + t.after(async () => { await client.close(); await server.close(); await Sentry.close(); }); + await server.connect(serverTransport); + await client.connect(clientTransport); + const target = { platform: "ios", deviceId: "B5C969F6-58A4-4C31-AB12-FB9E56D681DE" }; + const failed = await client.callTool({ name: "mobile_open_url", arguments: { ...target, url: "myapp://PRIVATE_ROUTE" } }); + assert.equal(failed.isError, true); + assert.match(JSON.stringify(failed.content), /PRIVATE_FAILURE/, "the agent still sees the command error"); + const missing = await client.callTool({ name: "mobile_launch_app", arguments: { ...target, bundleId: "com.private.app" } }); + assert.equal(missing.isError, true); + await Sentry.flush(); + const events = envelopes.flatMap(envelope => envelope[1]).filter(item => item[0].type === "event").map(item => item[1]); + assert.equal(events.length, 1); + const encoded = JSON.stringify(events); + contains(encoded, "App open_url command failed."); + contains(encoded, '"operation":"app_control.open_url"'); + contains(encoded, "PRIVATE", false); + contains(encoded, "B5C969F6", false); +}); From 55cfcefcbcb53b2433c1eb6e131e0f6fb2c076c8 Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Mon, 12 Oct 2026 14:26:03 +0200 Subject: [PATCH 02/21] feat(settings): add location presets to device settings --- src/ui/components/device-settings.tsx | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ui/components/device-settings.tsx b/src/ui/components/device-settings.tsx index a7e7c2c..5c39083 100644 --- a/src/ui/components/device-settings.tsx +++ b/src/ui/components/device-settings.tsx @@ -12,6 +12,15 @@ import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "./ui/collap const textSizeLabels = ["Extra small", "Small", "Medium", "Large", "Extra large", "2× large", "3× large", "Accessibility medium", "Accessibility large", "Accessibility extra large", "Accessibility 2× large", "Accessibility 3× large"]; const contentSizeOptions = contentSizes.map((value, index) => ({ value, label: textSizeLabels[index] })); +const locationPresets = [ + { value: "cupertino", label: "Cupertino, CA", latitude: 37.3349, longitude: -122.009 }, + { value: "san-francisco", label: "San Francisco, CA", latitude: 37.7749, longitude: -122.4194 }, + { value: "new-york", label: "New York, NY", latitude: 40.7128, longitude: -74.006 }, + { value: "london", label: "London, UK", latitude: 51.5074, longitude: -0.1278 }, + { value: "berlin", label: "Berlin, Germany", latitude: 52.52, longitude: 13.405 }, + { value: "tokyo", label: "Tokyo, Japan", latitude: 35.6762, longitude: 139.6503 }, + { value: "sydney", label: "Sydney, Australia", latitude: -33.8688, longitude: 151.2093 }, +]; const rotationOptions = [{ value: "auto", label: "Auto" }, { value: "portrait", label: "Portrait" }, { value: "landscape", label: "Landscape" }]; function SettingsSelect({ id, value, items, disabled, onChange }: { id: string; value: string; items: { value: string; label: string }[]; disabled: boolean; onChange: (value: string) => void }) { @@ -37,6 +46,7 @@ export function DeviceSettings({ platform }: { platform: "ios" | "android" }) { }, [state.deviceId, settings?.location?.latitude, settings?.location?.longitude]); useEffect(() => { if (state.disabled) setOpen(false); }, [state.disabled]); useEffect(() => { setOpen(false); setLocationOpen(false); }, [state.deviceId]); + const preset = settings?.location ? locationPresets.find(entry => entry.latitude === settings.location!.latitude && entry.longitude === settings.location!.longitude)?.value ?? "custom" : "custom"; return
{ if (element) bindDeviceSettings(element, store); }}> { if (!value) { setOpen(false); return; } @@ -56,7 +66,8 @@ export function DeviceSettings({ platform }: { platform: "ios" | "android" }) { {settings.orientation !== undefined &&
void store.change({ setting: "orientation", value: value as "auto" | "portrait" | "landscape" })} />
} {settings.locationSupported && Location -
{ event.preventDefault(); if (latitude.trim() && longitude.trim()) void store.change({ setting: "location", value: { latitude: Number(latitude), longitude: Number(longitude) } }); }}> +
{ const entry = locationPresets.find(item => item.value === value); if (entry) void store.change({ setting: "location", value: { latitude: entry.latitude, longitude: entry.longitude } }); }} />
+ { event.preventDefault(); if (latitude.trim() && longitude.trim()) void store.change({ setting: "location", value: { latitude: Number(latitude), longitude: Number(longitude) } }); }}>
setLatitude(event.target.value)} />
setLongitude(event.target.value)} />
{platform === "ios" && }
} From 5c1124eee7891a9512fa17a964b00ce66d2b3cda Mon Sep 17 00:00:00 2001 From: Laurids Kern Date: Mon, 12 Oct 2026 14:26:11 +0200 Subject: [PATCH 03/21] feat(settings): add App tab for controlling the selected app Split the device popover into App and Device tabs. The App tab picks an installed app with its icon, launches it on selection, relaunches or stops it, opens URLs with declared-scheme and recent suggestions, and sets permissions and test pushes. Without a detected foreground app, the picker opens on the newest running app, then the last app picked on that device. --- docs/telemetry.md | 6 + src/ui/app-tools.ts | 251 ++++++++++++++++++++++++++ src/ui/components/app-tools.tsx | 159 ++++++++++++++++ src/ui/components/device-settings.tsx | 17 +- src/ui/simulator-panel.ts | 9 + tests/app-tools.test.ts | 99 ++++++++++ tests/react-ui.test.ts | 42 ++++- 7 files changed, 579 insertions(+), 4 deletions(-) create mode 100644 src/ui/app-tools.ts create mode 100644 src/ui/components/app-tools.tsx create mode 100644 tests/app-tools.test.ts diff --git a/docs/telemetry.md b/docs/telemetry.md index 888d079..ed91356 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -174,6 +174,12 @@ unrequested permission, or a stopped device count only through `server.error.outcome`. Extracted app icons and Android labels stay in the server's memory cache and are returned only to the panel. +The device popover's App tab records `ui.app_tools.list`, `.icons`, `.launch`, +`.stop`, `.open_url`, `.permission`, and `.push` round trips in milliseconds with +bounded aggregate windows, and counts `ui.app_tools.failure`. Results that finish +after a surface or device change are not recorded. Recent URLs and the last picked +app stay in the panel's local storage and are never sent. + Saved chart cards use the `recording` surface and view. Existing readiness, interaction and frame-pacing coverage is preserved. `ui.recording.process` and `ui.recording.derive` measure result validation and chart/summary processing; diff --git a/src/ui/app-tools.ts b/src/ui/app-tools.ts new file mode 100644 index 0000000..c3ddc3f --- /dev/null +++ b/src/ui/app-tools.ts @@ -0,0 +1,251 @@ +import { appIconsSchema, appLinksSchema, appPermissionsSchema, installedAppsSchema, platformPermissions } from "../shared/app-control.ts"; +import type { AppPermission, AppPermissionEntry, InstalledApp, PermissionState } from "../shared/app-control.ts"; +import { countUiEvent, getUiTelemetryAttributes, recordUiTiming } from "./telemetry.ts"; + +export type AppToolAction = "list" | "icons" | "launch" | "stop" | "open_url" | "permission" | "push"; +export type AppToolsState = { + platform: "ios" | "android"; + deviceId: string; + disabled: boolean; + loading: boolean; + apps: InstalledApp[]; + icons: Record; + foregroundBundleId: string | null; + runningBundleIds: string[]; + bundleId: string | null; + permissions: AppPermissionEntry[]; + permissionsReady: boolean; + links: string[]; + busy: string; + error: string; + status: string; + recentUrls: string[]; +}; + +const recentKey = "mobile-dev:recent-urls:v1"; +const lastAppKey = "mobile-dev:last-app:v1"; +function lastApps(): Record { + try { + const value: unknown = JSON.parse(globalThis.localStorage?.getItem(lastAppKey) ?? "{}"); + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + } catch { return {}; } +} +function storedUrls(): string[] { + try { + const value: unknown = JSON.parse(globalThis.localStorage?.getItem(recentKey) ?? "[]"); + return Array.isArray(value) ? value.filter((url): url is string => typeof url === "string").slice(0, 5) : []; + } catch { return []; } +} + +export class AppToolsStore { + private state: AppToolsState = { platform: "ios", deviceId: "", disabled: true, loading: false, apps: [], icons: {}, foregroundBundleId: null, runningBundleIds: [], bundleId: null, permissions: [], permissionsReady: false, links: [], busy: "", error: "", status: "", recentUrls: storedUrls() }; + private listeners = new Set<() => void>(); + private revision = 0; + // simctl cannot read privacy decisions, so iOS rows show what this panel last applied. + private applied = new Map(); + // Icons are fetched once per app and device; the server caches extracted icons across panels. + private iconRequests = new Set(); + private labels = new Map(); + call?: (name: string, args: Record) => Promise; + subscribe = (listener: () => void) => { this.listeners.add(listener); return () => { this.listeners.delete(listener); }; }; + getSnapshot = () => this.state; + private update(patch: Partial) { this.state = { ...this.state, ...patch }; for (const listener of this.listeners) listener(); } + + configure(platform: "ios" | "android", deviceId: string, disabled: boolean) { + if (deviceId !== this.state.deviceId || platform !== this.state.platform) { + this.revision++; + this.applied.clear(); + this.iconRequests.clear(); + this.labels.clear(); + this.update({ platform, deviceId, disabled, loading: false, apps: [], icons: {}, foregroundBundleId: null, runningBundleIds: [], bundleId: null, permissions: [], permissionsReady: false, links: [], busy: "", error: "", status: "" }); + } else if (disabled !== this.state.disabled) this.update({ disabled }); + } + + get app(): InstalledApp | undefined { return this.state.apps.find(app => app.bundleId === this.state.bundleId); } + private target() { return { platform: this.state.platform, deviceId: this.state.deviceId }; } + + // Results that finish after a surface or device change are not attributed to the new context. + private async measure(action: AppToolAction, work: () => Promise): Promise { + const startedAt = performance.now(); + const context = getUiTelemetryAttributes(); + try { return await work(); } + catch (error) { if (getUiTelemetryAttributes() === context) countUiEvent("ui.app_tools.failure"); throw error; } + finally { if (getUiTelemetryAttributes() === context) recordUiTiming(`ui.app_tools.${action}`, performance.now() - startedAt); } + } + + // Opening the panel follows the foreground app; otherwise keep the last app, which may have crashed. + async load() { + if (!this.call || this.state.disabled || this.state.loading) return; + const revision = this.revision; + const call = this.call; + this.update({ loading: true, error: "" }); + try { + const data = installedAppsSchema.parse(await this.measure("list", () => call("mobile_list_apps", this.target()))); + if (revision !== this.revision) return; + const installed = (candidate: string | null | undefined) => candidate && data.apps.some(app => app.bundleId === candidate) ? candidate : null; + // Foreground detection can fail, so fall back to the newest running app, then the app last picked on this device. + const bundleId = data.foregroundBundleId ?? installed(data.runningBundleIds[0]) ?? installed(this.state.bundleId) ?? installed(lastApps()[this.state.deviceId]) ?? data.apps[0]?.bundleId ?? null; + this.update({ apps: this.labeled(data.apps), foregroundBundleId: data.foregroundBundleId, runningBundleIds: data.runningBundleIds, ...this.switchTo(bundleId) }); + void this.loadPermissions(); + void this.loadLinks(); + void this.loadIcons(); + } catch (error) { + if (revision === this.revision) this.update({ error: message(error) }); + } finally { + if (revision === this.revision) this.update({ loading: false }); + } + } + + select(bundleId: string) { + if (bundleId === this.state.bundleId) return; + try { globalThis.localStorage?.setItem(lastAppKey, JSON.stringify({ ...lastApps(), [this.state.deviceId]: bundleId })); } catch { /* Storage can be unavailable in sandboxed hosts. */ } + this.update({ ...this.switchTo(bundleId), error: "", status: "" }); + void this.loadPermissions(); + void this.loadLinks(); + } + + // Keep the previous app's rows and suggestions until the new app's arrive, so the popover does not collapse and regrow. + private switchTo(bundleId: string | null): Partial { + if (bundleId === this.state.bundleId) return {}; + return { bundleId, permissions: this.state.permissions.map(entry => ({ ...entry, state: this.applied.get(`${bundleId}:${entry.permission}`) ?? null })) }; + } + + // Picking an app in the panel means "show me this app", so it launches right away. + open(bundleId: string) { + this.select(bundleId); + return this.launch(); + } + + private states(entries: AppPermissionEntry[]): AppPermissionEntry[] { + return entries.map(entry => ({ ...entry, state: entry.state ?? this.applied.get(`${this.state.bundleId}:${entry.permission}`) ?? null })); + } + + // Android lists packages; app labels arrive with the icons. + private labeled(apps: InstalledApp[]) { + return apps.map(app => ({ ...app, name: app.name === app.bundleId ? this.labels.get(app.bundleId) ?? app.name : app.name })); + } + + private async loadIcons() { + const bundleIds = this.state.apps.map(app => app.bundleId).filter(bundleId => !this.iconRequests.has(bundleId)); + if (!this.call || bundleIds.length === 0) return; + for (const bundleId of bundleIds) this.iconRequests.add(bundleId); + const revision = this.revision; + const call = this.call; + try { + const data = appIconsSchema.parse(await this.measure("icons", () => call("mobile_app_icons", { ...this.target(), bundleIds }))); + if (revision !== this.revision) return; + const icons = { ...this.state.icons }; + for (const [bundleId, entry] of Object.entries(data.apps)) { + if (entry.icon) icons[bundleId] = entry.icon; + if (entry.name) this.labels.set(bundleId, entry.name); + } + this.update({ icons, apps: this.labeled(this.state.apps) }); + } catch { + // Initials remain; allow another attempt the next time the panel opens. + if (revision === this.revision) for (const bundleId of bundleIds) this.iconRequests.delete(bundleId); + } + } + + // Suggestions are optional, so a failed read leaves the recent URLs on their own. + private async loadLinks() { + const bundleId = this.state.bundleId; + if (!this.call || !bundleId) return; + const revision = this.revision; + try { + const data = appLinksSchema.parse(await this.call("mobile_app_links", { ...this.target(), bundleId })); + if (revision === this.revision && bundleId === this.state.bundleId) this.update({ links: data.links }); + } catch { + if (revision === this.revision && bundleId === this.state.bundleId) this.update({ links: [] }); + } + } + + private async loadPermissions() { + const bundleId = this.state.bundleId; + if (!this.call || !bundleId) return; + const revision = this.revision; + try { + const data = appPermissionsSchema.parse(await this.call("mobile_app_permissions", { ...this.target(), bundleId })); + if (revision === this.revision && bundleId === this.state.bundleId) this.update({ permissions: this.states(data.permissions), permissionsReady: true }); + } catch (error) { + if (revision === this.revision && bundleId === this.state.bundleId) this.update({ error: message(error) }); + } + } + + private async run(action: AppToolAction, key: string, status: string, work: (call: NonNullable, current: () => boolean) => Promise) { + if (!this.call || this.state.disabled || this.state.busy) return; + const revision = this.revision; + const call = this.call; + this.update({ busy: key, error: "", status: "" }); + try { + await this.measure(action, () => work(call, () => revision === this.revision)); + if (revision === this.revision) this.update({ status }); + } catch (error) { + if (revision === this.revision) this.update({ error: message(error) }); + } finally { + if (revision === this.revision) this.update({ busy: "" }); + } + } + + launch() { + const app = this.app; + if (!app) return Promise.resolve(); + return this.run("launch", "launch", `Relaunched ${app.name}.`, async (call, current) => { + await call("mobile_launch_app", { ...this.target(), bundleId: app.bundleId }); + if (current()) this.update({ foregroundBundleId: app.bundleId, runningBundleIds: [app.bundleId, ...this.state.runningBundleIds.filter(id => id !== app.bundleId)] }); + }); + } + + stop() { + const app = this.app; + if (!app) return Promise.resolve(); + return this.run("stop", "stop", `Stopped ${app.name}.`, async (call, current) => { + await call("mobile_stop_app", { ...this.target(), bundleId: app.bundleId }); + if (current()) this.update({ foregroundBundleId: this.state.foregroundBundleId === app.bundleId ? null : this.state.foregroundBundleId, runningBundleIds: this.state.runningBundleIds.filter(id => id !== app.bundleId) }); + }); + } + + openUrl(input: string) { + const url = input.trim(); + if (!url) return Promise.resolve(); + return this.run("open_url", "open_url", "Opened URL.", async call => { + await call("mobile_open_url", { ...this.target(), url }); + const recentUrls = [url, ...this.state.recentUrls.filter(entry => entry !== url)].slice(0, 5); + try { globalThis.localStorage?.setItem(recentKey, JSON.stringify(recentUrls)); } catch { /* Storage can be unavailable in sandboxed hosts. */ } + this.update({ recentUrls }); + }); + } + + setPermission(permission: AppPermission | "all", state: PermissionState) { + const app = this.app; + if (!app) return Promise.resolve(); + const label = permission === "all" ? "Reset all permissions." : `Updated ${permission} permission.`; + return this.run("permission", `permission:${permission}`, label, async (call, current) => { + const data = appPermissionsSchema.parse(await call("mobile_set_app_permission", { ...this.target(), bundleId: app.bundleId, permission, state })); + const changed = permission === "all" ? platformPermissions[this.state.platform] : [permission]; + for (const entry of changed) this.applied.set(`${app.bundleId}:${entry}`, state); + if (!current() || app.bundleId !== this.state.bundleId) return; + const updated = new Map(this.states(data.permissions).map(entry => [entry.permission, entry])); + this.update({ permissions: this.state.permissions.map(entry => updated.get(entry.permission) ?? (changed.includes(entry.permission) ? { ...entry, state } : entry)) }); + }); + } + + push(payload: Record) { + const app = this.app; + if (!app) return Promise.resolve(); + return this.run("push", "push", `Sent notification to ${app.name}.`, async call => { + await call("mobile_send_push", { ...this.target(), bundleId: app.bundleId, payload }); + }); + } + + dispose() { this.call = undefined; this.configure(this.state.platform, "", true); } +} + +function message(error: unknown) { return error instanceof Error ? error.message : String(error); } + +const stores = new WeakMap(); +export function bindAppTools(element: HTMLElement, store: AppToolsStore) { stores.set(element, store); } +export function getAppTools(element: HTMLElement) { + let store = stores.get(element); + if (!store) { store = new AppToolsStore(); stores.set(element, store); } + return store; +} diff --git a/src/ui/components/app-tools.tsx b/src/ui/components/app-tools.tsx new file mode 100644 index 0000000..236fe9f --- /dev/null +++ b/src/ui/components/app-tools.tsx @@ -0,0 +1,159 @@ +import { useId, useRef, useState, useSyncExternalStore } from "react"; +import { ChevronDownIcon, RotateCwIcon, SquareIcon } from "lucide-react"; +import type { AppPermission, PermissionState } from "../../shared/app-control"; +import type { AppToolsStore } from "../app-tools"; +import { Button } from "./ui/button"; +import { Input } from "./ui/input"; +import { Textarea } from "./ui/textarea"; +import { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger } from "./ui/base-select"; +import { ToggleGroup, ToggleGroupItem } from "./ui/toggle-group"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "./ui/collapsible"; + +const permissionLabels: Record = { + location: "Location", photos: "Photos", camera: "Camera", microphone: "Microphone", notifications: "Notifications", + contacts: "Contacts", calendar: "Calendar", reminders: "Reminders", motion: "Motion & fitness", +}; +const stateLabels: Record = { allow: "Allow", deny: "Deny", ask: "Ask" }; +const primaryPermissions = 4; + +function Section({ title, action, children }: { title: string; action?: React.ReactNode; children: React.ReactNode }) { + return
+

{title}

{action}
+ {children} +
; +} + +// Matches the device picker's running dot so both pickers read the same way. +function AppAvatar({ name, icon, running, small = false }: { name: string; icon?: string; running: boolean; small?: boolean }) { + const shape = small ? "size-6 rounded-md text-xs" : "size-8 rounded-lg text-sm"; + return + {icon ? : name.trim().charAt(0).toUpperCase() || "?"} + {running && } + ; +} + +export function AppTools({ store }: { store: AppToolsStore }) { + const state = useSyncExternalStore(store.subscribe, store.getSnapshot); + const id = useId(); + const [url, setUrl] = useState(""); + const urlInput = useRef(null); + const [morePermissions, setMorePermissions] = useState(false); + const [title, setTitle] = useState(""); + const [body, setBody] = useState(""); + const [payload, setPayload] = useState(); + const [payloadError, setPayloadError] = useState(""); + const app = state.apps.find(entry => entry.bundleId === state.bundleId); + const blocked = state.disabled || !!state.busy; + const isRunning = (bundleId: string) => bundleId === state.foregroundBundleId || state.runningBundleIds.includes(bundleId); + const running = app !== undefined && isRunning(app.bundleId); + + if (state.disabled) return

App tools work on running iOS simulators and Android devices.

; + if (!app) { + return
+

{state.loading ? "Loading apps…" : state.error ? "" : "No user apps are installed on this device."}

+ {state.error &&

{state.error}

} +
; + } + + const suggestions = state.links.filter(link => !state.recentUrls.includes(link)); + const permissions = state.permissions; + const visiblePermissions = morePermissions ? permissions : permissions.slice(0, primaryPermissions); + const formPayload = () => ({ aps: { alert: { title: title.trim() || app.name, ...(body.trim() ? { body: body.trim() } : {}) }, sound: "default" } }); + const sendPush = () => { + let data: Record; + if (payload === undefined) data = formPayload(); + else { + try { + const parsed: unknown = JSON.parse(payload); + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("Payload must be a JSON object."); + data = parsed as Record; + } catch (error) { setPayloadError(error instanceof Error ? error.message : String(error)); return; } + } + setPayloadError(""); + void store.push(data); + }; + + return
+
+ +
+ + +
+
+ +
+
{ event.preventDefault(); void store.openUrl(url); }}> + setUrl(event.target.value)} /> + +
+ {/* One fixed-height row, so switching apps never changes the popover height. */} +
+ {/* Declared schemes are usually incomplete, so they start a URL instead of opening one. */} + {suggestions.map(link => )} + {state.recentUrls.map(recent => )} + {suggestions.length === 0 && state.recentUrls.length === 0 && This app declares no URL schemes.} +
+
+ +
void store.setPermission("all", "ask")}>Reset all}> + {!state.permissionsReady ?

Loading permissions…

+ : permissions.length === 0 ?

This app does not request supported runtime permissions.

+ :
+ {visiblePermissions.map(entry =>
+ {permissionLabels[entry.permission]} + { if (value) void store.setPermission(entry.permission, value as PermissionState); }}> + {(["allow", "deny", "ask"] as const).map(option => {stateLabels[option]})} + +
)} + {permissions.length > primaryPermissions && } + {state.platform === "ios" &&

iOS Simulator does not report current choices; highlights show what you set here.

} +
} +
+ + {state.platform === "ios" &&
+ { setPayloadError(""); setPayload(open ? JSON.stringify(formPayload(), null, 2) : undefined); }}> +
{ event.preventDefault(); sendPush(); }}> + {payload === undefined &&
+ setTitle(event.target.value)} /> + setBody(event.target.value)} /> +
} +