From a1aba54684dbe0d87c06d871cc314a281aa2af9f Mon Sep 17 00:00:00 2001 From: Alban Gicquel Date: Sun, 11 Oct 2026 22:48:45 -0400 Subject: [PATCH] Queue synthetic events, pause and step the clock, and capture screenshots over MCP send_event now queues its event for the next frame that runs, delivered after the host's batch for that turn, so an injected key is pressed on exactly that frame instead of being cleared by beginFrame. Application gains pause, resume, step (granted frames with a fixed delta) and queueEvent. New MCP tools: clock_pause, clock_step, clock_resume, clock_info (with a settled flag for waiting on frames), screenshot and screenshot_info. send_event, the clock tools and the screenshot tools run while a frame is parked, and clock_info reports when one is. --- README.md | 11 ++ docs/gfx/screenshots.md | 5 + src/tecs/application.nupp | 178 ++++++++++++++++++++++- src/tecs/internal/mcpbindings.nupp | 9 ++ src/tecs/internal/mcpregistry.nupp | 14 ++ src/tecs/internal/mcpschemas.nupp | 72 ++++++++++ src/tecs/internal/mcpsubsystems.nupp | 205 +++++++++++++++++++++++++-- src/tecs/internal/mcptools.nupp | 27 ++-- src/tecs/mcp.nupp | 49 ++++++- tests/applicationtest.nupp | 160 +++++++++++++++++++++ tests/mcpsessiontest.nupp | 6 +- tests/mcptest.nupp | 38 +++++ tests/tecs/compatibilitytest.nupp | 12 ++ tests/tecs/mcpsubsystemstest.nupp | 149 +++++++++++++++++-- 14 files changed, 896 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 64a7f212..596c4ff6 100644 --- a/README.md +++ b/README.md @@ -357,6 +357,17 @@ suspends, later events wait for the next update. Watcher changes use the same boundary. The scheduler therefore commits a phase once and extraction never observes half of an external batch. +Synthetic input takes the same boundary. The debug server's `send_event` used +to fold its event into input state the moment the tool ran, which is between +frames: the next frame's `beginFrame` then cleared the edges, so an injected +key was down but never pressed, and a test harness had to queue events inside +the game instead. The application now queues them and delivers them after the +host's batch on the next frame that runs. Pausing is the application's rather +than the host's for the same reason: a held iteration keeps its events for the +frame that runs and a stepped frame takes a fixed delta, so a paused game +advances by exactly the frames granted, on any host, while the host keeps +rendering and serving screenshots of the held world. + ### Application lifecycle and presentation timing The application owns asset orchestration and optional debugging because both diff --git a/docs/gfx/screenshots.md b/docs/gfx/screenshots.md index 2e1a8c99..7b97c7b7 100644 --- a/docs/gfx/screenshots.md +++ b/docs/gfx/screenshots.md @@ -47,3 +47,8 @@ Only requested frames allocate readback storage and wait for GPU completion. Keep captures for as long as needed; releasing your references allows their pixel and PNG data to be collected. Several requests for one frame share its GPU readback. + +The debug server's `screenshot` tool requests the same capture from outside +the game and writes the PNG once the host has rendered the frame; +`screenshot_info` reports when it is written. See [](tecs.mcp) for the tool +and for pausing and stepping the simulation around it. diff --git a/src/tecs/application.nupp b/src/tecs/application.nupp index 82506881..8582834d 100644 --- a/src/tecs/application.nupp +++ b/src/tecs/application.nupp @@ -20,6 +20,29 @@ parks asynchronous startup as well as updates. A custom host must follow the same ordering. Diagnostics remain available during a parked operation; tools that could touch its in-progress world report a retryable refusal. +## Driving frames from outside + +A test harness or debugger drives an application deterministically with three +calls, which the MCP `send_event` and `clock_*` tools reach. `queueEvent` +queues a platform event for the next frame that runs, delivered after the +host's batch for that turn exactly as a platform event is, so per-frame edges +such as `keyPressed` survive the frame's `beginFrame`. `pause` holds the +simulation: an iteration that has no granted frame runs no system, folds no +input, leaves `frame` and `elapsed` unchanged, and keeps the host's events for +the frame that eventually runs, honoring only a quit. The host keeps rendering +the held world, so a screenshot requested while paused shows it. `step` grants +frames that run while paused, each with the fixed delta it names rather than +the host's measured one, so a paused application advances by exactly the +granted frames and simulated time. + +```nupp +app:pause() +app:queueEvent(tecs.platform.events.key(true, "Space", " ", " ", 0, false, 0)) +app:step(1, 1 / 60) +app:iterate(0.5, nil) -- runs one frame with dt 1/60 and sees Space pressed +app:iterate(0.5, nil) -- holds: no frame, no time +``` + Shutdown first unwinds a parked host operation, then attempts world shutdown, stops the loader, and closes the debug listener. Later application cleanup still runs if world shutdown raises. Resident asset handles retain their @@ -72,6 +95,10 @@ export record Application is mcpbindings.Application elapsed: number frame: integer + --- Read-only. Reports whether `pause` holds the simulation. Change it through + --- `pause` and `resume`. + paused: boolean + @private state: State @private plugin: Plugin? @private onEvent: (function(event: platformevents.Event, exclusive app: Application): nil)? @@ -81,6 +108,10 @@ export record Application is mcpbindings.Application @private debugMaxFrames: integer? @private failure: string? @private shutdownDone: boolean + @private steps: integer + @private stepDt: number + @private held: {platformevents.Event} + @private injected: {platformevents.Event} --- Polls optional debugging once per host turn, outside the world update scope. --- @param self the application to serve. @@ -96,6 +127,64 @@ export record Application is mcpbindings.Application clearCrash: function(exclusive self: Application): (boolean, string?) shutdown: function(exclusive self: Application): boolean getState: function(borrows self: Application): State + + --- Queues a platform event for the next frame that runs. + --- + --- The event reaches window state, input state, the world's observers and + --- `Config.onEvent` after that frame's `beginFrame`, behind the host's own + --- batch for the same turn, so per-frame edges such as `keyPressed` read + --- true for exactly that frame. While paused the event waits for the next + --- frame `step` or `resume` lets run. + --- @param self the application to deliver to + --- @param event the event, delivered as if the platform had sent it + queueEvent: function(exclusive self: Application, event: platformevents.Event): nil + + --- Holds the simulation from the next iteration on. + --- + --- A held iteration runs no system and folds no input, and leaves `frame` + --- and `elapsed` unchanged. Host events wait for the next frame that runs, + --- up to the latest 1024, except that a quit or close request still stops + --- the application. The + --- host keeps rendering the held world. Pausing again does nothing, and + --- frames already granted by `step` are kept. + --- @param self the application to hold + pause: function(exclusive self: Application): nil + + --- Lets the simulation run on the host's measured delta again. + --- + --- Discards any frames `step` granted and has not run yet. Resuming a + --- running application does nothing. + --- @param self the application to release + resume: function(exclusive self: Application): nil + + --- Grants frames that run while paused, one per host iteration. + --- + --- Each granted frame passes `dt` to the world in place of the host's + --- delta, so a script of steps reproduces the same simulated time on every + --- run and every host. The delta applies to every granted frame not yet + --- run, including ones an earlier call granted. + --- @param self the paused application + --- @param frames how many more frames to run, at least one + --- @param dt the fixed delta of each granted frame, in seconds + --- @raises string when the application is not paused + --- @raises string when `frames` is not a positive integer or `dt` is not a + --- positive finite number + step: function(exclusive self: Application, frames: integer, dt: number): nil + + --- Returns the frames `step` granted that have not started. + --- @param self the application to inspect + --- @return the count, zero while running or once every granted frame ran + pendingSteps: function(borrows self: Application): integer + + --- Returns the fixed delta granted frames run with. + --- @param self the application to inspect + --- @return the delta in seconds the last `step` named, or 1/60 before one + stepDelta: function(borrows self: Application): number + + --- Returns how many events wait for the next frame that runs. + --- @param self the application to inspect + --- @return the count of queued and held events + pendingEvents: function(borrows self: Application): integer end --- Settings copied by `newApplication`. @@ -112,6 +201,10 @@ export type Config = { debugMaxFrames: integer? } +-- The most events a paused application holds for its next frame. Past it the +-- oldest are dropped, as the host's own input routes drop theirs. +local HELD_LIMIT: integer = 1024 + -- The tool registry names one application per runtime. A second listener must -- not silently redirect the first application's tools into another world. local debugOwner: Application? = nil @@ -179,7 +272,12 @@ export function newApplication(config: Config?): Application debug = config ~= nil and config.debug == true, debugMaxFrames = maxFrames, failure = nil, - shutdownDone = false + shutdownDone = false, + paused = false, + steps = 0, + stepDt = 1 / 60, + held = {}, + injected = {} ) app.bindDebug = function(): nil debugOwner = app @@ -263,8 +361,45 @@ function Application.iterate(exclusive self: Application, dt: number, batch: {pl return false end + -- Arrival order: events held while paused, then this turn's host batch, + -- then what `queueEvent` added since the last iteration. + local incoming = self.held + for _, event in ipairs(batch or {}) do + incoming[#incoming + 1] = event + end + for _, event in ipairs(self.injected) do + incoming[#incoming + 1] = event + end + self.held = {} + self.injected = {} + if self.paused then + if self.steps <= 0 then + -- A held turn: nothing runs and the events wait for a frame that + -- does, but a quit is a request to the process rather than to a + -- frame, so it is still honored. + for index = 1, #incoming do + if incoming[index] is platformevents.QuitEvent then + self.quitRequested = true + return false + end + end + if #incoming > HELD_LIMIT then + -- A long pause under a live window must not grow without bound. + local kept: {platformevents.Event} = {} + for index = #incoming - HELD_LIMIT + 1, #incoming do + kept[#kept + 1] = incoming[index] + end + incoming = kept + end + self.held = incoming + + return true + end + self.steps = self.steps - 1 + dt = self.stepDt + end + self.input:beginFrame() - local incoming = batch or {} local handler = self.onEvent for index = 1, #incoming do local event = incoming[index] @@ -307,6 +442,45 @@ function Application.iterate(exclusive self: Application, dt: number, batch: {pl return not self.quitRequested end +function Application.queueEvent(exclusive self: Application, event: platformevents.Event): nil + self.injected[#self.injected + 1] = event +end + +function Application.pause(exclusive self: Application): nil + self.paused = true +end + +function Application.resume(exclusive self: Application): nil + self.paused = false + self.steps = 0 +end + +function Application.step(exclusive self: Application, frames: integer, dt: number): nil + if not self.paused then + error("application.step: the application is not paused", 2) + end + if frames < 1 or frames % 1 ~= 0 then + error("application.step: frames must be a positive integer", 2) + end + if not (dt > 0) or dt == math.huge then + error("application.step: dt must be a positive finite number", 2) + end + self.steps = self.steps + frames + self.stepDt = dt +end + +function Application.pendingSteps(borrows self: Application): integer + return self.steps +end + +function Application.stepDelta(borrows self: Application): number + return self.stepDt +end + +function Application.pendingEvents(borrows self: Application): integer + return #self.held + #self.injected +end + --- Requests that the host stop after the current callback returns. --- @param self the application to stop function Application.requestQuit(exclusive self: Application): nil diff --git a/src/tecs/internal/mcpbindings.nupp b/src/tecs/internal/mcpbindings.nupp index 798936ba..e10bdc0f 100644 --- a/src/tecs/internal/mcpbindings.nupp +++ b/src/tecs/internal/mcpbindings.nupp @@ -16,6 +16,7 @@ share one binding without either requiring the other. local ecs = require("tecs.ecs") local input = require("tecs.input") +local platformevents = require("tecs.platform.events") local window = require("tecs.platform.window") --- The application surface debug tools use, independent of lifecycle ownership. @@ -29,6 +30,14 @@ export interface Application getState: function(borrows self: Application): string requestQuit: function(exclusive self: Application): nil clearCrash: function(exclusive self: Application): (boolean, string?) + paused: boolean + queueEvent: function(exclusive self: Application, event: platformevents.Event): nil + pause: function(exclusive self: Application): nil + resume: function(exclusive self: Application): nil + step: function(exclusive self: Application, frames: integer, dt: number): nil + pendingSteps: function(borrows self: Application): integer + stepDelta: function(borrows self: Application): number + pendingEvents: function(borrows self: Application): integer end local boundWorld: ecs.World? = nil diff --git a/src/tecs/internal/mcpregistry.nupp b/src/tecs/internal/mcpregistry.nupp index 79f08f6a..b567710f 100644 --- a/src/tecs/internal/mcpregistry.nupp +++ b/src/tecs/internal/mcpregistry.nupp @@ -82,6 +82,18 @@ local revision: integer = 0 --- The traceback of a guarded game failure, once one has been recorded. local crashedText: string? = nil +--- Whether the tool call in progress runs while a lifecycle operation is parked. +local callingParked: boolean = false + +--- Reports whether the tool call in progress runs while a lifecycle +--- operation is parked, which a `whenParked` handler reads to tell an agent +--- that a frame is still mid-flight. +--- @return true inside a handler called during a parked operation, and false +--- otherwise, including outside any handler +export function parkedCall(): boolean + return callingParked +end + --- Records a crash so every world-touching tool reports it instead of running. --- @param traceback the failure text, or nil to clear the crashed state export function setCrashed(traceback: string?): nil @@ -202,7 +214,9 @@ export function callTool(tool: Tool, arguments: {[string]: any}, parked: boolean end -- A tool that raises must not take the server with it: the agent is told, -- and the frame carries on. + callingParked = parked == true local called, payload = pcall(tool.handler, arguments) + callingParked = false if not called then logger:warn("tool %s failed: %s", tool.name, tostring(payload)) diff --git a/src/tecs/internal/mcpschemas.nupp b/src/tecs/internal/mcpschemas.nupp index a4d5de51..7632e0e5 100644 --- a/src/tecs/internal/mcpschemas.nupp +++ b/src/tecs/internal/mcpschemas.nupp @@ -278,3 +278,75 @@ export const inputs: { ["additionalProperties"] = false } } + +-- Schemas for tools added after that listing, so not checked against it. +-- `screenshot` keeps the historical tool's `name` key and adds `path`. Names +-- and argument keys are external compatibility contracts all the same. +export const added: { + [string]: any +} = { + ["clock_info"] = {["properties"] = json.EMPTY_OBJECT, ["type"] = "object", ["additionalProperties"] = false}, + ["clock_pause"] = {["properties"] = json.EMPTY_OBJECT, ["type"] = "object", ["additionalProperties"] = false}, + ["clock_resume"] = {["properties"] = json.EMPTY_OBJECT, ["type"] = "object", ["additionalProperties"] = false}, + [ + "clock_step" + ] = { + [ + "properties" + ] = { + [ + "frames" + ] = { + ["type"] = "number", + ["description"] = "Sets how many more frames run, as a whole number. (default: 1)", + ["minimum"] = 1 + }, + [ + "dt" + ] = { + ["type"] = "number", + ["description"] = "Sets the fixed delta of each granted frame in seconds. (default: 1/60)", + ["minimum"] = 1e-06, + ["maximum"] = 1 + } + }, + ["type"] = "object", + ["additionalProperties"] = false + }, + [ + "screenshot" + ] = { + [ + "properties" + ] = { + [ + "name" + ] = { + ["type"] = "string", + ["description"] = "File name under the writable root. Defaults to screenshot.png." + }, + [ + "path" + ] = { + ["type"] = "string", + ["description"] = "Writes the PNG at this absolute path instead of under the writable root." + } + }, + ["type"] = "object", + ["additionalProperties"] = false + }, + [ + "screenshot_info" + ] = { + [ + "properties" + ] = { + [ + "id" + ] = {["type"] = "number", ["description"] = "Selects the capture `screenshot` reported.", ["minimum"] = 1} + }, + ["required"] = {"id"}, + ["type"] = "object", + ["additionalProperties"] = false + } +} diff --git a/src/tecs/internal/mcpsubsystems.nupp b/src/tecs/internal/mcpsubsystems.nupp index 382d622c..05dfb8c1 100644 --- a/src/tecs/internal/mcpsubsystems.nupp +++ b/src/tecs/internal/mcpsubsystems.nupp @@ -14,6 +14,7 @@ local materials = require("tecs.gpu.materials") local layers = require("tecs.gfx.layers") local physics = require("tecs.physics") local audio = require("tecs.audio") +local screenshot = require("tecs.gfx.screenshot") local files = require("nupp.io.files") local path = require("nupp.io.path") local json = require("nupp.codec.json") @@ -27,7 +28,7 @@ local function register( mutates: boolean?, independent: boolean? ): nil - local schema = schemas.inputs[name] + local schema = schemas.inputs[name] ?? schemas.added[name] registry.register({ name = name, description = description, @@ -241,13 +242,8 @@ register( layers = layers.MAX, camera = cameraInfo(), frame = app ~= nil and app.frame or 0, - observations = {gpuCounters = false, readback = false}, - unavailable = json.asArray({ - "presented frame ID", - "GPU timings", - "visible instance count", - "pixel readback" - }) + observations = {gpuCounters = false, readback = true}, + unavailable = json.asArray({"presented frame ID", "GPU timings", "visible instance count"}) } end ) @@ -432,6 +428,195 @@ register( true ) +------------------------------------------------------------------------------- +-- The simulation clock +-- +-- `clock_pause`, `clock_step`, `clock_resume` and `clock_info` reach the +-- application's pause, step and queue state only, never the world, so they run +-- while a frame is parked and after a crash. A harness polls `clock_info` until +-- `settled` rather than retrying a refused world tool. +------------------------------------------------------------------------------- + +local function clockInfo(): {[string]: any} + local app = bindings.requireApplication() + local parked = registry.parkedCall() + local steps = app:pendingSteps() + + return { + paused = app.paused, + frame = app.frame, + elapsed = app.elapsed, + pendingSteps = steps, + pendingEvents = app:pendingEvents(), + stepDt = app:stepDelta(), + parked = parked, + settled = app.paused and steps == 0 and not parked + } +end + +register( + "clock_info", + "Report whether the simulation is paused, the frame count, granted frames still to run, and whether a frame is " + .. "parked mid-flight. `settled` is true once a paused clock has run every granted frame.", + function(_args: {[string]: any}): {[string]: any} + return clockInfo() + end, + false, + true +) +register( + "clock_pause", + "Hold the simulation: host turns run no system and fold no input until clock_step grants frames or clock_resume " + .. "releases it. Rendering continues.", + function(_args: {[string]: any}): {[string]: any} + local app = bindings.requireApplication() + app:pause() + return clockInfo() + end, + true, + true +) +register( + "clock_step", + "Grant frames to a paused simulation, one per host turn, each with a fixed delta instead of the wall clock.", + function(args: {[string]: any}): {[string]: any} + local app = bindings.requireApplication() + if not app.paused then + error("not_paused: the clock is running; call clock_pause before clock_step", 0) + end + app:step(integer(args.frames or 1), args.dt or 1 / 60) + + return clockInfo() + end, + true, + true +) +register( + "clock_resume", + "Let the simulation run on the host's measured delta again, discarding granted frames not yet run.", + function(_args: {[string]: any}): {[string]: any} + local app = bindings.requireApplication() + app:resume() + return clockInfo() + end, + true, + true +) + +------------------------------------------------------------------------------- +-- Screenshots +-- +-- The host reads back a frame only after it renders one, so a tool call can +-- only request a capture. The server writes each finished capture from its +-- next poll, and `screenshot_info` reports it. +------------------------------------------------------------------------------- + +local type Shot = { + id: integer, + path: string, + capture: screenshot.Capture?, + ready: boolean, + saved: boolean, + reason: string?, + width: integer, + height: integer, + bytes: integer +} + +local shots: {[integer]: Shot} = {} +local waiting: {Shot} = {} +local shotSerial: integer = 0 + +--- Writes every requested screenshot whose frame the host has read back. +--- +--- `tecs.mcp.poll` calls this once per poll, so a capture lands on disk on the +--- turn after the host renders it whether or not anyone asks. +export function settleCaptures(): nil + if #waiting == 0 then + return + end + local still: {Shot} = {} + for _, shot in ipairs(waiting) do + local capture = shot.capture + if capture ~= nil and capture.ready then + shot.capture = nil + local _, width, height = screenshot.pixels(capture) + local png, failure = screenshot.encode(capture) + if png == nil then + shot.reason = failure + else + local saved, reason = screenshot.save(capture, shot.path) + shot.saved = saved + shot.reason = reason + shot.width, shot.height, shot.bytes = width, height, #png + end + shot.ready = true + else + still[#still + 1] = shot + end + end + waiting = still +end + +local function shotInfo(borrows shot: Shot): {[string]: any} + return { + id = shot.id, + path = shot.path, + ready = shot.ready, + saved = shot.saved, + reason = shot.reason ?? json.NULL, + width = shot.width, + height = shot.height, + bytes = shot.bytes + } +end + +register( + "screenshot", + "Capture the next rendered frame to a PNG under the writable root or at an absolute path. Returns at once with " + .. "the capture's id and path; screenshot_info reports when the file is written.", + function(args: {[string]: any}): {[string]: any} + local destination: string + if args.path ~= nil then + assert(path.newPath(args.path):isAbsolute(), "screenshot path must be absolute") + destination = args.path + else + destination = artifact(args.name or "screenshot.png") + end + shotSerial = shotSerial + 1 + const shot: Shot = { + id = shotSerial, + path = destination, + capture = screenshot.request(), + ready = false, + saved = false, + reason = nil, + width = 0, + height = 0, + bytes = 0 + } + shots[shot.id] = shot + waiting[#waiting + 1] = shot + + return shotInfo(shot) + end, + true, + true +) +register( + "screenshot_info", + "Report whether a requested screenshot has been written, with its size or why it failed.", + function(args: {[string]: any}): {[string]: any} + settleCaptures() + local shot = shots[integer(args.id)] + assert(shot ~= nil, "unknown screenshot: " .. tostring(args.id)) + + return shotInfo(shot) + end, + false, + true +) + local families: { [string]: boolean } = { @@ -445,7 +630,9 @@ local families: { snapshot = true, profile = true, systems = true, - states = true + states = true, + clock = true, + screenshot = true } local function groupOf(name: string): string diff --git a/src/tecs/internal/mcptools.nupp b/src/tecs/internal/mcptools.nupp index f6bac279..f900a82a 100644 --- a/src/tecs/internal/mcptools.nupp +++ b/src/tecs/internal/mcptools.nupp @@ -24,7 +24,6 @@ local ecs = require("tecs.ecs") local files = require("nupp.io.files") local json = require("nupp.codec.json") local platformevents = require("tecs.platform.events") -local platformwindow = require("tecs.platform.window") local time = require("nupp.time") local bindings = require("tecs.internal.mcpbindings") local registry = require("tecs.internal.mcpregistry") @@ -325,10 +324,19 @@ registry.register({ -- reading a recording or the event reference already knows; the Teal tool -- itself exposed none of them, so it could list `gamepadAxis` among its -- kinds and then not let anyone push one. + -- + -- The event is queued for the next frame that runs rather than folded into + -- the input on the spot. A tool call lands between frames, before the next + -- frame's `beginFrame` clears the per-frame edges, so a key folded + -- immediately was down but never pressed. Queuing only touches the + -- application's queue, which is why the tool may also run while a frame is + -- parked. name = "send_event", - description = "Pushes an event into the input stream, as if it had come from the platform.", + description = "Queues an event for the next frame that runs, delivered after " + .. "the host's own input for that frame as if it had come from the platform.", destructive = true, whenCrashed = true, + whenParked = true, inputSchema = { type = "object", properties = { @@ -382,17 +390,12 @@ registry.register({ local event = buildEvent(kind, arguments) local app = bindings.requireApplication() - -- The same three folds one host iteration performs on a sealed batch, - -- so a synthetic event reaches window state, input state and the - -- world's observers exactly as a real one does. - platformwindow.handleEvent(app.window, event) - app.input:handleEvent(event) - platformevents.dispatch(app.world, event) - if event is platformevents.Quit or event is platformevents.WindowCloseRequested then - app:requestQuit() - end + -- The next frame folds it with the host's own batch, so it reaches + -- window state, input state, the world's observers and a quit request + -- exactly as a real event does. + app:queueEvent(event) - return {pushed = kind} + return {pushed = kind, frame = app.frame, pending = app:pendingEvents()} end }) diff --git a/src/tecs/mcp.nupp b/src/tecs/mcp.nupp index 28f102ec..630a7b2f 100644 --- a/src/tecs/mcp.nupp +++ b/src/tecs/mcp.nupp @@ -12,10 +12,12 @@ including after a crash or while gameplay is suspended. For a standalone server, call `poll` outside the world update scope and `destroy` at teardown. Each poll answers at most one tool request; handlers must answer promptly. -While a lifecycle operation is parked, `ping`, `context` and `get_logs` remain -available. Other tools report that the operation must complete first. A custom -tool may opt in with `whenParked = true` only if it never touches the in-progress -world. `readOnly` alone grants no such permission. +While a lifecycle operation is parked, `ping`, `context`, `get_logs`, +`send_event`, the `clock_*` tools and the `screenshot` tools remain available. +Other tools report that the operation must complete first; `clock_info` +reports `parked` so an agent can wait for the frame instead of retrying them. A +custom tool may opt in with `whenParked = true` only if it never touches the +in-progress world. `readOnly` alone grants no such permission. ```nupp local server = assert(tecs.mcp.listen(7100)) @@ -57,11 +59,43 @@ component update and `set` for a complete value or a missing component. structured tools to `run_lua`; the Lua sandbox limits accidents but does not create a security boundary. +## Driving frames deterministically + +`send_event` queues its event for the next frame that runs, behind the host's +own input for that frame, so a key it sends is pressed on exactly that frame. +`clock_pause` holds the simulation: host turns run no system and fold no input, +and `frame` and `elapsed` stand still, while the host keeps rendering the held +world. `clock_step` grants frames to a paused clock, one per host turn, each +with a fixed `dt` (1/60 s by default) instead of the measured one, and +`clock_resume` returns to the host's delta. `clock_info` reports `settled` once +a paused clock has run every granted frame and no frame is parked mid-flight, +which is when the world is safe to read. + +`screenshot` requests a capture of the next rendered frame and answers at once +with an `id` and the `path` it will write, under the writable root or at an +absolute `path`. The server writes the PNG on its first poll after the host +reads the frame back, in a windowed or an `--offscreen` host, and +`screenshot_info` reports `ready`, `saved`, the size, or the failure. A +`--headless` host renders nothing, so its captures never become ready. + +A harness turn, as JSON-RPC `tools/call` requests: + +```text +clock_pause {} +send_event {"kind": "keyDown", "physicalKey": "Space"} +clock_step {"frames": 1} +clock_info {} -- repeat until "settled": true +send_event {"kind": "keyUp", "physicalKey": "Space"} +clock_step {"frames": 1} +screenshot {"path": "/tmp/frame.png"} +screenshot_info {"id": 1} -- repeat until "ready": true +``` + ## After a crash `setCrashed` records a guarded gameplay failure. The server keeps polling after -one: `ping`, `context`, `get_logs`, `send_event` and `clear_crash` remain -available, and every tool that touches the world reports the stored traceback +one: `ping`, `context`, `get_logs`, `send_event`, `clear_crash`, the +`clock_*` tools and the `screenshot` tools remain available, and every tool that touches the world reports the stored traceback instead of running. ## Subsystem tools @@ -74,7 +108,7 @@ sampler; stop writes collapsed stacks under the writable root. `help`, `describe` and `capabilities` enumerate the live registry. They and the sampler remain available while parked or crashed. The server does not provide -GPU readback, shader or image reload, watcher orchestration, or physics debug +pixel sampling, shader or image reload, watcher orchestration, or physics debug overlay tools. Snapshot files use TECS framing over Nupp's binary codec and refuse unsupported framing or physics encodings. ]] @@ -247,6 +281,7 @@ export function poll(exclusive self: Server, parked: boolean?): boolean self.endpoint:setTools(registry.describeJSON()) end self.endpoint:pump() + subsystems.settleCaptures() local request = self.endpoint:nextRequest() if request == nil then diff --git a/tests/applicationtest.nupp b/tests/applicationtest.nupp index 3c16467c..acebe261 100644 --- a/tests/applicationtest.nupp +++ b/tests/applicationtest.nupp @@ -208,4 +208,164 @@ function suite.rejectsInvalidHostLifecycleCalls(): nil ) end +--- Builds a running application whose one Update system records each frame's +--- delta and whether Space was pressed on it. +local function recorder(): (application.Application, {number}, {boolean}) + local deltas: {number} = {} + local pressed: {boolean} = {} + local app = application.newApplication({ + plugin = function(exclusive running: application.Application): nil + const input = running.input + running.world:addSystem({ + name = "test.application.recorder", + phase = ecs.phases.Update, + run = function(dt: number): nil + deltas[#deltas + 1] = dt + pressed[#pressed + 1] = input:keyPressed("Space") + end, + }) + end, + }) + app:init() + + return app, deltas, pressed +end + +local function space(down: boolean): platformevents.Event + return platformevents.key(down, "Space", " ", nil, 0, false, 0) +end + +function suite.deliversAQueuedEventAsAnEdgeOfTheNextFrame(): nil + local app, _, pressed = recorder() + app:queueEvent(space(true)) + test.equal(app:pendingEvents(), 1) + test.assert(not app.input:keyDown("Space")) + + app:iterate(1 / 60, nil) + test.assert(pressed[1]) + test.equal(app:pendingEvents(), 0) + + -- The edge lasts one frame, and the key stays down until it comes up. + app:iterate(1 / 60, nil) + test.assert(not pressed[2]) + test.assert(app.input:keyDown("Space")) + app:queueEvent(space(false)) + app:iterate(1 / 60, nil) + test.assert(app.input:keyReleased("Space")) + app:shutdown() +end + +function suite.deliversQueuedEventsAfterTheHostBatch(): nil + local order: {string} = {} + local app = application.newApplication({ + onEvent = function(event: platformevents.Event, exclusive _running: application.Application): nil + order[#order + 1] = event.kind + end, + }) + app:init() + app:queueEvent(platformevents.text("queued", 0)) + app:iterate(1 / 60, {platformevents.focused(true, 0)}) + test.equal(#order, 2) + test.equal(order[1], "windowFocusGained") + test.equal(order[2], "textInput") + app:shutdown() +end + +function suite.holdsAPausedSimulationAndRunsGrantedFramesOnAFixedDelta(): nil + local app, deltas, pressed = recorder() + app:iterate(0.5, nil) + test.equal(#deltas, 1) + + app:pause() + test.assert(app.paused) + local frame, elapsed = app.frame, app.elapsed + app:queueEvent(space(true)) + -- Held turns run nothing, advance nothing, and keep the events. + test.assert(app:iterate(0.5, {platformevents.focused(true, 0)})) + test.assert(app:iterate(0.5, nil)) + test.equal(#deltas, 1) + test.equal(app.frame, frame) + test.equal(app.elapsed, elapsed) + test.equal(app:pendingEvents(), 2) + test.assert(not app.window:hasFocus()) + + app:step(2, 0.25) + test.equal(app:pendingSteps(), 2) + test.equal(app:stepDelta(), 0.25) + app:iterate(0.5, nil) + test.equal(#deltas, 2) + test.equal(deltas[2], 0.25) + test.assert(pressed[2]) + test.assert(app.window:hasFocus()) + app:iterate(9, nil) + test.equal(deltas[3], 0.25) + test.assert(not pressed[3]) + test.equal(app.frame, frame + 2) + test.equal(app.elapsed, elapsed + 0.5) + test.equal(app:pendingSteps(), 0) + + -- Granted frames used up, the clock holds again. + app:iterate(0.5, nil) + test.equal(#deltas, 3) + + -- Resuming discards a grant and returns to the host's delta. + app:step(5, 0.1) + app:resume() + test.assert(not app.paused) + test.equal(app:pendingSteps(), 0) + app:iterate(0.5, nil) + test.equal(deltas[4], 0.5) + app:shutdown() +end + +function suite.honorsAQuitWhilePaused(): nil + local app = recorder() + app:pause() + test.assert(not app:iterate(1 / 60, {platformevents.quit(0)})) + test.assert(app.quitRequested) + app:shutdown() +end + +function suite.refusesToStepARunningClockOrAnInvalidGrant(): nil + local app = recorder() + test.raises( + function(): nil + app:step(1, 1 / 60) + end, + "not paused" + ) + app:pause() + test.raises( + function(): nil + app:step(0, 1 / 60) + end, + "positive integer" + ) + test.raises( + function(): nil + app:step(1, 0) + end, + "positive finite" + ) + test.raises( + function(): nil + app:step(1, math.huge) + end, + "positive finite" + ) + app:shutdown() +end + +function suite.boundsTheEventsAPausedApplicationHolds(): nil + local app = recorder() + app:pause() + local batch: {platformevents.Event} = {} + for index = 1, 1500 do + batch[index] = platformevents.text(tostring(index), 0) + end + app:iterate(1 / 60, batch) + test.equal(app:pendingEvents(), 1024) + app:shutdown() +end + return suite diff --git a/tests/mcpsessiontest.nupp b/tests/mcpsessiontest.nupp index 5dc98d76..405b0678 100644 --- a/tests/mcpsessiontest.nupp +++ b/tests/mcpsessiontest.nupp @@ -180,7 +180,7 @@ function suite.inspectsAndControlsARunningApplicationOverTheTransport(): nil callOverHTTP(port, "states_pop") test.equal(callOverHTTP(port, "states_info").depth as integer, 0) - -- Control: a synthetic event reaches the same folds a real one does. + -- Control: a synthetic event is queued for the folds a real one takes. local sent = callOverHTTP(port, "send_event", {kind = "keyDown", physicalKey = "KeyF"}) test.equal(sent.pushed as string, "keyDown") @@ -190,6 +190,10 @@ function suite.inspectsAndControlsARunningApplicationOverTheTransport(): nil test.equal(callOverHTTP(port, "stats").entities as integer, 3) end) + -- The synthetic key arrives with the next frame, as a platform key does. + test.assert(not app.input:keyDown("KeyF")) + app:iterate(1 / 60, nil) + test.assert(app.input:keyPressed("KeyF")) test.assert(app.input:keyDown("KeyF")) -- The stopped system does not run, so the counter it owns stands still. diff --git a/tests/mcptest.nupp b/tests/mcptest.nupp index a5a64ab9..49c01e07 100644 --- a/tests/mcptest.nupp +++ b/tests/mcptest.nupp @@ -297,6 +297,9 @@ function suite.pushesTouchAndGamepadEventsThroughSendEvent(): nil }), "fingerDown" ) + -- Queued for the next frame rather than folded on the spot. + test.equal(#app.input:touches(), 0) + app:iterate(1 / 60, nil) local held = app.input:touches() test.equal(#held, 1) test.equal(held[1].finger, "1") @@ -307,6 +310,7 @@ function suite.pushesTouchAndGamepadEventsThroughSendEvent(): nil test.equal(pushed({kind = "gamepadAdded", which = 4}), "gamepadAdded") test.equal(pushed({kind = "gamepadButtonDown", which = 4, button = 1}), "gamepadButtonDown") test.equal(pushed({kind = "gamepadAxis", which = 4, axis = 1, value = 0.8}), "gamepadAxis") + app:iterate(1 / 60, nil) local pad = assert(app.input:gamepadById(4)) test.assert(pad:buttonDown("south")) @@ -318,4 +322,38 @@ function suite.pushesTouchAndGamepadEventsThroughSendEvent(): nil app:shutdown() end +function suite.sendEventKeyIsPressedOnTheNextFrame(): nil + local pressedOn: {integer} = {} + local frames: integer = 0 + local app: application.Application = application.newApplication({ + plugin = function(exclusive running: application.Application): nil + const input = running.input + running.world:addSystem({ + name = "test.mcp.pressed", + phase = ecs.phases.Update, + run = function(_dt: number): nil + frames = frames + 1 + if input:keyPressed("KeyQ") then + pressedOn[#pressedOn + 1] = frames + end + end + }) + end + }) + app:init() + mcp.bind(app.world, app) + app:iterate(1 / 60, nil) + + local sent = call("send_event", {kind = "keyDown", physicalKey = "KeyQ"}).structuredContent as {[string]: any} + test.equal(sent.pending as integer, 1) + app:iterate(1 / 60, nil) + app:iterate(1 / 60, nil) + test.equal(#pressedOn, 1) + test.equal(pressedOn[1], 2) + test.assert(app.input:keyDown("KeyQ")) + + mcp.bind(nil, nil) + app:shutdown() +end + return suite diff --git a/tests/tecs/compatibilitytest.nupp b/tests/tecs/compatibilitytest.nupp index cdee9e05..30de1770 100644 --- a/tests/tecs/compatibilitytest.nupp +++ b/tests/tecs/compatibilitytest.nupp @@ -687,6 +687,10 @@ function suite.mcpToolNames(): nil "camera_move", "capabilities", "clear_crash", + "clock_info", + "clock_pause", + "clock_resume", + "clock_step", "components_info", "context", "describe", @@ -711,6 +715,8 @@ function suite.mcpToolNames(): nil "resources_info", "resources_list", "run_lua", + "screenshot", + "screenshot_info", "send_event", "set", "snapshot_load", @@ -769,6 +775,12 @@ function suite.mcpArgumentKeys(): nil equalList(argumentKeys("systems_start"), {"name"}) equalList(argumentKeys("systems_stop"), {"name"}) equalList(argumentKeys("states_push"), {"name"}) + equalList(argumentKeys("clock_info"), {}) + equalList(argumentKeys("clock_pause"), {}) + equalList(argumentKeys("clock_resume"), {}) + equalList(argumentKeys("clock_step"), {"dt", "frames"}) + equalList(argumentKeys("screenshot"), {"name", "path"}) + equalList(argumentKeys("screenshot_info"), {"id"}) end function suite.sendEventArgumentKeys(): nil diff --git a/tests/tecs/mcpsubsystemstest.nupp b/tests/tecs/mcpsubsystemstest.nupp index 06f55b5a..a3b61420 100644 --- a/tests/tecs/mcpsubsystemstest.nupp +++ b/tests/tecs/mcpsubsystemstest.nupp @@ -7,6 +7,8 @@ local physics = require("tecs.physics") local audio = require("tecs.audio") local audiobackend = require("tecs.platform.audiobackend") local mcp = require("tecs.mcp") +local application = require("tecs.application") +local screenshot = require("tecs.gfx.screenshot") local registry = require("tecs.internal.mcpregistry") local schemas = require("tecs.internal.mcpschemas") local entityworld = require("tecs.internal.world") @@ -71,14 +73,7 @@ function suite.restoredInputSchemasMatchTheHistoricalWireListing(): nil end end test.equal(checked, 22) - for _, name in ipairs({ - "screenshot", - "sample_pixels", - "reload_shaders", - "reload_image", - "watch", - "physics_debug" - }) do + for _, name in ipairs({"sample_pixels", "reload_shaders", "reload_image", "watch", "physics_debug"}) do test.equal(registry.findTool(name), nil) end end @@ -221,4 +216,142 @@ function suite.profileToolsWriteAReportAndCanStartAgain(): nil test.equal(call("profile_stop").isError, true) end +--- Builds a bound application whose Update system counts frames and records +--- whether Space was pressed on each. +local function steppedApplication(): (application.Application, {number}, {boolean}) + local deltas: {number} = {} + local pressed: {boolean} = {} + local app = application.newApplication({ + plugin = function(exclusive running: application.Application): nil + const input = running.input + running.world:addSystem({ + name = "test.mcp.clock", + phase = ecs.phases.Update, + run = function(dt: number): nil + deltas[#deltas + 1] = dt + pressed[#pressed + 1] = input:keyPressed("Space") + end + }) + end + }) + app:init() + mcp.bind(app.world, app) + + return app, deltas, pressed +end + +function suite.clockToolsPauseStepAndResumeTheSimulation(): nil + local app, deltas, pressed = steppedApplication() + app:iterate(0.5, nil) + + local info = content("clock_info") + test.equal(info.paused, false) + test.equal(info.settled, false) + test.equal(info.frame, 1) + test.equal(call("clock_step").isError, true) + + test.equal(content("clock_pause").paused, true) + content("send_event", {kind = "keyDown", physicalKey = "Space", logicalKey = " "}) + app:iterate(0.5, nil) + app:iterate(0.5, nil) + test.equal(#deltas, 1) + info = content("clock_info") + test.equal(info.settled, true) + test.equal(info.pendingEvents, 1) + + local stepped = content("clock_step", {frames = 2, dt = 0.125}) + test.equal(stepped.pendingSteps, 2) + test.equal(stepped.stepDt, 0.125) + test.equal(stepped.settled, false) + app:iterate(0.5, nil) + app:iterate(0.5, nil) + app:iterate(0.5, nil) + test.equal(#deltas, 3) + test.equal(deltas[2], 0.125) + test.equal(deltas[3], 0.125) + test.assert(pressed[2]) + test.assert(not pressed[3]) + info = content("clock_info") + test.equal(info.frame, 3) + test.equal(info.elapsed, 0.75) + test.equal(info.settled, true) + + test.equal(call("clock_step", {frames = 1.5}).isError, true) + test.equal(call("clock_step", {dt = 0}).isError, true) + test.equal(call("clock_step", {frames = 0}).isError, true) + + content("clock_step", {frames = 4}) + test.equal(content("clock_resume").paused, false) + test.equal(content("clock_info").pendingSteps, 0) + app:iterate(0.5, nil) + test.equal(deltas[4], 0.5) + + mcp.bind(nil, nil) + app:shutdown() +end + +function suite.clockAndInputToolsAnswerWhileAFrameIsParked(): nil + local app = steppedApplication() + content("clock_pause") + for _, name in ipairs({"clock_info", "clock_step", "send_event", "screenshot", "screenshot_info"}) do + test.equal(assert(registry.findTool(name)).whenParked, true) + end + local parked = registry.callTool(assert(registry.findTool("clock_info")), {}, true) + test.equal(parked.structuredContent.parked, true) + test.equal(parked.structuredContent.settled, false) + local sent = registry.callTool( + assert(registry.findTool("send_event")), + {kind = "keyDown", physicalKey = "KeyA"}, + true + ) + test.equal(sent.isError, nil) + test.equal(app:pendingEvents(), 1) + -- A tool that touches the world still waits for the frame. + test.equal(registry.callTool(assert(registry.findTool("stats")), {}, true).isError, true) + + mcp.bind(nil, nil) + app:shutdown() +end + +function suite.screenshotToolWritesTheNextRenderedFrame(): nil + local app = steppedApplication() + screenshot.cancelPending() + local requested = content("screenshot", {name = "mcp-screenshot-test.png"}) + test.equal(requested.ready, false) + test.assert(requested.path:find("mcp-screenshot-test.png", 1, true) ~= nil) + test.equal(content("screenshot_info", {id = requested.id}).ready, false) + + -- What the host does after it renders: it consumes the request and + -- answers with the frame. + local id = assert(screenshot.nextRequest()) + screenshot.complete(id, 1, 1, "\1\2\3\255", "PNG-BYTES", nil) + local server = assert(mcp.listen(0)) + mcp.poll(server) + test.equal(assert(files.read(requested.path)), "PNG-BYTES") + local written = content("screenshot_info", {id = requested.id}) + test.equal(written.ready, true) + test.equal(written.saved, true) + test.equal(written.width, 1) + test.equal(written.height, 1) + test.equal(written.bytes, 9) + assert(files.remove(requested.path)) + + -- A failed readback is reported rather than written. + local failing = content("screenshot", {name = "mcp-screenshot-failed.png"}) + screenshot.complete(assert(screenshot.nextRequest()), 0, 0, "", "", "GPU readback failed") + local failed = content("screenshot_info", {id = failing.id}) + test.equal(failed.ready, true) + test.equal(failed.saved, false) + test.equal(failed.reason, "GPU readback failed") + + test.equal(call("screenshot", {path = "relative.png"}).isError, true) + test.equal(call("screenshot_info", {id = 999999}).isError, true) + test.equal(content("describe", {command = "screenshot"}).action.name, "screenshot") + test.equal(#content("describe", {command = "clock"}).verbs, 4) + + mcp.destroy(server) + mcp.bind(nil, nil) + app:shutdown() +end + return suite