diff --git a/README.md b/README.md index 64a7f212..596c4ff6 100644 --- a/README.md +++ b/README.md @@ -357,6 +357,17 @@ suspends, later events wait for the next update. Watcher changes use the same boundary. The scheduler therefore commits a phase once and extraction never observes half of an external batch. +Synthetic input takes the same boundary. The debug server's `send_event` used +to fold its event into input state the moment the tool ran, which is between +frames: the next frame's `beginFrame` then cleared the edges, so an injected +key was down but never pressed, and a test harness had to queue events inside +the game instead. The application now queues them and delivers them after the +host's batch on the next frame that runs. Pausing is the application's rather +than the host's for the same reason: a held iteration keeps its events for the +frame that runs and a stepped frame takes a fixed delta, so a paused game +advances by exactly the frames granted, on any host, while the host keeps +rendering and serving screenshots of the held world. + ### Application lifecycle and presentation timing The application owns asset orchestration and optional debugging because both diff --git a/docs/gfx/screenshots.md b/docs/gfx/screenshots.md index 2e1a8c99..7b97c7b7 100644 --- a/docs/gfx/screenshots.md +++ b/docs/gfx/screenshots.md @@ -47,3 +47,8 @@ Only requested frames allocate readback storage and wait for GPU completion. Keep captures for as long as needed; releasing your references allows their pixel and PNG data to be collected. Several requests for one frame share its GPU readback. + +The debug server's `screenshot` tool requests the same capture from outside +the game and writes the PNG once the host has rendered the frame; +`screenshot_info` reports when it is written. See [](tecs.mcp) for the tool +and for pausing and stepping the simulation around it. diff --git a/src/tecs/application.nupp b/src/tecs/application.nupp index 82506881..8582834d 100644 --- a/src/tecs/application.nupp +++ b/src/tecs/application.nupp @@ -20,6 +20,29 @@ parks asynchronous startup as well as updates. A custom host must follow the same ordering. Diagnostics remain available during a parked operation; tools that could touch its in-progress world report a retryable refusal. +## Driving frames from outside + +A test harness or debugger drives an application deterministically with three +calls, which the MCP `send_event` and `clock_*` tools reach. `queueEvent` +queues a platform event for the next frame that runs, delivered after the +host's batch for that turn exactly as a platform event is, so per-frame edges +such as `keyPressed` survive the frame's `beginFrame`. `pause` holds the +simulation: an iteration that has no granted frame runs no system, folds no +input, leaves `frame` and `elapsed` unchanged, and keeps the host's events for +the frame that eventually runs, honoring only a quit. The host keeps rendering +the held world, so a screenshot requested while paused shows it. `step` grants +frames that run while paused, each with the fixed delta it names rather than +the host's measured one, so a paused application advances by exactly the +granted frames and simulated time. + +```nupp +app:pause() +app:queueEvent(tecs.platform.events.key(true, "Space", " ", " ", 0, false, 0)) +app:step(1, 1 / 60) +app:iterate(0.5, nil) -- runs one frame with dt 1/60 and sees Space pressed +app:iterate(0.5, nil) -- holds: no frame, no time +``` + Shutdown first unwinds a parked host operation, then attempts world shutdown, stops the loader, and closes the debug listener. Later application cleanup still runs if world shutdown raises. Resident asset handles retain their @@ -72,6 +95,10 @@ export record Application is mcpbindings.Application elapsed: number frame: integer + --- Read-only. Reports whether `pause` holds the simulation. Change it through + --- `pause` and `resume`. + paused: boolean + @private state: State @private plugin: Plugin? @private onEvent: (function(event: platformevents.Event, exclusive app: Application): nil)? @@ -81,6 +108,10 @@ export record Application is mcpbindings.Application @private debugMaxFrames: integer? @private failure: string? @private shutdownDone: boolean + @private steps: integer + @private stepDt: number + @private held: {platformevents.Event} + @private injected: {platformevents.Event} --- Polls optional debugging once per host turn, outside the world update scope. --- @param self the application to serve. @@ -96,6 +127,64 @@ export record Application is mcpbindings.Application clearCrash: function(exclusive self: Application): (boolean, string?) shutdown: function(exclusive self: Application): boolean getState: function(borrows self: Application): State + + --- Queues a platform event for the next frame that runs. + --- + --- The event reaches window state, input state, the world's observers and + --- `Config.onEvent` after that frame's `beginFrame`, behind the host's own + --- batch for the same turn, so per-frame edges such as `keyPressed` read + --- true for exactly that frame. While paused the event waits for the next + --- frame `step` or `resume` lets run. + --- @param self the application to deliver to + --- @param event the event, delivered as if the platform had sent it + queueEvent: function(exclusive self: Application, event: platformevents.Event): nil + + --- Holds the simulation from the next iteration on. + --- + --- A held iteration runs no system and folds no input, and leaves `frame` + --- and `elapsed` unchanged. Host events wait for the next frame that runs, + --- up to the latest 1024, except that a quit or close request still stops + --- the application. The + --- host keeps rendering the held world. Pausing again does nothing, and + --- frames already granted by `step` are kept. + --- @param self the application to hold + pause: function(exclusive self: Application): nil + + --- Lets the simulation run on the host's measured delta again. + --- + --- Discards any frames `step` granted and has not run yet. Resuming a + --- running application does nothing. + --- @param self the application to release + resume: function(exclusive self: Application): nil + + --- Grants frames that run while paused, one per host iteration. + --- + --- Each granted frame passes `dt` to the world in place of the host's + --- delta, so a script of steps reproduces the same simulated time on every + --- run and every host. The delta applies to every granted frame not yet + --- run, including ones an earlier call granted. + --- @param self the paused application + --- @param frames how many more frames to run, at least one + --- @param dt the fixed delta of each granted frame, in seconds + --- @raises string when the application is not paused + --- @raises string when `frames` is not a positive integer or `dt` is not a + --- positive finite number + step: function(exclusive self: Application, frames: integer, dt: number): nil + + --- Returns the frames `step` granted that have not started. + --- @param self the application to inspect + --- @return the count, zero while running or once every granted frame ran + pendingSteps: function(borrows self: Application): integer + + --- Returns the fixed delta granted frames run with. + --- @param self the application to inspect + --- @return the delta in seconds the last `step` named, or 1/60 before one + stepDelta: function(borrows self: Application): number + + --- Returns how many events wait for the next frame that runs. + --- @param self the application to inspect + --- @return the count of queued and held events + pendingEvents: function(borrows self: Application): integer end --- Settings copied by `newApplication`. @@ -112,6 +201,10 @@ export type Config = { debugMaxFrames: integer? } +-- The most events a paused application holds for its next frame. Past it the +-- oldest are dropped, as the host's own input routes drop theirs. +local HELD_LIMIT: integer = 1024 + -- The tool registry names one application per runtime. A second listener must -- not silently redirect the first application's tools into another world. local debugOwner: Application? = nil @@ -179,7 +272,12 @@ export function newApplication(config: Config?): Application debug = config ~= nil and config.debug == true, debugMaxFrames = maxFrames, failure = nil, - shutdownDone = false + shutdownDone = false, + paused = false, + steps = 0, + stepDt = 1 / 60, + held = {}, + injected = {} ) app.bindDebug = function(): nil debugOwner = app @@ -263,8 +361,45 @@ function Application.iterate(exclusive self: Application, dt: number, batch: {pl return false end + -- Arrival order: events held while paused, then this turn's host batch, + -- then what `queueEvent` added since the last iteration. + local incoming = self.held + for _, event in ipairs(batch or {}) do + incoming[#incoming + 1] = event + end + for _, event in ipairs(self.injected) do + incoming[#incoming + 1] = event + end + self.held = {} + self.injected = {} + if self.paused then + if self.steps <= 0 then + -- A held turn: nothing runs and the events wait for a frame that + -- does, but a quit is a request to the process rather than to a + -- frame, so it is still honored. + for index = 1, #incoming do + if incoming[index] is platformevents.QuitEvent then + self.quitRequested = true + return false + end + end + if #incoming > HELD_LIMIT then + -- A long pause under a live window must not grow without bound. + local kept: {platformevents.Event} = {} + for index = #incoming - HELD_LIMIT + 1, #incoming do + kept[#kept + 1] = incoming[index] + end + incoming = kept + end + self.held = incoming + + return true + end + self.steps = self.steps - 1 + dt = self.stepDt + end + self.input:beginFrame() - local incoming = batch or {} local handler = self.onEvent for index = 1, #incoming do local event = incoming[index] @@ -307,6 +442,45 @@ function Application.iterate(exclusive self: Application, dt: number, batch: {pl return not self.quitRequested end +function Application.queueEvent(exclusive self: Application, event: platformevents.Event): nil + self.injected[#self.injected + 1] = event +end + +function Application.pause(exclusive self: Application): nil + self.paused = true +end + +function Application.resume(exclusive self: Application): nil + self.paused = false + self.steps = 0 +end + +function Application.step(exclusive self: Application, frames: integer, dt: number): nil + if not self.paused then + error("application.step: the application is not paused", 2) + end + if frames < 1 or frames % 1 ~= 0 then + error("application.step: frames must be a positive integer", 2) + end + if not (dt > 0) or dt == math.huge then + error("application.step: dt must be a positive finite number", 2) + end + self.steps = self.steps + frames + self.stepDt = dt +end + +function Application.pendingSteps(borrows self: Application): integer + return self.steps +end + +function Application.stepDelta(borrows self: Application): number + return self.stepDt +end + +function Application.pendingEvents(borrows self: Application): integer + return #self.held + #self.injected +end + --- Requests that the host stop after the current callback returns. --- @param self the application to stop function Application.requestQuit(exclusive self: Application): nil diff --git a/src/tecs/internal/mcpbindings.nupp b/src/tecs/internal/mcpbindings.nupp index 798936ba..e10bdc0f 100644 --- a/src/tecs/internal/mcpbindings.nupp +++ b/src/tecs/internal/mcpbindings.nupp @@ -16,6 +16,7 @@ share one binding without either requiring the other. local ecs = require("tecs.ecs") local input = require("tecs.input") +local platformevents = require("tecs.platform.events") local window = require("tecs.platform.window") --- The application surface debug tools use, independent of lifecycle ownership. @@ -29,6 +30,14 @@ export interface Application getState: function(borrows self: Application): string requestQuit: function(exclusive self: Application): nil clearCrash: function(exclusive self: Application): (boolean, string?) + paused: boolean + queueEvent: function(exclusive self: Application, event: platformevents.Event): nil + pause: function(exclusive self: Application): nil + resume: function(exclusive self: Application): nil + step: function(exclusive self: Application, frames: integer, dt: number): nil + pendingSteps: function(borrows self: Application): integer + stepDelta: function(borrows self: Application): number + pendingEvents: function(borrows self: Application): integer end local boundWorld: ecs.World? = nil diff --git a/src/tecs/internal/mcpregistry.nupp b/src/tecs/internal/mcpregistry.nupp index 79f08f6a..b567710f 100644 --- a/src/tecs/internal/mcpregistry.nupp +++ b/src/tecs/internal/mcpregistry.nupp @@ -82,6 +82,18 @@ local revision: integer = 0 --- The traceback of a guarded game failure, once one has been recorded. local crashedText: string? = nil +--- Whether the tool call in progress runs while a lifecycle operation is parked. +local callingParked: boolean = false + +--- Reports whether the tool call in progress runs while a lifecycle +--- operation is parked, which a `whenParked` handler reads to tell an agent +--- that a frame is still mid-flight. +--- @return true inside a handler called during a parked operation, and false +--- otherwise, including outside any handler +export function parkedCall(): boolean + return callingParked +end + --- Records a crash so every world-touching tool reports it instead of running. --- @param traceback the failure text, or nil to clear the crashed state export function setCrashed(traceback: string?): nil @@ -202,7 +214,9 @@ export function callTool(tool: Tool, arguments: {[string]: any}, parked: boolean end -- A tool that raises must not take the server with it: the agent is told, -- and the frame carries on. + callingParked = parked == true local called, payload = pcall(tool.handler, arguments) + callingParked = false if not called then logger:warn("tool %s failed: %s", tool.name, tostring(payload)) diff --git a/src/tecs/internal/mcpschemas.nupp b/src/tecs/internal/mcpschemas.nupp index a4d5de51..7632e0e5 100644 --- a/src/tecs/internal/mcpschemas.nupp +++ b/src/tecs/internal/mcpschemas.nupp @@ -278,3 +278,75 @@ export const inputs: { ["additionalProperties"] = false } } + +-- Schemas for tools added after that listing, so not checked against it. +-- `screenshot` keeps the historical tool's `name` key and adds `path`. Names +-- and argument keys are external compatibility contracts all the same. +export const added: { + [string]: any +} = { + ["clock_info"] = {["properties"] = json.EMPTY_OBJECT, ["type"] = "object", ["additionalProperties"] = false}, + ["clock_pause"] = {["properties"] = json.EMPTY_OBJECT, ["type"] = "object", ["additionalProperties"] = false}, + ["clock_resume"] = {["properties"] = json.EMPTY_OBJECT, ["type"] = "object", ["additionalProperties"] = false}, + [ + "clock_step" + ] = { + [ + "properties" + ] = { + [ + "frames" + ] = { + ["type"] = "number", + ["description"] = "Sets how many more frames run, as a whole number. (default: 1)", + ["minimum"] = 1 + }, + [ + "dt" + ] = { + ["type"] = "number", + ["description"] = "Sets the fixed delta of each granted frame in seconds. (default: 1/60)", + ["minimum"] = 1e-06, + ["maximum"] = 1 + } + }, + ["type"] = "object", + ["additionalProperties"] = false + }, + [ + "screenshot" + ] = { + [ + "properties" + ] = { + [ + "name" + ] = { + ["type"] = "string", + ["description"] = "File name under the writable root. Defaults to screenshot.png." + }, + [ + "path" + ] = { + ["type"] = "string", + ["description"] = "Writes the PNG at this absolute path instead of under the writable root." + } + }, + ["type"] = "object", + ["additionalProperties"] = false + }, + [ + "screenshot_info" + ] = { + [ + "properties" + ] = { + [ + "id" + ] = {["type"] = "number", ["description"] = "Selects the capture `screenshot` reported.", ["minimum"] = 1} + }, + ["required"] = {"id"}, + ["type"] = "object", + ["additionalProperties"] = false + } +} diff --git a/src/tecs/internal/mcpsubsystems.nupp b/src/tecs/internal/mcpsubsystems.nupp index 382d622c..05dfb8c1 100644 --- a/src/tecs/internal/mcpsubsystems.nupp +++ b/src/tecs/internal/mcpsubsystems.nupp @@ -14,6 +14,7 @@ local materials = require("tecs.gpu.materials") local layers = require("tecs.gfx.layers") local physics = require("tecs.physics") local audio = require("tecs.audio") +local screenshot = require("tecs.gfx.screenshot") local files = require("nupp.io.files") local path = require("nupp.io.path") local json = require("nupp.codec.json") @@ -27,7 +28,7 @@ local function register( mutates: boolean?, independent: boolean? ): nil - local schema = schemas.inputs[name] + local schema = schemas.inputs[name] ?? schemas.added[name] registry.register({ name = name, description = description, @@ -241,13 +242,8 @@ register( layers = layers.MAX, camera = cameraInfo(), frame = app ~= nil and app.frame or 0, - observations = {gpuCounters = false, readback = false}, - unavailable = json.asArray({ - "presented frame ID", - "GPU timings", - "visible instance count", - "pixel readback" - }) + observations = {gpuCounters = false, readback = true}, + unavailable = json.asArray({"presented frame ID", "GPU timings", "visible instance count"}) } end ) @@ -432,6 +428,195 @@ register( true ) +------------------------------------------------------------------------------- +-- The simulation clock +-- +-- `clock_pause`, `clock_step`, `clock_resume` and `clock_info` reach the +-- application's pause, step and queue state only, never the world, so they run +-- while a frame is parked and after a crash. A harness polls `clock_info` until +-- `settled` rather than retrying a refused world tool. +------------------------------------------------------------------------------- + +local function clockInfo(): {[string]: any} + local app = bindings.requireApplication() + local parked = registry.parkedCall() + local steps = app:pendingSteps() + + return { + paused = app.paused, + frame = app.frame, + elapsed = app.elapsed, + pendingSteps = steps, + pendingEvents = app:pendingEvents(), + stepDt = app:stepDelta(), + parked = parked, + settled = app.paused and steps == 0 and not parked + } +end + +register( + "clock_info", + "Report whether the simulation is paused, the frame count, granted frames still to run, and whether a frame is " + .. "parked mid-flight. `settled` is true once a paused clock has run every granted frame.", + function(_args: {[string]: any}): {[string]: any} + return clockInfo() + end, + false, + true +) +register( + "clock_pause", + "Hold the simulation: host turns run no system and fold no input until clock_step grants frames or clock_resume " + .. "releases it. Rendering continues.", + function(_args: {[string]: any}): {[string]: any} + local app = bindings.requireApplication() + app:pause() + return clockInfo() + end, + true, + true +) +register( + "clock_step", + "Grant frames to a paused simulation, one per host turn, each with a fixed delta instead of the wall clock.", + function(args: {[string]: any}): {[string]: any} + local app = bindings.requireApplication() + if not app.paused then + error("not_paused: the clock is running; call clock_pause before clock_step", 0) + end + app:step(integer(args.frames or 1), args.dt or 1 / 60) + + return clockInfo() + end, + true, + true +) +register( + "clock_resume", + "Let the simulation run on the host's measured delta again, discarding granted frames not yet run.", + function(_args: {[string]: any}): {[string]: any} + local app = bindings.requireApplication() + app:resume() + return clockInfo() + end, + true, + true +) + +------------------------------------------------------------------------------- +-- Screenshots +-- +-- The host reads back a frame only after it renders one, so a tool call can +-- only request a capture. The server writes each finished capture from its +-- next poll, and `screenshot_info` reports it. +------------------------------------------------------------------------------- + +local type Shot = { + id: integer, + path: string, + capture: screenshot.Capture?, + ready: boolean, + saved: boolean, + reason: string?, + width: integer, + height: integer, + bytes: integer +} + +local shots: {[integer]: Shot} = {} +local waiting: {Shot} = {} +local shotSerial: integer = 0 + +--- Writes every requested screenshot whose frame the host has read back. +--- +--- `tecs.mcp.poll` calls this once per poll, so a capture lands on disk on the +--- turn after the host renders it whether or not anyone asks. +export function settleCaptures(): nil + if #waiting == 0 then + return + end + local still: {Shot} = {} + for _, shot in ipairs(waiting) do + local capture = shot.capture + if capture ~= nil and capture.ready then + shot.capture = nil + local _, width, height = screenshot.pixels(capture) + local png, failure = screenshot.encode(capture) + if png == nil then + shot.reason = failure + else + local saved, reason = screenshot.save(capture, shot.path) + shot.saved = saved + shot.reason = reason + shot.width, shot.height, shot.bytes = width, height, #png + end + shot.ready = true + else + still[#still + 1] = shot + end + end + waiting = still +end + +local function shotInfo(borrows shot: Shot): {[string]: any} + return { + id = shot.id, + path = shot.path, + ready = shot.ready, + saved = shot.saved, + reason = shot.reason ?? json.NULL, + width = shot.width, + height = shot.height, + bytes = shot.bytes + } +end + +register( + "screenshot", + "Capture the next rendered frame to a PNG under the writable root or at an absolute path. Returns at once with " + .. "the capture's id and path; screenshot_info reports when the file is written.", + function(args: {[string]: any}): {[string]: any} + local destination: string + if args.path ~= nil then + assert(path.newPath(args.path):isAbsolute(), "screenshot path must be absolute") + destination = args.path + else + destination = artifact(args.name or "screenshot.png") + end + shotSerial = shotSerial + 1 + const shot: Shot = { + id = shotSerial, + path = destination, + capture = screenshot.request(), + ready = false, + saved = false, + reason = nil, + width = 0, + height = 0, + bytes = 0 + } + shots[shot.id] = shot + waiting[#waiting + 1] = shot + + return shotInfo(shot) + end, + true, + true +) +register( + "screenshot_info", + "Report whether a requested screenshot has been written, with its size or why it failed.", + function(args: {[string]: any}): {[string]: any} + settleCaptures() + local shot = shots[integer(args.id)] + assert(shot ~= nil, "unknown screenshot: " .. tostring(args.id)) + + return shotInfo(shot) + end, + false, + true +) + local families: { [string]: boolean } = { @@ -445,7 +630,9 @@ local families: { snapshot = true, profile = true, systems = true, - states = true + states = true, + clock = true, + screenshot = true } local function groupOf(name: string): string diff --git a/src/tecs/internal/mcptools.nupp b/src/tecs/internal/mcptools.nupp index f6bac279..f900a82a 100644 --- a/src/tecs/internal/mcptools.nupp +++ b/src/tecs/internal/mcptools.nupp @@ -24,7 +24,6 @@ local ecs = require("tecs.ecs") local files = require("nupp.io.files") local json = require("nupp.codec.json") local platformevents = require("tecs.platform.events") -local platformwindow = require("tecs.platform.window") local time = require("nupp.time") local bindings = require("tecs.internal.mcpbindings") local registry = require("tecs.internal.mcpregistry") @@ -325,10 +324,19 @@ registry.register({ -- reading a recording or the event reference already knows; the Teal tool -- itself exposed none of them, so it could list `gamepadAxis` among its -- kinds and then not let anyone push one. + -- + -- The event is queued for the next frame that runs rather than folded into + -- the input on the spot. A tool call lands between frames, before the next + -- frame's `beginFrame` clears the per-frame edges, so a key folded + -- immediately was down but never pressed. Queuing only touches the + -- application's queue, which is why the tool may also run while a frame is + -- parked. name = "send_event", - description = "Pushes an event into the input stream, as if it had come from the platform.", + description = "Queues an event for the next frame that runs, delivered after " + .. "the host's own input for that frame as if it had come from the platform.", destructive = true, whenCrashed = true, + whenParked = true, inputSchema = { type = "object", properties = { @@ -382,17 +390,12 @@ registry.register({ local event = buildEvent(kind, arguments) local app = bindings.requireApplication() - -- The same three folds one host iteration performs on a sealed batch, - -- so a synthetic event reaches window state, input state and the - -- world's observers exactly as a real one does. - platformwindow.handleEvent(app.window, event) - app.input:handleEvent(event) - platformevents.dispatch(app.world, event) - if event is platformevents.Quit or event is platformevents.WindowCloseRequested then - app:requestQuit() - end + -- The next frame folds it with the host's own batch, so it reaches + -- window state, input state, the world's observers and a quit request + -- exactly as a real event does. + app:queueEvent(event) - return {pushed = kind} + return {pushed = kind, frame = app.frame, pending = app:pendingEvents()} end }) diff --git a/src/tecs/mcp.nupp b/src/tecs/mcp.nupp index 28f102ec..630a7b2f 100644 --- a/src/tecs/mcp.nupp +++ b/src/tecs/mcp.nupp @@ -12,10 +12,12 @@ including after a crash or while gameplay is suspended. For a standalone server, call `poll` outside the world update scope and `destroy` at teardown. Each poll answers at most one tool request; handlers must answer promptly. -While a lifecycle operation is parked, `ping`, `context` and `get_logs` remain -available. Other tools report that the operation must complete first. A custom -tool may opt in with `whenParked = true` only if it never touches the in-progress -world. `readOnly` alone grants no such permission. +While a lifecycle operation is parked, `ping`, `context`, `get_logs`, +`send_event`, the `clock_*` tools and the `screenshot` tools remain available. +Other tools report that the operation must complete first; `clock_info` +reports `parked` so an agent can wait for the frame instead of retrying them. A +custom tool may opt in with `whenParked = true` only if it never touches the +in-progress world. `readOnly` alone grants no such permission. ```nupp local server = assert(tecs.mcp.listen(7100)) @@ -57,11 +59,43 @@ component update and `set` for a complete value or a missing component. structured tools to `run_lua`; the Lua sandbox limits accidents but does not create a security boundary. +## Driving frames deterministically + +`send_event` queues its event for the next frame that runs, behind the host's +own input for that frame, so a key it sends is pressed on exactly that frame. +`clock_pause` holds the simulation: host turns run no system and fold no input, +and `frame` and `elapsed` stand still, while the host keeps rendering the held +world. `clock_step` grants frames to a paused clock, one per host turn, each +with a fixed `dt` (1/60 s by default) instead of the measured one, and +`clock_resume` returns to the host's delta. `clock_info` reports `settled` once +a paused clock has run every granted frame and no frame is parked mid-flight, +which is when the world is safe to read. + +`screenshot` requests a capture of the next rendered frame and answers at once +with an `id` and the `path` it will write, under the writable root or at an +absolute `path`. The server writes the PNG on its first poll after the host +reads the frame back, in a windowed or an `--offscreen` host, and +`screenshot_info` reports `ready`, `saved`, the size, or the failure. A +`--headless` host renders nothing, so its captures never become ready. + +A harness turn, as JSON-RPC `tools/call` requests: + +```text +clock_pause {} +send_event {"kind": "keyDown", "physicalKey": "Space"} +clock_step {"frames": 1} +clock_info {} -- repeat until "settled": true +send_event {"kind": "keyUp", "physicalKey": "Space"} +clock_step {"frames": 1} +screenshot {"path": "/tmp/frame.png"} +screenshot_info {"id": 1} -- repeat until "ready": true +``` + ## After a crash `setCrashed` records a guarded gameplay failure. The server keeps polling after -one: `ping`, `context`, `get_logs`, `send_event` and `clear_crash` remain -available, and every tool that touches the world reports the stored traceback +one: `ping`, `context`, `get_logs`, `send_event`, `clear_crash`, the +`clock_*` tools and the `screenshot` tools remain available, and every tool that touches the world reports the stored traceback instead of running. ## Subsystem tools @@ -74,7 +108,7 @@ sampler; stop writes collapsed stacks under the writable root. `help`, `describe` and `capabilities` enumerate the live registry. They and the sampler remain available while parked or crashed. The server does not provide -GPU readback, shader or image reload, watcher orchestration, or physics debug +pixel sampling, shader or image reload, watcher orchestration, or physics debug overlay tools. Snapshot files use TECS framing over Nupp's binary codec and refuse unsupported framing or physics encodings. ]] @@ -247,6 +281,7 @@ export function poll(exclusive self: Server, parked: boolean?): boolean self.endpoint:setTools(registry.describeJSON()) end self.endpoint:pump() + subsystems.settleCaptures() local request = self.endpoint:nextRequest() if request == nil then diff --git a/tests/applicationtest.nupp b/tests/applicationtest.nupp index 3c16467c..acebe261 100644 --- a/tests/applicationtest.nupp +++ b/tests/applicationtest.nupp @@ -208,4 +208,164 @@ function suite.rejectsInvalidHostLifecycleCalls(): nil ) end +--- Builds a running application whose one Update system records each frame's +--- delta and whether Space was pressed on it. +local function recorder(): (application.Application, {number}, {boolean}) + local deltas: {number} = {} + local pressed: {boolean} = {} + local app = application.newApplication({ + plugin = function(exclusive running: application.Application): nil + const input = running.input + running.world:addSystem({ + name = "test.application.recorder", + phase = ecs.phases.Update, + run = function(dt: number): nil + deltas[#deltas + 1] = dt + pressed[#pressed + 1] = input:keyPressed("Space") + end, + }) + end, + }) + app:init() + + return app, deltas, pressed +end + +local function space(down: boolean): platformevents.Event + return platformevents.key(down, "Space", " ", nil, 0, false, 0) +end + +function suite.deliversAQueuedEventAsAnEdgeOfTheNextFrame(): nil + local app, _, pressed = recorder() + app:queueEvent(space(true)) + test.equal(app:pendingEvents(), 1) + test.assert(not app.input:keyDown("Space")) + + app:iterate(1 / 60, nil) + test.assert(pressed[1]) + test.equal(app:pendingEvents(), 0) + + -- The edge lasts one frame, and the key stays down until it comes up. + app:iterate(1 / 60, nil) + test.assert(not pressed[2]) + test.assert(app.input:keyDown("Space")) + app:queueEvent(space(false)) + app:iterate(1 / 60, nil) + test.assert(app.input:keyReleased("Space")) + app:shutdown() +end + +function suite.deliversQueuedEventsAfterTheHostBatch(): nil + local order: {string} = {} + local app = application.newApplication({ + onEvent = function(event: platformevents.Event, exclusive _running: application.Application): nil + order[#order + 1] = event.kind + end, + }) + app:init() + app:queueEvent(platformevents.text("queued", 0)) + app:iterate(1 / 60, {platformevents.focused(true, 0)}) + test.equal(#order, 2) + test.equal(order[1], "windowFocusGained") + test.equal(order[2], "textInput") + app:shutdown() +end + +function suite.holdsAPausedSimulationAndRunsGrantedFramesOnAFixedDelta(): nil + local app, deltas, pressed = recorder() + app:iterate(0.5, nil) + test.equal(#deltas, 1) + + app:pause() + test.assert(app.paused) + local frame, elapsed = app.frame, app.elapsed + app:queueEvent(space(true)) + -- Held turns run nothing, advance nothing, and keep the events. + test.assert(app:iterate(0.5, {platformevents.focused(true, 0)})) + test.assert(app:iterate(0.5, nil)) + test.equal(#deltas, 1) + test.equal(app.frame, frame) + test.equal(app.elapsed, elapsed) + test.equal(app:pendingEvents(), 2) + test.assert(not app.window:hasFocus()) + + app:step(2, 0.25) + test.equal(app:pendingSteps(), 2) + test.equal(app:stepDelta(), 0.25) + app:iterate(0.5, nil) + test.equal(#deltas, 2) + test.equal(deltas[2], 0.25) + test.assert(pressed[2]) + test.assert(app.window:hasFocus()) + app:iterate(9, nil) + test.equal(deltas[3], 0.25) + test.assert(not pressed[3]) + test.equal(app.frame, frame + 2) + test.equal(app.elapsed, elapsed + 0.5) + test.equal(app:pendingSteps(), 0) + + -- Granted frames used up, the clock holds again. + app:iterate(0.5, nil) + test.equal(#deltas, 3) + + -- Resuming discards a grant and returns to the host's delta. + app:step(5, 0.1) + app:resume() + test.assert(not app.paused) + test.equal(app:pendingSteps(), 0) + app:iterate(0.5, nil) + test.equal(deltas[4], 0.5) + app:shutdown() +end + +function suite.honorsAQuitWhilePaused(): nil + local app = recorder() + app:pause() + test.assert(not app:iterate(1 / 60, {platformevents.quit(0)})) + test.assert(app.quitRequested) + app:shutdown() +end + +function suite.refusesToStepARunningClockOrAnInvalidGrant(): nil + local app = recorder() + test.raises( + function(): nil + app:step(1, 1 / 60) + end, + "not paused" + ) + app:pause() + test.raises( + function(): nil + app:step(0, 1 / 60) + end, + "positive integer" + ) + test.raises( + function(): nil + app:step(1, 0) + end, + "positive finite" + ) + test.raises( + function(): nil + app:step(1, math.huge) + end, + "positive finite" + ) + app:shutdown() +end + +function suite.boundsTheEventsAPausedApplicationHolds(): nil + local app = recorder() + app:pause() + local batch: {platformevents.Event} = {} + for index = 1, 1500 do + batch[index] = platformevents.text(tostring(index), 0) + end + app:iterate(1 / 60, batch) + test.equal(app:pendingEvents(), 1024) + app:shutdown() +end + return suite diff --git a/tests/mcpsessiontest.nupp b/tests/mcpsessiontest.nupp index 5dc98d76..405b0678 100644 --- a/tests/mcpsessiontest.nupp +++ b/tests/mcpsessiontest.nupp @@ -180,7 +180,7 @@ function suite.inspectsAndControlsARunningApplicationOverTheTransport(): nil callOverHTTP(port, "states_pop") test.equal(callOverHTTP(port, "states_info").depth as integer, 0) - -- Control: a synthetic event reaches the same folds a real one does. + -- Control: a synthetic event is queued for the folds a real one takes. local sent = callOverHTTP(port, "send_event", {kind = "keyDown", physicalKey = "KeyF"}) test.equal(sent.pushed as string, "keyDown") @@ -190,6 +190,10 @@ function suite.inspectsAndControlsARunningApplicationOverTheTransport(): nil test.equal(callOverHTTP(port, "stats").entities as integer, 3) end) + -- The synthetic key arrives with the next frame, as a platform key does. + test.assert(not app.input:keyDown("KeyF")) + app:iterate(1 / 60, nil) + test.assert(app.input:keyPressed("KeyF")) test.assert(app.input:keyDown("KeyF")) -- The stopped system does not run, so the counter it owns stands still. diff --git a/tests/mcptest.nupp b/tests/mcptest.nupp index a5a64ab9..49c01e07 100644 --- a/tests/mcptest.nupp +++ b/tests/mcptest.nupp @@ -297,6 +297,9 @@ function suite.pushesTouchAndGamepadEventsThroughSendEvent(): nil }), "fingerDown" ) + -- Queued for the next frame rather than folded on the spot. + test.equal(#app.input:touches(), 0) + app:iterate(1 / 60, nil) local held = app.input:touches() test.equal(#held, 1) test.equal(held[1].finger, "1") @@ -307,6 +310,7 @@ function suite.pushesTouchAndGamepadEventsThroughSendEvent(): nil test.equal(pushed({kind = "gamepadAdded", which = 4}), "gamepadAdded") test.equal(pushed({kind = "gamepadButtonDown", which = 4, button = 1}), "gamepadButtonDown") test.equal(pushed({kind = "gamepadAxis", which = 4, axis = 1, value = 0.8}), "gamepadAxis") + app:iterate(1 / 60, nil) local pad = assert(app.input:gamepadById(4)) test.assert(pad:buttonDown("south")) @@ -318,4 +322,38 @@ function suite.pushesTouchAndGamepadEventsThroughSendEvent(): nil app:shutdown() end +function suite.sendEventKeyIsPressedOnTheNextFrame(): nil + local pressedOn: {integer} = {} + local frames: integer = 0 + local app: application.Application = application.newApplication({ + plugin = function(exclusive running: application.Application): nil + const input = running.input + running.world:addSystem({ + name = "test.mcp.pressed", + phase = ecs.phases.Update, + run = function(_dt: number): nil + frames = frames + 1 + if input:keyPressed("KeyQ") then + pressedOn[#pressedOn + 1] = frames + end + end + }) + end + }) + app:init() + mcp.bind(app.world, app) + app:iterate(1 / 60, nil) + + local sent = call("send_event", {kind = "keyDown", physicalKey = "KeyQ"}).structuredContent as {[string]: any} + test.equal(sent.pending as integer, 1) + app:iterate(1 / 60, nil) + app:iterate(1 / 60, nil) + test.equal(#pressedOn, 1) + test.equal(pressedOn[1], 2) + test.assert(app.input:keyDown("KeyQ")) + + mcp.bind(nil, nil) + app:shutdown() +end + return suite diff --git a/tests/tecs/compatibilitytest.nupp b/tests/tecs/compatibilitytest.nupp index cdee9e05..30de1770 100644 --- a/tests/tecs/compatibilitytest.nupp +++ b/tests/tecs/compatibilitytest.nupp @@ -687,6 +687,10 @@ function suite.mcpToolNames(): nil "camera_move", "capabilities", "clear_crash", + "clock_info", + "clock_pause", + "clock_resume", + "clock_step", "components_info", "context", "describe", @@ -711,6 +715,8 @@ function suite.mcpToolNames(): nil "resources_info", "resources_list", "run_lua", + "screenshot", + "screenshot_info", "send_event", "set", "snapshot_load", @@ -769,6 +775,12 @@ function suite.mcpArgumentKeys(): nil equalList(argumentKeys("systems_start"), {"name"}) equalList(argumentKeys("systems_stop"), {"name"}) equalList(argumentKeys("states_push"), {"name"}) + equalList(argumentKeys("clock_info"), {}) + equalList(argumentKeys("clock_pause"), {}) + equalList(argumentKeys("clock_resume"), {}) + equalList(argumentKeys("clock_step"), {"dt", "frames"}) + equalList(argumentKeys("screenshot"), {"name", "path"}) + equalList(argumentKeys("screenshot_info"), {"id"}) end function suite.sendEventArgumentKeys(): nil diff --git a/tests/tecs/mcpsubsystemstest.nupp b/tests/tecs/mcpsubsystemstest.nupp index 06f55b5a..a3b61420 100644 --- a/tests/tecs/mcpsubsystemstest.nupp +++ b/tests/tecs/mcpsubsystemstest.nupp @@ -7,6 +7,8 @@ local physics = require("tecs.physics") local audio = require("tecs.audio") local audiobackend = require("tecs.platform.audiobackend") local mcp = require("tecs.mcp") +local application = require("tecs.application") +local screenshot = require("tecs.gfx.screenshot") local registry = require("tecs.internal.mcpregistry") local schemas = require("tecs.internal.mcpschemas") local entityworld = require("tecs.internal.world") @@ -71,14 +73,7 @@ function suite.restoredInputSchemasMatchTheHistoricalWireListing(): nil end end test.equal(checked, 22) - for _, name in ipairs({ - "screenshot", - "sample_pixels", - "reload_shaders", - "reload_image", - "watch", - "physics_debug" - }) do + for _, name in ipairs({"sample_pixels", "reload_shaders", "reload_image", "watch", "physics_debug"}) do test.equal(registry.findTool(name), nil) end end @@ -221,4 +216,142 @@ function suite.profileToolsWriteAReportAndCanStartAgain(): nil test.equal(call("profile_stop").isError, true) end +--- Builds a bound application whose Update system counts frames and records +--- whether Space was pressed on each. +local function steppedApplication(): (application.Application, {number}, {boolean}) + local deltas: {number} = {} + local pressed: {boolean} = {} + local app = application.newApplication({ + plugin = function(exclusive running: application.Application): nil + const input = running.input + running.world:addSystem({ + name = "test.mcp.clock", + phase = ecs.phases.Update, + run = function(dt: number): nil + deltas[#deltas + 1] = dt + pressed[#pressed + 1] = input:keyPressed("Space") + end + }) + end + }) + app:init() + mcp.bind(app.world, app) + + return app, deltas, pressed +end + +function suite.clockToolsPauseStepAndResumeTheSimulation(): nil + local app, deltas, pressed = steppedApplication() + app:iterate(0.5, nil) + + local info = content("clock_info") + test.equal(info.paused, false) + test.equal(info.settled, false) + test.equal(info.frame, 1) + test.equal(call("clock_step").isError, true) + + test.equal(content("clock_pause").paused, true) + content("send_event", {kind = "keyDown", physicalKey = "Space", logicalKey = " "}) + app:iterate(0.5, nil) + app:iterate(0.5, nil) + test.equal(#deltas, 1) + info = content("clock_info") + test.equal(info.settled, true) + test.equal(info.pendingEvents, 1) + + local stepped = content("clock_step", {frames = 2, dt = 0.125}) + test.equal(stepped.pendingSteps, 2) + test.equal(stepped.stepDt, 0.125) + test.equal(stepped.settled, false) + app:iterate(0.5, nil) + app:iterate(0.5, nil) + app:iterate(0.5, nil) + test.equal(#deltas, 3) + test.equal(deltas[2], 0.125) + test.equal(deltas[3], 0.125) + test.assert(pressed[2]) + test.assert(not pressed[3]) + info = content("clock_info") + test.equal(info.frame, 3) + test.equal(info.elapsed, 0.75) + test.equal(info.settled, true) + + test.equal(call("clock_step", {frames = 1.5}).isError, true) + test.equal(call("clock_step", {dt = 0}).isError, true) + test.equal(call("clock_step", {frames = 0}).isError, true) + + content("clock_step", {frames = 4}) + test.equal(content("clock_resume").paused, false) + test.equal(content("clock_info").pendingSteps, 0) + app:iterate(0.5, nil) + test.equal(deltas[4], 0.5) + + mcp.bind(nil, nil) + app:shutdown() +end + +function suite.clockAndInputToolsAnswerWhileAFrameIsParked(): nil + local app = steppedApplication() + content("clock_pause") + for _, name in ipairs({"clock_info", "clock_step", "send_event", "screenshot", "screenshot_info"}) do + test.equal(assert(registry.findTool(name)).whenParked, true) + end + local parked = registry.callTool(assert(registry.findTool("clock_info")), {}, true) + test.equal(parked.structuredContent.parked, true) + test.equal(parked.structuredContent.settled, false) + local sent = registry.callTool( + assert(registry.findTool("send_event")), + {kind = "keyDown", physicalKey = "KeyA"}, + true + ) + test.equal(sent.isError, nil) + test.equal(app:pendingEvents(), 1) + -- A tool that touches the world still waits for the frame. + test.equal(registry.callTool(assert(registry.findTool("stats")), {}, true).isError, true) + + mcp.bind(nil, nil) + app:shutdown() +end + +function suite.screenshotToolWritesTheNextRenderedFrame(): nil + local app = steppedApplication() + screenshot.cancelPending() + local requested = content("screenshot", {name = "mcp-screenshot-test.png"}) + test.equal(requested.ready, false) + test.assert(requested.path:find("mcp-screenshot-test.png", 1, true) ~= nil) + test.equal(content("screenshot_info", {id = requested.id}).ready, false) + + -- What the host does after it renders: it consumes the request and + -- answers with the frame. + local id = assert(screenshot.nextRequest()) + screenshot.complete(id, 1, 1, "\1\2\3\255", "PNG-BYTES", nil) + local server = assert(mcp.listen(0)) + mcp.poll(server) + test.equal(assert(files.read(requested.path)), "PNG-BYTES") + local written = content("screenshot_info", {id = requested.id}) + test.equal(written.ready, true) + test.equal(written.saved, true) + test.equal(written.width, 1) + test.equal(written.height, 1) + test.equal(written.bytes, 9) + assert(files.remove(requested.path)) + + -- A failed readback is reported rather than written. + local failing = content("screenshot", {name = "mcp-screenshot-failed.png"}) + screenshot.complete(assert(screenshot.nextRequest()), 0, 0, "", "", "GPU readback failed") + local failed = content("screenshot_info", {id = failing.id}) + test.equal(failed.ready, true) + test.equal(failed.saved, false) + test.equal(failed.reason, "GPU readback failed") + + test.equal(call("screenshot", {path = "relative.png"}).isError, true) + test.equal(call("screenshot_info", {id = 999999}).isError, true) + test.equal(content("describe", {command = "screenshot"}).action.name, "screenshot") + test.equal(#content("describe", {command = "clock"}).verbs, 4) + + mcp.destroy(server) + mcp.bind(nil, nil) + app:shutdown() +end + return suite