1.9.1: upload files through the sandbox browser's live view

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Upload files to a page through the sandbox browser's live view: a file chooser the page opens while someone drives is answered from the driver's own machine (picker or drop), over the live-view WebSocket. Ships in 1.9.1 with `@activeagents/browser-sidecar` 1.9.1.
+
 ## [1.9.0] - 2026-10-07
 
 Releases `activeagent` and `actionagent` 1.9.0 from one tag. A minor release.

actionagent/frontend/components/dashboard/LiveView.jsx

@@ -5,6 +5,8 @@ import {
   agentBanner,
   closeMessage,
   controlLabel,
+  encodeChunk,
+  fileChooserWaitLabel,
   initialLiveViewState,
   keyMessage,
   leavesView,
@@ -14,6 +16,9 @@ import {
   releaseMessage,
   takeOverButton,
   textMessage,
+  UPLOAD_CHUNK_BYTES,
+  uploadPlan,
+  uploadProgressLabel,
   wheelMessage,
 } from '../../utils/liveView.mjs';
 
@@ -34,11 +39,63 @@ async function requestTicket(sessionId, mode) {
 // its paste event carries the clipboard's text.
 const isPasteShortcut = (event) => (event.metaKey || event.ctrlKey) && event.key.toLowerCase() === 'v';
 
+// An upload waits while this much is still queued on the socket, so frames
+// keep arriving and the browser does not hold a whole file in its buffer.
+const UPLOAD_BUFFERED_BYTES = 1024 * 1024;
+const pause = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); });
+
+// What the person driving is shown when the page asks for files: a file
+// input of their own, as the page's would take (`chooser`, the sidecar's
+// file_chooser), and Cancel; or how far the upload has got.
+export function FileChooserPrompt({ chooser, upload, onFiles, onCancel }) {
+  const { darkMode } = useTheme();
+  const muted = darkMode ? 'text-gray-400' : 'text-gray-500';
+  const strong = darkMode ? 'text-white' : 'text-gray-900';
+  const several = chooser.multiple;
+
+  return (
+    <div
+      role="dialog"
+      aria-label="Choose a file for the page"
+      data-testid="live-view-file-chooser"
+      className={`absolute inset-x-4 top-4 p-3 rounded border shadow space-y-2 ${darkMode ? 'bg-gray-800 border-gray-600' : 'bg-white border-gray-300'}`}
+    >
+      <p className={`text-sm font-medium ${strong}`}>The page asked for {several ? 'files' : 'a file'}</p>
+      <p className={`text-xs ${muted}`}>
+        Choose {several ? `up to ${chooser.max_files} files` : 'one'} from this computer, or drop {several ? 'them' : 'it'} on the
+        view. {chooser.accept ? `The page takes ${chooser.accept}. ` : ''}Only the browser receives the files, and they stay in the
+        sandbox until the browser stops.
+      </p>
+      {upload ? (
+        <p className={`text-xs ${strong}`} role="status">{uploadProgressLabel(upload)}</p>
+      ) : (
+        <input
+          type="file"
+          multiple={several}
+          accept={chooser.accept || undefined}
+          onChange={(event) => onFiles(event.target.files)}
+          aria-label={several ? 'Files for the page' : 'File for the page'}
+          className={`block text-xs ${strong}`}
+        />
+      )}
+      <button
+        type="button"
+        onClick={onCancel}
+        className={`px-3 py-1 text-sm rounded ${darkMode ? 'bg-gray-700 text-gray-300 hover:bg-gray-600' : 'bg-gray-200 text-gray-700 hover:bg-gray-300'}`}
+      >
+        Cancel
+      </button>
+    </div>
+  );
+}
+
 // A sandbox browser's live view: the page on screen, drawn from the frames
 // the browser streams, with "Watch live", "Take over" and "Hand back".
 // While this viewer holds control, its mouse and wheel over the frame go to
 // the page, and so do its keys while the frame has keyboard focus. It holds
-// control, and the agent waits, until Hand back or Stop watching. `headed`
+// control, and the agent waits, until Hand back or Stop watching. When the
+// page asks for a file while this viewer drives, it is chosen here (or
+// dropped on the view) and sent to the browser. `headed`
 // is whether the browser also has a window open on the dashboard's machine,
 // and `watchOnOpen` starts watching when the view is first shown.
 export default function LiveView({ sessionId, headed = false, watchOnOpen = false }) {
@@ -47,6 +104,7 @@ export default function LiveView({ sessionId, headed = false, watchOnOpen = fals
   const [connection, setConnection] = useState('idle'); // idle | connecting | open
   const [notice, setNotice] = useState(null);
   const [taking, setTaking] = useState(false);
+  const [upload, setUpload] = useState(null); // { sent, total }
   const socketRef = useRef(null);
   const closingRef = useRef(false);
   // Counts calls to watch and disconnect, so a watch whose ticket arrives
@@ -59,6 +117,8 @@ export default function LiveView({ sessionId, headed = false, watchOnOpen = fals
   const moveFrame = useRef({ message: null, scheduled: false });
   const lastPoint = useRef(null);
   const lastEscape = useRef(null);
+  // Counts uploads begun and cancelled, so a cancelled one sends no more.
+  const uploadRef = useRef(0);
   const mine = state.control.mine;
 
   const send = useCallback((message) => {
@@ -135,11 +195,17 @@ export default function LiveView({ sessionId, headed = false, watchOnOpen = fals
         }
         if (message.type === 'ready') setConnection('open');
         if (message.type === 'frame') draw(message);
+        if ((message.type === 'file_upload' && message.status !== 'progress') || (message.type === 'file_chooser' && !message.chooser)) {
+          uploadRef.current += 1;
+          setUpload(null);
+        }
         dispatch(message);
       };
       ws.onclose = (event) => {
         if (socketRef.current !== ws) return;
         socketRef.current = null;
+        uploadRef.current += 1;
+        setUpload(null);
         setConnection('idle');
         dispatch({ type: 'reset' });
         setNotice(closeMessage(event.code, { requested: closingRef.current }));
@@ -176,6 +242,60 @@ export default function LiveView({ sessionId, headed = false, watchOnOpen = fals
     canvasRef.current?.blur();
   };
 
+  // Sends the files a person chose, or dropped on the view, to the page's
+  // file chooser: their names and sizes, then their bytes a chunk at a time.
+  const uploadFiles = async (fileList) => {
+    const chooser = state.fileChooser;
+    const plan = uploadPlan(fileList, chooser);
+    if (plan.error) {
+      setNotice(plan.error);
+      return;
+    }
+    const ws = socketRef.current;
+    if (!ws || ws.readyState !== WebSocket.OPEN) return;
+
+    uploadRef.current += 1;
+    const attempt = uploadRef.current;
+    setNotice(null);
+    dispatch({ type: 'clear_error' });
+    send({ type: 'file_upload', chooser: chooser.id, files: plan.files });
+    let sent = 0;
+    setUpload({ sent, total: plan.total });
+    try {
+      for (const file of Array.from(fileList)) {
+        for (let offset = 0; offset < file.size; offset += UPLOAD_CHUNK_BYTES) {
+          while (ws.bufferedAmount > UPLOAD_BUFFERED_BYTES && uploadRef.current === attempt) await pause(50);
+          if (uploadRef.current !== attempt || ws.readyState !== WebSocket.OPEN) return;
+          const bytes = new Uint8Array(await file.slice(offset, offset + UPLOAD_CHUNK_BYTES).arrayBuffer());
+          send({ type: 'file_chunk', chooser: chooser.id, data: encodeChunk(bytes) });
+          sent += bytes.length;
+          setUpload({ sent, total: plan.total });
+        }
+      }
+    } catch (e) {
+      setNotice(`Could not read the file: ${e.message}`);
+      cancelChooser();
+    }
+  };
+
+  const cancelChooser = () => {
+    uploadRef.current += 1;
+    setUpload(null);
+    if (state.fileChooser) send({ type: 'file_cancel', chooser: state.fileChooser.id });
+  };
+
+  const chooserOpen = mine && Boolean(state.fileChooser);
+  const drop = chooserOpen ? {
+    onDragOver: (event) => {
+      event.preventDefault();
+      event.dataTransfer.dropEffect = 'copy';
+    },
+    onDrop: (event) => {
+      event.preventDefault();
+      if (!upload) void uploadFiles(event.dataTransfer.files);
+    },
+  } : {};
+
   // Focus goes to the frame when this view takes control, and leaves it when
   // control is handed back or taken away.
   useEffect(() => {
@@ -262,6 +382,7 @@ export default function LiveView({ sessionId, headed = false, watchOnOpen = fals
   const holder = controlLabel(state);
   const takeOverAction = takeOverButton(state, { taking });
   const page = pageLabel(state.page);
+  const waitingForFile = fileChooserWaitLabel(state);
   const open = connection === 'open';
   const { frameSize } = state;
 
@@ -299,15 +420,26 @@ export default function LiveView({ sessionId, headed = false, watchOnOpen = fals
       {connection !== 'idle' && (
         <div className="space-y-1">
           {page && <p className={`text-xs font-mono truncate ${muted}`} title={page}>{page}</p>}
-          <canvas
-            ref={canvasRef}
-            tabIndex={mine ? 0 : -1}
-            role={mine ? 'application' : 'img'}
-            aria-label={mine ? "The sandbox's browser: your mouse and keys go to it. Press Escape twice to leave it." : "Live view of the sandbox's browser"}
-            className={`block w-full rounded border bg-black ${mine ? 'cursor-default outline-none focus:ring-2 focus:ring-blue-500' : ''} ${darkMode ? 'border-gray-700' : 'border-gray-200'}`}
-            style={{ aspectRatio: frameSize ? `${frameSize.width} / ${frameSize.height}` : '16 / 10' }}
-            {...pointer}
-          />
+          {waitingForFile && <p className={`text-xs ${muted}`} role="status">{waitingForFile}</p>}
+          <div className="relative" data-testid="live-view-frame" {...drop}>
+            <canvas
+              ref={canvasRef}
+              tabIndex={mine ? 0 : -1}
+              role={mine ? 'application' : 'img'}
+              aria-label={mine ? "The sandbox's browser: your mouse and keys go to it. Press Escape twice to leave it." : "Live view of the sandbox's browser"}
+              className={`block w-full rounded border bg-black ${mine ? 'cursor-default outline-none focus:ring-2 focus:ring-blue-500' : ''} ${darkMode ? 'border-gray-700' : 'border-gray-200'}`}
+              style={{ aspectRatio: frameSize ? `${frameSize.width} / ${frameSize.height}` : '16 / 10' }}
+              {...pointer}
+            />
+            {chooserOpen && (
+              <FileChooserPrompt
+                chooser={state.fileChooser}
+                upload={upload}
+                onFiles={(files) => void uploadFiles(files)}
+                onCancel={cancelChooser}
+              />
+            )}
+          </div>
           {mine && (
             <p className={`text-xs ${muted}`}>
               Your mouse and keys go to the browser, and the agent waits until you hand back. Press Escape twice to move focus

actionagent/frontend/components/dashboard/projects/ProjectBrowser.jsx

@@ -6,7 +6,7 @@ import { dashboardPath, navigateTo } from '../../../utils/dashboardPath';
 const RUNTIMES_PATH = '/settings?tab=integrations';
 
 // The project's Browser tab: its sandbox's browser (BrowserPanel), started,
-// stopped and watched live from here, with Take over, as on
+// stopped and watched live from here, with Take over and file uploads, as on
 // Settings -> Integrations, which still lists every runtime. `modes` is what
 // the backend can start; `watchOnOpen` starts watching once the live view
 // is there, for the header's Watch live; `onChange` follows a start or stop.
@@ -24,8 +24,8 @@ export default function ProjectBrowser({ project, modes = [], watchOnOpen = fals
     body = (
       <>
         <p style={note}>
-          A browser of the sandbox's own, opened on {project.repository}'s running app. Watch it live, take over to click and type,
-          and hand back. Agent runs reach it as <span style={{ fontFamily: MONO }}>browser:{sandbox.session_id}</span>.
+          A browser of the sandbox's own, opened on {project.repository}'s running app. Watch it live, take over to click, type and
+          upload files, and hand back. Agent runs reach it as <span style={{ fontFamily: MONO }}>browser:{sandbox.session_id}</span>.
         </p>
         <BrowserPanel key={sandbox.session_id} sandbox={sandbox} modes={modes} watchOnOpen={watchOnOpen} onChange={onChange} />
       </>

actionagent/frontend/test/liveView.test.mjs

@@ -6,6 +6,8 @@ import {
   browserStatus,
   closeMessage,
   controlLabel,
+  encodeChunk,
+  fileChooserWaitLabel,
   framePoint,
   initialLiveViewState,
   isBrowserActive,
@@ -19,6 +21,9 @@ import {
   releaseMessage,
   takeOverButton,
   textMessage,
+  UPLOAD_CHUNK_BYTES,
+  uploadPlan,
+  uploadProgressLabel,
   wheelMessage,
 } from '../utils/liveView.mjs';
 
@@ -176,3 +181,46 @@ test("a sandbox browser's status, and how one is started", () => {
   assert.deepEqual(browserStartBody({ inWindow: true, modes: ['headless'] }), { mode: 'headless' });
   assert.deepEqual(browserStartBody(), { mode: 'headless' });
 });
+
+const chooser = { id: 'c1', multiple: false, accept: '.txt', since: 1, max_files: 10, max_file_bytes: 1024 };
+
+test('the file chooser the page opened is followed from ready to answered', () => {
+  let state = liveViewReducer(initialLiveViewState, { type: 'ready', control: {}, agent: null, page: null, file_chooser: chooser });
+  assert.deepEqual(state.fileChooser, chooser);
+  state = liveViewReducer(state, { type: 'file_chooser', chooser: null });
+  assert.equal(state.fileChooser, null);
+  state = liveViewReducer(state, { type: 'file_chooser', chooser });
+  assert.equal(liveViewReducer(state, { type: 'file_upload', status: 'progress', received: 1, total: 2 }), state);
+  assert.equal(liveViewReducer(state, { type: 'file_upload', status: 'failed', code: 'too_large', message: 'a.txt is larger than 1 KB' }).error, 'a.txt is larger than 1 KB');
+  assert.equal(liveViewReducer(state, { type: 'reset' }).fileChooser, null);
+});
+
+test('chosen files are checked against what the page and the sidecar take before any is sent', () => {
+  const file = (name, size) => ({ name, size });
+  assert.deepEqual(uploadPlan([file('a.txt', 3)], chooser), { files: [{ name: 'a.txt', size: 3 }], total: 3 });
+  assert.equal(uploadPlan([file('a.txt', 3), file('b.txt', 3)], chooser).error, 'This field takes one file.');
+  assert.deepEqual(uploadPlan([file('a.txt', 3), file('b.txt', 4)], { ...chooser, multiple: true }).total, 7);
+  assert.equal(uploadPlan([file('a', 1), file('b', 1)], { ...chooser, multiple: true, max_files: 1 }).error, 'This field takes one file.');
+  assert.equal(uploadPlan(Array.from({ length: 3 }, (_, i) => file(`${i}`, 1)), { ...chooser, multiple: true, max_files: 2 }).error, 'This field takes at most 2 files.');
+  assert.equal(uploadPlan([file('big.bin', 2048)], chooser).error, 'big.bin is larger than 1 KB.');
+  assert.equal(uploadPlan([], chooser).error, 'Choose a file to upload.');
+  assert.equal(uploadPlan([file('a', 1)], null).error, 'The page is no longer asking for a file.');
+});
+
+test('a chunk is base64 of its bytes, at the size the sidecar takes', () => {
+  assert.equal(UPLOAD_CHUNK_BYTES, 32 * 1024);
+  assert.equal(encodeChunk(new TextEncoder().encode('hello world')), 'aGVsbG8gd29ybGQ=');
+  const large = new Uint8Array(UPLOAD_CHUNK_BYTES).map((_, i) => i % 256);
+  assert.deepEqual(Uint8Array.from(atob(encodeChunk(large)), (c) => c.charCodeAt(0)), large);
+  assert.equal(uploadProgressLabel({ sent: 512, total: 2048 }), 'Uploading 2 KB… 25%');
+  assert.equal(uploadProgressLabel({ sent: 0, total: 0 }), 'Uploading 0 bytes… 100%');
+  assert.equal(uploadProgressLabel(null), null);
+});
+
+test('viewers who are not driving are told the page waits for a file', () => {
+  const waiting = { ...initialLiveViewState, fileChooser: chooser };
+  assert.equal(fileChooserWaitLabel({ ...waiting, control: { held: true, mine: false, yours: false, by: 'Ada' } }), 'The page is asking Ada for a file.');
+  assert.match(fileChooserWaitLabel({ ...waiting, control: { held: true, mine: false, yours: true, by: 'Ada' } }), /view you are driving from/);
+  assert.equal(fileChooserWaitLabel({ ...waiting, control: { held: true, mine: true, yours: true, by: 'Ada' } }), null);
+  assert.equal(fileChooserWaitLabel(initialLiveViewState), null);
+});

actionagent/frontend/test/liveViewViews.test.mjs

@@ -0,0 +1,71 @@
+import assert from 'node:assert/strict';
+import { mkdirSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import test, { after, before } from 'node:test';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { build } from 'esbuild';
+
+// The live view's file chooser prompt, bundled with esbuild and rendered to
+// static markup.
+
+const root = fileURLToPath(new URL('..', import.meta.url));
+const bundlePath = fileURLToPath(new URL(`../node_modules/.cache/live-view-views-${process.pid}.mjs`, import.meta.url));
+
+let render;
+
+before(async () => {
+  Object.defineProperty(globalThis, 'localStorage', { value: { getItem: () => 'light', setItem() {} }, configurable: true, writable: true });
+
+  const { outputFiles } = await build({
+    stdin: {
+      contents: `
+        import React from 'react';
+        import { renderToStaticMarkup } from 'react-dom/server';
+        import { ThemeProvider } from './contexts/ThemeContext.jsx';
+        import { FileChooserPrompt } from './components/dashboard/LiveView.jsx';
+
+        export const render = (props) =>
+          renderToStaticMarkup(React.createElement(ThemeProvider, null, React.createElement(FileChooserPrompt, props)));
+      `,
+      resolveDir: root,
+      loader: 'jsx',
+    },
+    bundle: true,
+    write: false,
+    format: 'esm',
+    platform: 'node',
+    external: ['react', 'react-dom', 'react-dom/server', '@rails/actioncable'],
+    logLevel: 'error',
+  });
+  mkdirSync(dirname(bundlePath), { recursive: true });
+  writeFileSync(bundlePath, outputFiles[0].text);
+  ({ render } = await import(pathToFileURL(bundlePath).href));
+});
+
+after(() => rmSync(bundlePath, { force: true }));
+
+const noop = () => {};
+const chooser = { id: 'c1', multiple: false, accept: '.pdf,.txt', since: 1, max_files: 10, max_file_bytes: 1024 };
+
+test('the person driving gets a file input that takes what the page takes', () => {
+  const html = render({ chooser, upload: null, onFiles: noop, onCancel: noop });
+  assert.match(html, /The page asked for a file/);
+  assert.match(html, /<input type="file" accept="\.pdf,\.txt"/);
+  assert.doesNotMatch(html, /multiple/);
+  assert.match(html, /The page takes \.pdf,\.txt/);
+  assert.match(html, />Cancel</);
+});
+
+test('a field that takes several files lets several be chosen', () => {
+  const html = render({ chooser: { ...chooser, multiple: true, accept: null }, upload: null, onFiles: noop, onCancel: noop });
+  assert.match(html, /The page asked for files/);
+  assert.match(html, /up to 10 files/);
+  assert.match(html, /multiple=""/);
+  assert.doesNotMatch(html, /accept=/);
+});
+
+test('while files are sent, the input gives way to the progress', () => {
+  const html = render({ chooser, upload: { sent: 512, total: 1024 }, onFiles: noop, onCancel: noop });
+  assert.match(html, /Uploading 1 KB… 50%/);
+  assert.doesNotMatch(html, /type="file"/);
+});