Add channel exports with resumable one-way sync

README.md

@@ -60,18 +60,18 @@ arena channel contents --help
 
 ## Global Flags
 
-| Flag      | Description                                     |
-| --------- | ----------------------------------------------- |
-| `--json`  | Output as JSON (`import --json` streams NDJSON) |
-| `--quiet` | Compact JSON output when supported              |
-| `--yes`   | Bypass destructive confirmation prompts         |
-| `--help`  | Show help                                       |
+| Flag      | Description                                               |
+| --------- | --------------------------------------------------------- |
+| `--json`  | Output as JSON (`import`/`download --json` stream NDJSON) |
+| `--quiet` | Compact JSON output when supported                        |
+| `--yes`   | Bypass destructive confirmation prompts                   |
+| `--help`  | Show help                                                 |
 
 ## Output & Errors
 
 - `stdout`:
   - Successful command output.
-  - `--json` returns JSON objects; `import --json` returns NDJSON progress events.
+  - `--json` returns JSON objects; `import --json` and `download --json` return NDJSON progress events.
   - `--quiet` only changes JSON formatting (compact, single-line); it does **not** change response fields.
 - `stderr`:
   - Human-readable failures in non-interactive mode.
@@ -440,6 +440,43 @@ Options:
 - `--upload-concurrency <n>`
 - `--poll-interval <ms>`
 
+#### `download` (aliases: `export`, `sync`)
+
+Download a channel to a local directory. Run the same command again to sync it from Are.na.
+
+```bash
+arena download worldmaking ./refs
+arena sync worldmaking ./refs
+arena sync worldmaking ./refs --prune
+arena export worldmaking ./refs --zip ./worldmaking.zip
+arena download worldmaking --type Image --size large --json
+```
+
+Each export contains:
+
+- Images and attachments, `.txt` files of Markdown for text, and `.webloc` shortcuts for links, embeds, and connected channels. Files are named like Are.na's web channel export: `<position>_<title>.<extension>` (for example `3_brutalist-housing.jpg`). When blocks are reordered, removed, or retitled, sync renames the existing files instead of downloading them again.
+- `<channel-slug>.csv`, with the same columns as the web export: ID, filename, title, description, created and updated dates, and source URL.
+- `index.html`, an offline index in channel order with images, text, descriptions, and links to each item's source and Are.na page.
+- `manifest.json`, a snapshot of the channel and its items (including connection positions and metadata) plus a SHA-256 checksum for every file. Keep it with the directory: it is the sync state.
+
+Sync is **one way, from Are.na**. Unchanged files are skipped without a request, changed or missing files are downloaded again, and a file you edited locally is reported as a conflict instead of being overwritten. Files that leave the export, such as removed blocks, stay on disk until you pass `--prune`, which deletes only files the export created and you haven't edited. If another block now needs a kept file's name, the kept file is renamed to `<name>~block-<id>.<ext>`. Pruning is skipped if any download fails or the channel changes during the sync.
+
+Options:
+
+- `--dir <path>` — destination, instead of the second argument; defaults to the channel slug
+- `--size <original|large|medium|small|square>` — image size; default `original`
+- `--type <Image|Attachment|Text|Link|Embed|Channel|PendingBlock>` — only write files of one type (the manifest still lists every item); cannot be combined with `--prune`
+- `--link-format <webloc|url>` — macOS (`webloc`, default) or Windows/Linux (`url`) shortcuts
+- `--concurrency <n>` — concurrent transfers, 1–16; default `4`
+- `--prune` — remove files from earlier syncs that are no longer part of the export
+- `--overwrite` — download everything again and replace local edits, or replace a non-ZIP file at the `--zip` path
+- `--zip <path>` — also write the export to a ZIP; an existing ZIP at that path is replaced
+- `--json` — stream NDJSON events; the final `completed` event has `downloaded`, `unchanged`, `renamed`, `failed`, `pruned`, `retained`, and `cancelled`
+
+Exit code is `1` if anything failed and `130` if cancelled. Failed items are recorded in the manifest and left out of the ZIP. Ctrl+C keeps finished files, and the next run picks up where it left off; a lock, temporary files, or renames left unfinished by a killed process are cleaned up or completed automatically. A transfer is retried if no data arrives for `ARENA_REQUEST_TIMEOUT_MS` (default 30s).
+
+Private channels use your saved login. Connected channels are saved as shortcuts, not exported recursively; embeds are saved as source shortcuts, not videos; blocks still processing are recorded in the manifest only. Changes are detected from asset URLs and revision timestamps, so use `--overwrite` if a file changed upstream without either.
+
 ### Connections
 
 #### `connect`

package.json

@@ -40,14 +40,16 @@
     "openapi-fetch": "^0.17.0",
     "react": "^19.0.0",
     "swr": "^2.4.1",
-    "terminal-image": "^4.2.0"
+    "terminal-image": "^4.2.0",
+    "yazl": "^3.3.1"
   },
   "overrides": {
     "simple-xml-to-json": "1.2.3"
   },
   "devDependencies": {
     "@types/node": "^25.3.3",
     "@types/react": "^19.0.0",
+    "@types/yazl": "^3.3.1",
     "husky": "^9.1.7",
     "lint-staged": "^16.3.2",
     "openapi-typescript": "^7.13.0",

src/cli.contract.test.ts

@@ -80,3 +80,20 @@ test("add command reads value from piped stdin", () => {
     `expected stdin to supply the value, got: ${output}`,
   );
 });
+
+test("download aliases share help and reject invalid options before requesting the API", () => {
+  for (const command of ["download", "export", "sync"]) {
+    const help = runCli([command, "--help"]);
+    assert.equal(help.status, 0);
+    assert.match(help.stdout, /--prune/);
+    const invalid = runCli([
+      command,
+      "channel",
+      "--concurrency",
+      "17",
+      "--json",
+    ]);
+    assert.equal(invalid.status, 1);
+    assert.match(JSON.parse(invalid.stderr).error, /concurrency/);
+  }
+});

src/cli.tsx

@@ -53,6 +53,7 @@ function TopLevelHelp() {
         <Text> add Add text/URLs to a channel</Text>
         <Text> upload Upload local files</Text>
         <Text> import Bulk import from a directory</Text>
+        <Text> download Download or sync a channel to disk</Text>
       </Box>
 
       <Box flexDirection="column" marginBottom={1}>
@@ -66,7 +67,7 @@ function TopLevelHelp() {
 
       <Box flexDirection="column" marginBottom={1}>
         <Text dimColor>Global Flags</Text>
-        <Text> --json Output JSON (import streams NDJSON events)</Text>
+        <Text> --json Output JSON (import/download stream NDJSON events)</Text>
         <Text> --quiet Compact JSON output when supported</Text>
         <Text> --yes Bypass destructive confirmation prompts</Text>
         <Text> --version Show CLI version</Text>

src/commands/download.tsx

@@ -0,0 +1,201 @@
+import React, { useEffect, useState } from "react";
+import { join } from "node:path";
+import { Box, Text, useApp } from "ink";
+import { flag, intFlag, requireArg, type Flags } from "../lib/args";
+import { ScreenFrame } from "../components/ScreenChrome";
+import { Spinner } from "../components/Spinner";
+import { cancellationSignal } from "../lib/network";
+import { exitCodeFromError } from "../lib/exit-codes";
+import { plural, progressBar, truncate } from "../lib/format";
+import { DOWNLOAD_TYPES, IMAGE_SIZES } from "../lib/download-format";
+import {
+  executeDownload,
+  downloadExitCode,
+  validateDownloadOptions,
+  type DownloadOptions,
+  type DownloadSummary,
+  type DownloadEvent,
+} from "../lib/download";
+
+function valueFlag(flags: Flags, name: string): string | undefined {
+  if (
+    flags[name] !== undefined &&
+    (typeof flags[name] !== "string" || !String(flags[name]).trim())
+  )
+    throw new Error(`--${name} requires a value`);
+  return flag(flags, name);
+}
+function booleanFlag(flags: Flags, name: string): boolean {
+  const value = flags[name];
+  if (value === undefined || value === false || value === "false") return false;
+  if (value === true || value === "true") return true;
+  throw new Error(`--${name} expects true or false`);
+}
+function choiceFlag<T extends string>(
+  flags: Flags,
+  name: string,
+  choices: readonly T[],
+): T | undefined {
+  const raw = valueFlag(flags, name);
+  if (raw === undefined) return undefined;
+  const choice = choices.find(
+    (value) => value.toLowerCase() === raw.toLowerCase(),
+  );
+  if (!choice)
+    throw new Error(`--${name} must be one of: ${choices.join(", ")}`);
+  return choice;
+}
+export function parseDownloadOptions(
+  args: string[],
+  flags: Flags,
+): DownloadOptions {
+  if (args.length > 2)
+    throw new Error("Usage: arena download <channel> [directory] [flags]");
+  const directory = valueFlag(flags, "dir");
+  if (directory && args[1])
+    throw new Error("Specify a directory either positionally or with --dir");
+  const options: DownloadOptions = {
+    channel: requireArg(args, 0, "channel"),
+    directory: directory ?? args[1],
+    size: choiceFlag(flags, "size", IMAGE_SIZES) ?? "original",
+    type: choiceFlag(flags, "type", DOWNLOAD_TYPES),
+    linkFormat:
+      choiceFlag(flags, "link-format", ["webloc", "url"] as const) ?? "webloc",
+    concurrency: intFlag(flags, "concurrency") ?? 4,
+    overwrite: booleanFlag(flags, "overwrite"),
+    prune: booleanFlag(flags, "prune"),
+    zip: valueFlag(flags, "zip"),
+  };
+  validateDownloadOptions(options);
+  return options;
+}
+
+interface Progress {
+  label: string;
+  completed: number;
+  total: number;
+}
+
+export function DownloadCommand(options: DownloadOptions) {
+  const { exit } = useApp();
+  const [progress, setProgress] = useState<Progress>({
+    label: "Listing channel contents",
+    completed: 0,
+    total: 0,
+  });
+  const [summary, setSummary] = useState<DownloadSummary | null>(null);
+  const [error, setError] = useState<string | null>(null);
+  useEffect(() => {
+    const controller = new AbortController();
+    const signal = AbortSignal.any([cancellationSignal(), controller.signal]);
+    let mounted = true;
+    void executeDownload({
+      ...options,
+      signal,
+      onEvent(event) {
+        if (!mounted) return;
+        if (event.type === "list_progress")
+          setProgress({
+            label: `Listing ${event.fetched}/${event.total} items`,
+            completed: 0,
+            total: 0,
+          });
+        if (event.type === "file_progress")
+          setProgress({
+            label: `${event.status} · ${truncate(event.file, 64)}`,
+            completed: event.completed,
+            total: event.total,
+          });
+        if (event.type === "archive_started")
+          setProgress((current) => ({
+            ...current,
+            label: "Creating ZIP archive",
+          }));
+      },
+    })
+      .then((result) => {
+        if (!mounted) return;
+        process.exitCode = downloadExitCode(result);
+        setSummary(result);
+      })
+      .catch((failure: unknown) => {
+        if (!mounted) return;
+        process.exitCode = signal.aborted ? 130 : exitCodeFromError(failure);
+        setError(failure instanceof Error ? failure.message : String(failure));
+      });
+    return () => {
+      mounted = false;
+      controller.abort();
+    };
+  }, []);
+  useEffect(() => {
+    if (summary || error) exit();
+  }, [summary, error, exit]);
+  if (error) return <Text color="red">✕ {error}</Text>;
+  if (!summary) {
+    if (progress.total === 0) return <Spinner label={progress.label} />;
+    return (
+      <Box flexDirection="column">
+        <Text>
+          {progress.completed}/{progress.total}{" "}
+          {progressBar(progress.completed, progress.total)}
+        </Text>
+        <Spinner label={progress.label} />
+      </Box>
+    );
+  }
+  const ok = downloadExitCode(summary) === 0;
+  const details = [
+    summary.metadata_only ? `${summary.metadata_only} metadata only` : null,
+    summary.retained
+      ? options.type
+        ? `${plural(summary.retained, "earlier file")} kept outside this filtered export`
+        : `${plural(summary.retained, "earlier file")} not in this export (remove with --prune)`
+      : null,
+  ].filter(Boolean);
+  return (
+    <ScreenFrame title={`download ${options.channel}`}>
+      <Box flexDirection="column">
+        <Text color={ok ? "green" : "yellow"}>
+          {ok ? "✓" : "!"}{" "}
+          {summary.cancelled
+            ? "Sync cancelled"
+            : summary.failed
+              ? "Sync finished with errors"
+              : "Sync complete"}
+        </Text>
+        <Text dimColor>
+          {summary.downloaded} downloaded · {summary.unchanged} unchanged ·{" "}
+          {summary.renamed ? `${summary.renamed} renamed · ` : ""}
+          {summary.failed} failed
+          {options.prune ? ` · ${summary.pruned} pruned` : ""}
+        </Text>
+        {details.length ? <Text dimColor>{details.join(" · ")}</Text> : null}
+        <Text>→ {join(summary.directory, "index.html")}</Text>
+        {summary.zip ? <Text>→ {summary.zip}</Text> : null}
+        {summary.failures.slice(0, 5).map((failure, index) => (
+          <Text color="red" key={index}>
+            {truncate(failure.file, 64)} · {failure.error}
+          </Text>
+        ))}
+        {summary.failures.length > 5 ? (
+          <Text dimColor>…and {summary.failures.length - 5} more</Text>
+        ) : null}
+      </Box>
+    </ScreenFrame>
+  );
+}
+export async function runDownloadJsonStream(
+  options: DownloadOptions,
+  write: (event: DownloadEvent) => void,
+): Promise<number> {
+  const signal = cancellationSignal();
+  try {
+    return downloadExitCode(
+      await executeDownload({ ...options, signal, onEvent: write }),
+    );
+  } catch (error) {
+    if (signal.aborted) return 130;
+    throw error;
+  }
+}