From 3692938856735d5d7a926c773dacc164b81b0b5e Mon Sep 17 00:00:00 2001 From: anshag01 <104688627+anshag01@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:54:20 -0400 Subject: [PATCH 01/15] Document local startup and require a compatible Node runtime --- README.md | 52 ++++++++++++++++++++++++++++++++++---- web-interface/package.json | 3 +++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a1e34d1..684da20 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,52 @@ Mist: UTMIST's Compute Platform -## Runtime Requirements +## Runtime requirements -Note that you will need the following installed: +- Go 1.25.1 (also satisfies the backend's Go 1.24.6 requirement) +- Node.js 22.16.0 or newer, with npm +- Docker Engine and the Docker Compose plugin (`docker compose`) +- Permission to run `docker ps` as the same user that runs the backend -- docker -- docker-compose -- go +## Run locally + +Ensure `go version`, `node --version`, and `docker ps` work before starting. +Run the following from the repository root: + +```bash +docker compose up -d redis +docker build -t pytorch-cpu src/docker/pytorch-cpu +``` + +Start the backend in one terminal. Run it from `src` so it finds its logging configuration: + +```bash +cd src +go run . +``` + +Start the frontend in another terminal: + +```bash +cd web-interface +npm install --package-lock=false +npm run dev -- --port 3001 +``` + +Open http://localhost:3001/jobs. The API uses http://localhost:3000. +The repository includes a Bun lockfile; npm installation does not use that lockfile. + +The optional CLI can be built from a third terminal: + +```bash +cd cli +go build -o ../bin/mist . +../bin/mist --help +``` + +The frontend and CLI currently contain mock data and placeholder actions. The +backend's job execution is also incomplete; starting these services does not yet +provide end-to-end workload execution. + +Stop the backend/frontend with Ctrl+C in their terminals, then run +`docker compose stop redis` from the repository root to stop Redis without deleting its data. diff --git a/web-interface/package.json b/web-interface/package.json index b9e0451..aed11e6 100644 --- a/web-interface/package.json +++ b/web-interface/package.json @@ -2,6 +2,9 @@ "name": "web-interface", "private": true, "type": "module", + "engines": { + "node": ">=22.16.0" + }, "imports": { "#/*": "./src/*" }, From 2d9ff9ddf41a641090c8c6db0fa7586fae617be8 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 20:28:32 -0400 Subject: [PATCH 02/15] Document k3s GPU pilot and execution plan --- .gitignore | 1 + README.md | 3 + deploy/k3s/README.md | 47 +++++ deploy/k3s/gpu-smoke-job.yaml | 28 +++ deploy/k3s/nvidia-gpu-operator-values.yaml | 10 + deploy/k3s/setup-runbook.md | 203 +++++++++++++++++++++ docs/kubernetes-pilot.md | 71 +++++++ web-interface/src/routeTree.gen.ts | 62 +++---- 8 files changed, 394 insertions(+), 31 deletions(-) create mode 100644 deploy/k3s/README.md create mode 100644 deploy/k3s/gpu-smoke-job.yaml create mode 100644 deploy/k3s/nvidia-gpu-operator-values.yaml create mode 100644 deploy/k3s/setup-runbook.md create mode 100644 docs/kubernetes-pilot.md diff --git a/.gitignore b/.gitignore index e660fd9..86c9b92 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ bin/ +/logs/ diff --git a/README.md b/README.md index 684da20..e785670 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ Mist: UTMIST's Compute Platform +For the proposed Kubernetes execution pilot and migration steps, see +[docs/kubernetes-pilot.md](docs/kubernetes-pilot.md). + ## Runtime requirements - Go 1.25.1 (also satisfies the backend's Go 1.24.6 requirement) diff --git a/deploy/k3s/README.md b/deploy/k3s/README.md new file mode 100644 index 0000000..f632d99 --- /dev/null +++ b/deploy/k3s/README.md @@ -0,0 +1,47 @@ +# NVIDIA GPU smoke test on k3s + +For the full as-built setup, recovery procedure, verification evidence, and +handoff notes, read the [single-node setup runbook](setup-runbook.md). + +This pilot node runs Ubuntu 22.04, k3s 1.36, and two RTX A4000 GPUs. The host +driver is pre-installed. NVIDIA GPU Operator v26.7.1 is configured with +[`nvidia-gpu-operator-values.yaml`](nvidia-gpu-operator-values.yaml). Pin the +chart version when installing or upgrading: + +```bash +sudo env KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm upgrade --install gpu-operator \ + oci://nvcr.io/nvidia/cloud-native-charts/gpu-operator \ + --version v26.7.1 --namespace gpu-operator --create-namespace \ + --values deploy/k3s/nvidia-gpu-operator-values.yaml --wait +``` + +Run from the repository root. Check that the operator pods are healthy and the +node advertises two GPUs before submitting a workload: + +```bash +sudo k3s kubectl -n gpu-operator get pods +sudo k3s kubectl describe node utmist-z1opa08 +``` + +The `Allocatable` section must include `nvidia.com/gpu: 2`. Then run the +single-GPU CUDA smoke test: + +```bash +sudo k3s kubectl create namespace mist +sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml +sudo k3s kubectl -n mist wait --for=condition=complete job/gpu-smoke --timeout=180s +sudo k3s kubectl -n mist logs job/gpu-smoke +``` + +The expected log contains `Test PASSED`. To rerun the Job, delete only the +finished smoke Job and apply the manifest again: + +```bash +sudo k3s kubectl -n mist delete job gpu-smoke +sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml +``` + +No Service or NodePort is needed for a batch Job. This test verifies that k3s +can allocate an NVIDIA GPU and execute CUDA; it does not connect Mist's API to +Kubernetes yet. See [the execution pilot](../../docs/kubernetes-pilot.md) for +that application work. diff --git a/deploy/k3s/gpu-smoke-job.yaml b/deploy/k3s/gpu-smoke-job.yaml new file mode 100644 index 0000000..0aa711a --- /dev/null +++ b/deploy/k3s/gpu-smoke-job.yaml @@ -0,0 +1,28 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: gpu-smoke + namespace: mist + labels: + app.kubernetes.io/name: mist-gpu-smoke +spec: + backoffLimit: 0 + activeDeadlineSeconds: 600 + template: + metadata: + labels: + app.kubernetes.io/name: mist-gpu-smoke + spec: + restartPolicy: Never + containers: + - name: vectoradd + image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0 + resources: + requests: + cpu: 250m + memory: 256Mi + nvidia.com/gpu: 1 + limits: + cpu: '1' + memory: 512Mi + nvidia.com/gpu: 1 diff --git a/deploy/k3s/nvidia-gpu-operator-values.yaml b/deploy/k3s/nvidia-gpu-operator-values.yaml new file mode 100644 index 0000000..5d897e5 --- /dev/null +++ b/deploy/k3s/nvidia-gpu-operator-values.yaml @@ -0,0 +1,10 @@ +# This node already has a working NVIDIA host driver. Let the operator install +# the container toolkit and device plugin without replacing that driver. +driver: + enabled: false + +# CDI handles allocated workload GPUs. Enable the NRI plugin for GPU management +# containers without changing k3s's containerd runtime configuration. This is +# intended for the pilot; NVIDIA documents the NRI plugin as not yet GA. +cdi: + nriPluginEnabled: true diff --git a/deploy/k3s/setup-runbook.md b/deploy/k3s/setup-runbook.md new file mode 100644 index 0000000..1ae5ea4 --- /dev/null +++ b/deploy/k3s/setup-runbook.md @@ -0,0 +1,203 @@ +# Single-node k3s and NVIDIA GPU pilot: setup and handoff + +This runbook records the configuration observed on **2026-09-28**, a repeatable +setup and validation procedure, and the failure encountered during setup. Run +commands from the Mist repository root unless noted. The +[Kubernetes execution pilot](../../docs/kubernetes-pilot.md) describes the +separate work needed to connect Mist's API to Kubernetes Jobs. + +## What the pieces do + +The machine already had two NVIDIA GPUs and a working host driver. A **single +k3s server** was installed on it. A server includes a local kubelet and +container runtime and can run Jobs without a separate `k3s-agent` service. +The NVIDIA GPU Operator added the Kubernetes integration: + +1. Its **device plugin** registers `nvidia.com/gpu` with the kubelet, allowing + the scheduler to reserve a GPU for a Pod that requests one. +2. Its **container toolkit** prepares CDI device information that containerd + uses to expose the assigned GPU and host driver libraries to a workload. + The enabled NRI plugin handles GPU management containers and avoids + rewriting k3s's containerd runtime configuration. +3. A CUDA sample Job requests `nvidia.com/gpu: 1` and exercises the device. + +The operator was configured **not** to replace the existing host driver. Those +values are in +[`nvidia-gpu-operator-values.yaml`](nvidia-gpu-operator-values.yaml). NVIDIA +states that its NRI plugin is not yet generally available; this is a pilot +configuration. See the [GPU Operator installation guide](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) +and [CDI/NRI guide](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/cdi.html). + +## Observed state on 2026-09-28 + +| Item | Observed value | +| --- | --- | +| Node | `utmist-z1opa08`, one control-plane and worker node | +| OS | Ubuntu 22.04.5 LTS | +| k3s | `v1.36.4+k3s1`; `k3s.service` enabled and active | +| Container runtime | k3s containerd `2.3.4-k3s1.36` | +| GPUs | Two NVIDIA RTX A4000 cards | +| Host driver | NVIDIA `590.48.01`, already present before GPU Operator deployment | +| GPU Operator | Helm release `gpu-operator`, chart `v26.7.1`, namespace `gpu-operator` | +| Test namespace | `mist` | +| Node resource | Capacity and allocatable: `nvidia.com/gpu: 2` | + +The node registered at 19:23 EDT. GPU Operator resources appeared around 19:39 +EDT; Helm release revision 2 was deployed at 19:45 EDT. The `gpu-smoke` Job +completed at about 19:44 EDT. After the service fix, a new `gpu-confirm` Job +completed at 19:54 EDT. Both sample logs contained `Test PASSED`. Each Job +requested **one** GPU. The node advertised two, but these tests did not +independently identify or exercise each physical card. + +These times and versions come from systemd, Kubernetes object timestamps, +`helm list`, `nvidia-smi`, and the completed Job logs. The exact historical +k3s installation command was not retained, so the commands below are a +reproduction recipe rather than a transcript. + +## Reproduce or verify this pilot + +Check versions against the target host before rebuilding or upgrading. Have a +working NVIDIA driver and `helm` available. Never commit the kubeconfig, +cluster token, or private keys to this repository. + +### 1. Confirm the GPUs work on the host + +```bash +nvidia-smi -L +nvidia-smi --query-gpu=name,driver_version --format=csv,noheader +``` + +On this node the first command lists two RTX A4000 cards. Resolve host driver +or hardware failures before diagnosing Kubernetes GPU allocation. + +### 2. Run one k3s server + +For a fresh **single-node** install, pin the version used in this pilot: + +```bash +curl -sfL https://get.k3s.io | sudo env INSTALL_K3S_VERSION='v1.36.4+k3s1' sh -s - server +sudo systemctl is-active k3s +sudo k3s kubectl get nodes -o wide +``` + +Expect `active` and a `Ready` node. The installer writes an administrative +kubeconfig at `/etc/rancher/k3s/k3s.yaml`; `sudo k3s kubectl` uses it without +copying credentials. Any copied kubeconfig must be kept private. The +[k3s quick-start guide](https://docs.k3s.io/quick-start) explains that a +single server can host workloads and that agents join additional machines. + +This same host must **not** run a separate `k3s-agent` service: + +```bash +systemctl is-active k3s-agent +systemctl is-enabled k3s-agent +``` + +Expected outputs here are `inactive` and `disabled`. These commands can +return nonzero for those expected states. + +### 3. Install the GPU Operator + +The tracked Helm values disable Operator driver installation and enable its +NRI plugin for GPU management containers. Keep the chart version pinned when +reproducing this setup: + +```bash +sudo env KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm upgrade --install gpu-operator \ + oci://nvcr.io/nvidia/cloud-native-charts/gpu-operator \ + --version v26.7.1 --namespace gpu-operator --create-namespace \ + --values deploy/k3s/nvidia-gpu-operator-values.yaml --wait + +sudo k3s kubectl -n gpu-operator get pods +sudo k3s kubectl describe node utmist-z1opa08 +``` + +The device plugin and container toolkit Pods should be Running. In the node's +`Capacity` and `Allocatable` sections, expect `nvidia.com/gpu: 2` for this +hardware. A completed validator Pod is normal. The operator may need a little +time to register devices after k3s starts; check again after its Pods are +healthy. + +### 4. Execute CUDA through k3s + +The tracked [`gpu-smoke-job.yaml`](gpu-smoke-job.yaml) requests and limits one +`nvidia.com/gpu` device. Create the namespace if it does not already exist: + +```bash +sudo k3s kubectl create namespace mist +sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml +sudo k3s kubectl -n mist wait --for=condition=complete job/gpu-smoke --timeout=180s +sudo k3s kubectl -n mist describe job gpu-smoke +sudo k3s kubectl -n mist logs job/gpu-smoke +``` + +Expect `1 Succeeded`, `nvidia.com/gpu: 1` in the Job's requests and limits, +and `Test PASSED` in the logs. This establishes that k3s scheduled a GPU Pod, +the runtime exposed a device inside the container, and CUDA executed. Merely +seeing the host GPUs or the Kubernetes resource count is a weaker check. + +A completed Job does not rerun on `kubectl apply`. To repeat this test, delete +**only the completed smoke Job**, then apply the manifest again: + +```bash +sudo k3s kubectl -n mist delete job gpu-smoke +sudo k3s kubectl apply -f deploy/k3s/gpu-smoke-job.yaml +sudo k3s kubectl -n mist wait --for=condition=complete job/gpu-smoke --timeout=180s +sudo k3s kubectl -n mist logs job/gpu-smoke +``` + +Save logs first if they are needed for an incident review. This batch Job +does not need a Service or NodePort. + +## Failure encountered: duplicate agent on port 6444 + +Both `k3s.service` and `k3s-agent.service` had been enabled on the **same +host**. The agent listened on `127.0.0.1:6444`, which the server's API process +also needed. The server repeatedly exited with: + +```text +failed to listen on 127.0.0.1:6444: bind: address already in use +``` + +During the conflict, `kubectl` could not connect to `127.0.0.1:6443`. +`nvidia-smi` still saw the GPUs, but new Kubernetes workloads could not use +them because the control plane was unavailable. Agent logs also referenced +`https://myserver:6443`, an example server name rather than this node's +working endpoint. The fix on 2026-09-28 was: + +```bash +sudo systemctl stop k3s-agent.service +sudo systemctl disable k3s-agent.service +sudo systemctl is-active k3s +sudo k3s kubectl get nodes +``` + +The server recovered without replacing the host driver or reinstalling the +operator. If it does not recover, inspect logs before restarting it, +especially when other workloads are running: + +```bash +systemctl status k3s k3s-agent --no-pager +journalctl -u k3s -u k3s-agent -n 100 --no-pager +ss -ltn '( sport = :6443 or sport = :6444 )' +``` + +## Troubleshooting and limits + +- **API connection refused:** Check `k3s.service` and its journal first. On + this node, confirm the duplicate agent has not been enabled again. +- **Node Ready but no `nvidia.com/gpu`:** Check `nvidia-smi -L`, the device + plugin and toolkit Pods, and + `sudo k3s kubectl -n gpu-operator logs daemonset/nvidia-device-plugin-daemonset --tail=100`. + Compare live Helm values with the tracked file if the release has changed. +- **GPU Job Pending:** Use `sudo k3s kubectl -n mist describe pod ` + to inspect scheduler events and requested resources. Two GPUs are the + observed capacity, not a promise that both are free at every moment. +- **Pod starts but CUDA fails:** Inspect its logs, host `nvidia-smi`, toolkit + Pod logs, and device plugin logs. An advertised resource alone does not + prove CUDA works. + +The smoke test proves **one requested GPU worked through k3s on the test +date**. It does not prove independent operation of both cards, GPU sharing, +performance, reboot durability, multi-node behavior, or Mist API integration. +See [the pilot plan](../../docs/kubernetes-pilot.md) for application work. diff --git a/docs/kubernetes-pilot.md b/docs/kubernetes-pilot.md new file mode 100644 index 0000000..d673648 --- /dev/null +++ b/docs/kubernetes-pilot.md @@ -0,0 +1,71 @@ +# Kubernetes execution pilot + +## Decision + +Pilot **k3s on one disposable CPU node or VM** and use Kubernetes `batch/v1` Jobs as Mist's execution primitive. Keep the existing Docker/Redis development path working while the pilot is built. Do not install k3s on the shared Tenstorrent box until a CPU job has completed through Mist and the device integration has been validated on that box. + +A separate single-node NVIDIA GPU hardware pilot now exists. Its +[setup runbook](../deploy/k3s/setup-runbook.md) records GPU scheduling and CUDA +validation; it does not complete the first CPU job path through Mist described +below. + +Kubernetes should own pod placement, resource accounting, restarts, and Job lifecycle. Mist should own authentication, job admission, user/team policy, priority, credits, and the user-facing API. A second Mist scheduler competing with Kubernetes for node placement would make both systems harder to reason about. + +```mermaid +flowchart LR + User[Web UI / CLI] --> API[Mist API] + API --> Policy[Auth, team policy, credits] + Policy --> Jobs[Kubernetes Jobs API] + Jobs --> CPU[CPU nodes] + Jobs --> TT[Tenstorrent nodes] + Jobs --> State[Job and Pod status / logs] + State --> API + Jobs --> Artifacts[Shared artifact storage] +``` + +## Why this is a pilot, not a deployment recipe + +The current `/jobs` endpoint enqueues a Redis message. The supervisor launches a CPU Docker container with `sleep 1000`, waits two seconds, then reports success; it does not run the submitted payload. For a GPU job it can report success without starting a container. All supervisors share one Redis consumer group, so a supervisor that reads an incompatible GPU job leaves it pending instead of offering it to the right worker. The web Jobs page and CLI still use sample data, and `/auth/login` is a placeholder. Installing k3s alone fixes none of these application paths. + +The first end-to-end slice should replace those behaviors for **CPU jobs**: + +1. Define a validated submission contract: image, command/arguments, CPU and memory limits, optional artifact location, owner, and maximum run time. Use an approved image list or registry policy before accepting arbitrary images from users. +2. Add a Kubernetes executor behind the Mist API. Create a namespaced `batch/v1` Job with `restartPolicy: Never`, explicit resource requests/limits, `backoffLimit`, `activeDeadlineSeconds`, and Mist job/owner labels. Give the API only namespaced permissions for the resources it uses. +3. Make submission, status, cancellation, and logs read the Kubernetes Job/Pod state. Persist the Mist job ID to Kubernetes Job name mapping and owner data independently of the Job TTL. Return a real failure when the workload fails. +4. Connect the CLI and Web UI to those endpoints. A user should be able to run a small CPU command and see its actual output and exit status. +5. Add namespace quotas, per-job limits, network restrictions, and durable audit/usage records before inviting pilot users. Treat priority and credits as Mist admission rules; map admitted priority to Kubernetes `PriorityClass` only after defining preemption policy. + +## Tenstorrent milestone + +Tenstorrent's `tt-operator` supports Wormhole and Blackhole devices. Its core stack needs Kubernetes 1.27+, and its Dynamic Resource Allocation (DRA) path needs **1.33+**. The operator also documents host kernel-header and registry requirements; its bundled PMIx webhook needs cert-manager unless disabled. Check the actual QuietBox generation, driver/firmware ownership, topology, kernel, and operator support before choosing chart values. Do not model a Tenstorrent device as a generic GPU string or assume that exposing `/dev/tenstorrent` is sufficient scheduling isolation. + +For a first TT workload: install and validate the vendor stack on a test node, verify that Kubernetes advertises the expected allocatable devices/claims, run one vendor example Job, then teach Mist to request that resource. Keep the existing host driver in place until the operator's driver management has been planned and tested. Multi-node jobs and topology-aware placement come later. + +## Storage and operations + +K3s includes a local-path storage provisioner, but its volumes are tied to a node. Use it only for disposable pilot data. Select shared storage or an object store for input datasets, logs, checkpoints, and output artifacts before adding more nodes. Keep retained job metadata and logs after Kubernetes Job cleanup. + +Start the API without public ingress and use `kubectl port-forward` for the pilot. Do not put the current placeholder authentication endpoint or a cluster administrator credential behind an internet-facing service. Deploy a dedicated Mist namespace and service account with the least permissions needed for Jobs, Pods, and logs. Back up the k3s datastore before any production use. + +## Pilot acceptance checks + +- One submitted CPU command runs to completion and returns its real exit code. +- A failing command is shown as failed, with retrievable logs. +- A canceled Job stops its pod and remains canceled in Mist's history. +- An incompatible hardware request stays pending or is rejected clearly; it is never reported as successful without execution. +- CPU/memory limits and namespace quota prevent a user from monopolizing the node. +- The Web UI and CLI show the same job state as the API. + +Only then move the pilot to a Tenstorrent node and add the vendor operator. + +The NVIDIA test node's [setup and incident runbook](../deploy/k3s/setup-runbook.md) +and [repeatable CUDA smoke Job](../deploy/k3s/README.md) record the current +hardware pilot. NVIDIA validation is separate from Tenstorrent device support. + +## References + +- [K3s installation requirements](https://docs.k3s.io/installation/requirements) +- [K3s packaged components and storage](https://docs.k3s.io/) +- [Kubernetes Job API](https://kubernetes.io/docs/reference/kubernetes-api/batch/job-v1/) +- [Tenstorrent operator platform support](https://docs.tenstorrent.com/tt-operator/latest/platform-support.html) +- [Tenstorrent operator prerequisites](https://docs.tenstorrent.com/tt-operator/latest/prerequisites.html) diff --git a/web-interface/src/routeTree.gen.ts b/web-interface/src/routeTree.gen.ts index 6391516..5b1d466 100644 --- a/web-interface/src/routeTree.gen.ts +++ b/web-interface/src/routeTree.gen.ts @@ -9,19 +9,14 @@ // Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified. import { Route as rootRouteImport } from './routes/__root' -import { Route as ProfileRouteImport } from './routes/profile' -import { Route as MachinesRouteImport } from './routes/machines' -import { Route as JobsRouteImport } from './routes/jobs' import { Route as DashboardRouteImport } from './routes/dashboard' +import { Route as JobsRouteImport } from './routes/jobs' +import { Route as MachinesRouteImport } from './routes/machines' +import { Route as ProfileRouteImport } from './routes/profile' -const ProfileRoute = ProfileRouteImport.update({ - id: '/profile', - path: '/profile', - getParentRoute: () => rootRouteImport, -} as any) -const MachinesRoute = MachinesRouteImport.update({ - id: '/machines', - path: '/machines', +const DashboardRoute = DashboardRouteImport.update({ + id: '/dashboard', + path: '/dashboard', getParentRoute: () => rootRouteImport, } as any) const JobsRoute = JobsRouteImport.update({ @@ -29,9 +24,14 @@ const JobsRoute = JobsRouteImport.update({ path: '/jobs', getParentRoute: () => rootRouteImport, } as any) -const DashboardRoute = DashboardRouteImport.update({ - id: '/dashboard', - path: '/dashboard', +const MachinesRoute = MachinesRouteImport.update({ + id: '/machines', + path: '/machines', + getParentRoute: () => rootRouteImport, +} as any) +const ProfileRoute = ProfileRouteImport.update({ + id: '/profile', + path: '/profile', getParentRoute: () => rootRouteImport, } as any) @@ -71,18 +71,11 @@ export interface RootRouteChildren { declare module '@tanstack/react-router' { interface FileRoutesByPath { - '/profile': { - id: '/profile' - path: '/profile' - fullPath: '/profile' - preLoaderRoute: typeof ProfileRouteImport - parentRoute: typeof rootRouteImport - } - '/machines': { - id: '/machines' - path: '/machines' - fullPath: '/machines' - preLoaderRoute: typeof MachinesRouteImport + '/dashboard': { + id: '/dashboard' + path: '/dashboard' + fullPath: '/dashboard' + preLoaderRoute: typeof DashboardRouteImport parentRoute: typeof rootRouteImport } '/jobs': { @@ -92,11 +85,18 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof JobsRouteImport parentRoute: typeof rootRouteImport } - '/dashboard': { - id: '/dashboard' - path: '/dashboard' - fullPath: '/dashboard' - preLoaderRoute: typeof DashboardRouteImport + '/machines': { + id: '/machines' + path: '/machines' + fullPath: '/machines' + preLoaderRoute: typeof MachinesRouteImport + parentRoute: typeof rootRouteImport + } + '/profile': { + id: '/profile' + path: '/profile' + fullPath: '/profile' + preLoaderRoute: typeof ProfileRouteImport parentRoute: typeof rootRouteImport } } From 2f9b17801526d28ca03965a7ac3b7f672ceb66ae Mon Sep 17 00:00:00 2001 From: utmist-tt <339893541+utmist-tt@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:13:49 -0400 Subject: [PATCH 03/15] Preserve working Kubernetes pilot and foundation execution plan --- .dockerignore | 5 + README.md | 92 ++- cli/cmd/api_client.go | 63 ++ cli/cmd/config_test.go | 46 +- cli/cmd/job.go | 3 +- cli/cmd/job_cancel.go | 67 +- cli/cmd/job_cancel_test.go | 55 +- cli/cmd/job_list.go | 76 +- cli/cmd/job_list_test.go | 35 +- cli/cmd/job_logs.go | 21 + cli/cmd/job_status.go | 46 +- cli/cmd/job_status_test.go | 39 +- cli/cmd/job_submit.go | 89 +- cli/cmd/job_submit_test.go | 85 +- cli/cmd/root.go | 4 + cli/docs/setup.md | 52 +- deploy/k3s/Dockerfile.api | 7 + deploy/k3s/README.md | 304 ++++++- deploy/k3s/mist-api-forward.service | 11 + deploy/k3s/mist-api-results.json | 191 +++++ deploy/k3s/mist-api.yaml | 162 ++++ deploy/k3s/mist-tenstorrent-claims.yaml | 72 ++ deploy/k3s/mist-web.service | 13 + deploy/k3s/setup-runbook.md | 79 +- deploy/k3s/submit_tenstorrent_smoke.py | 59 ++ .../k3s/tenstorrent-allocation-results.json | 153 ++++ deploy/k3s/tenstorrent-operator-values.yaml | 29 + deploy/k3s/tenstorrent-workload-policy.yaml | 40 + deploy/k3s/training-nvidia-job.yaml | 50 ++ deploy/k3s/training-smoke-results.json | 61 ++ deploy/k3s/training-tenstorrent-job.yaml | 106 +++ deploy/k3s/training_smoke.py | 209 +++++ deploy/k3s/verify_mist_api.py | 184 +++++ deploy/k3s/verify_tenstorrent_allocation.py | 225 ++++++ docs/foundation-execution-plan.md | 338 ++++++++ docs/kubernetes-pilot.md | 161 ++-- src/api.go | 94 ++- src/go.mod | 35 + src/go.sum | 121 +++ src/jobs.md | 97 ++- src/kubernetes.go | 759 ++++++++++++++++++ src/kubernetes_test.go | 122 +++ src/util.go | 29 +- web-interface/README.md | 31 + web-interface/src/api.ts | 51 ++ .../src/components/JobsPage.test.tsx | 51 ++ web-interface/src/components/JobsPage.tsx | 127 +++ web-interface/src/routes/jobs.tsx | 255 +----- web-interface/vite.config.ts | 8 + 49 files changed, 4250 insertions(+), 762 deletions(-) create mode 100644 .dockerignore create mode 100644 cli/cmd/api_client.go create mode 100644 cli/cmd/job_logs.go create mode 100644 deploy/k3s/Dockerfile.api create mode 100644 deploy/k3s/mist-api-forward.service create mode 100644 deploy/k3s/mist-api-results.json create mode 100644 deploy/k3s/mist-api.yaml create mode 100644 deploy/k3s/mist-tenstorrent-claims.yaml create mode 100644 deploy/k3s/mist-web.service create mode 100644 deploy/k3s/submit_tenstorrent_smoke.py create mode 100644 deploy/k3s/tenstorrent-allocation-results.json create mode 100644 deploy/k3s/tenstorrent-operator-values.yaml create mode 100644 deploy/k3s/tenstorrent-workload-policy.yaml create mode 100644 deploy/k3s/training-nvidia-job.yaml create mode 100644 deploy/k3s/training-smoke-results.json create mode 100644 deploy/k3s/training-tenstorrent-job.yaml create mode 100644 deploy/k3s/training_smoke.py create mode 100644 deploy/k3s/verify_mist_api.py create mode 100644 deploy/k3s/verify_tenstorrent_allocation.py create mode 100644 docs/foundation-execution-plan.md create mode 100644 src/kubernetes.go create mode 100644 src/kubernetes_test.go create mode 100644 web-interface/src/api.ts create mode 100644 web-interface/src/components/JobsPage.test.tsx create mode 100644 web-interface/src/components/JobsPage.tsx diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c81d217 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +web-interface/node_modules +web-interface/dist +logs +**/__pycache__ diff --git a/README.md b/README.md index e785670..631746d 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,92 @@ # Mist -Mist: UTMIST's Compute Platform +UTMIST's compute platform. The Kubernetes pilot runs submitted CPU, NVIDIA, +and Tenstorrent workloads as real Jobs. The Jobs page and CLI use the same +API for submission, status, logs, and cancellation. -For the proposed Kubernetes execution pilot and migration steps, see -[docs/kubernetes-pilot.md](docs/kubernetes-pilot.md). +## Current pilot -## Runtime requirements +- `utmist-z1opa08`: k3s server, CPU workloads, two NVIDIA RTX A4000 GPUs. +- `utmist-tt`: QuietBox worker, four n300 boards with two Wormhole chips each. +- Request one or two NVIDIA GPUs, or one to four Tenstorrent boards. Other + jobs can use the remaining devices; requests wait when capacity is occupied. +- Jobs page: http://127.0.0.1:3001/jobs. API: http://127.0.0.1:3000. -- Go 1.25.1 (also satisfies the backend's Go 1.24.6 requirement) -- Node.js 22.16.0 or newer, with npm -- Docker Engine and the Docker Compose plugin (`docker compose`) -- Permission to run `docker ps` as the same user that runs the backend +The local pilot uses one configured owner, without user authentication, +credits, or team quotas. Keep access local. Tenstorrent scripts need compatible +TT-NN code; distributed training and arbitrary models have not been validated. -## Run locally +See [installation and live checks](deploy/k3s/README.md#mist-api-cli-and-jobs-page) +and [architecture and remaining work](docs/kubernetes-pilot.md). -Ensure `go version`, `node --version`, and `docker ps` work before starting. -Run the following from the repository root: +## Run locally against this cluster + +Use Go 1.25.1 and Node.js 22.16.0 or newer. Apply the workload prerequisites +and claim templates described in the deployment guide first. Kubernetes is +now the default executor; Redis and Docker supervisors are unnecessary for +this path. This server has both runtimes under `~/.local/share/mist-runtimes`: ```bash -docker compose up -d redis -docker build -t pytorch-cpu src/docker/pytorch-cpu +export PATH=/home/utmist/.local/share/mist-runtimes/go-1.25.1/bin:/home/utmist/.local/share/mist-runtimes/node-22.16.0/bin:$PATH ``` -Start the backend in one terminal. Run it from `src` so it finds its logging configuration: - ```bash +export KUBECONFIG=/home/utmist/.kube/config +export MIST_PILOT_OWNER=utmist cd src go run . ``` -Start the frontend in another terminal: +If the deployed API's port-forward already occupies port 3000, use that API +or stop only `mist-api-forward.service` before starting a local backend. + +In another terminal: ```bash cd web-interface npm install --package-lock=false -npm run dev -- --port 3001 +npm run dev -- --host 127.0.0.1 --port 3001 --strictPort +``` + +The Vite proxy forwards `/api` to the API on port 3000. The installed user +services already run both endpoints on this machine: + +```bash +systemctl --user status mist-api-forward mist-web ``` -Open http://localhost:3001/jobs. The API uses http://localhost:3000. -The repository includes a Bun lockfile; npm installation does not use that lockfile. +## CLI -The optional CLI can be built from a third terminal: +From the repository root: ```bash -cd cli -go build -o ../bin/mist . -../bin/mist --help +go -C cli build -o ../bin/mist . +printf "print('Hello from a real Mist job')\n" > /tmp/hello.py +bin/mist job submit /tmp/hello.py --compute CPU +bin/mist job list --all +bin/mist job status +bin/mist job logs +bin/mist job cancel ``` -The frontend and CLI currently contain mock data and placeholder actions. The -backend's job execution is also incomplete; starting these services does not yet -provide end-to-end workload execution. +Use `--compute NVIDIA --devices 1` for one GPU, or +`--compute TT --devices 1` for one two-chip board. The submitted script must +actually use the chosen accelerator. `--api-url` or `MIST_API_URL` selects an +API endpoint. See [CLI setup](cli/docs/setup.md) for resource options. + +## Tests and legacy development + +```bash +go -C src test -run TestKubernetes ./... +go -C cli test ./... +cd web-interface +npm test -- --run +npm run build +npx tsc --noEmit +``` -Stop the backend/frontend with Ctrl+C in their terminals, then run -`docker compose stop redis` from the repository root to stop Redis without deleting its data. +The older Docker/Redis executor is available with `MIST_EXECUTOR=docker`. +Its supervisor execution is incomplete. For its existing tests, start a +local Redis instance and run `go -C src test ./...`; Docker integration tests +also need Docker permissions and their test image. This path is retained for +compatibility and is separate from the validated Kubernetes executor. diff --git a/cli/cmd/api_client.go b/cli/cmd/api_client.go new file mode 100644 index 0000000..83413b2 --- /dev/null +++ b/cli/cmd/api_client.go @@ -0,0 +1,63 @@ +package cmd + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +func (ctx *AppContext) api(method, path string, body, out interface{}) error { + if ctx == nil { + return fmt.Errorf("missing application context") + } + base := ctx.APIBaseURL + if base == "" && ctx.Config != nil { + base = ctx.Config.APIBaseURL + } + if base == "" { + base = "http://127.0.0.1:3000" + } + var data []byte + var err error + if body != nil { + data, err = json.Marshal(body) + if err != nil { + return err + } + } + req, err := http.NewRequest(method, strings.TrimRight(base, "/")+path, bytes.NewReader(data)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + if ctx.Config != nil && ctx.Config.AccessToken != "" { + req.Header.Set("Authorization", "Bearer "+ctx.Config.AccessToken) + } + client := ctx.HTTPClient + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + response, err := client.Do(req) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + data, _ := io.ReadAll(io.LimitReader(response.Body, 8192)) + var errorBody struct { + Error string `json:"error"` + } + if json.Unmarshal(data, &errorBody) == nil && errorBody.Error != "" { + return fmt.Errorf("API %d: %s", response.StatusCode, errorBody.Error) + } + return fmt.Errorf("API %d: %s", response.StatusCode, strings.TrimSpace(string(data))) + } + if out != nil { + return json.NewDecoder(response.Body).Decode(out) + } + return nil +} diff --git a/cli/cmd/config_test.go b/cli/cmd/config_test.go index 60758db..d3ecb15 100644 --- a/cli/cmd/config_test.go +++ b/cli/cmd/config_test.go @@ -1,4 +1,4 @@ -package cmd +package cmd import ( "testing" @@ -6,52 +6,52 @@ import ( // "bytes" ) -// No Flag config -func TestConfigNoFlags(t *testing.T){ +// No Flag config +func TestConfigNoFlags(t *testing.T) { cmd := &ConfigCmd{} - output := CaptureOutput(func(){ - _ = cmd.Run() + output := CaptureOutput(func() { + _ = cmd.Run(&AppContext{}) }) - if want := "No config action specified. Use --help for options."; !contains(output, want){ - t.Errorf("expected output to contain %q, got %q", want, output) + if want := "No config action specified. Use --help for options."; !contains(output, want) { + t.Errorf("expected output to contain %q, got %q", want, output) } } // Set Default Cluster to tt-gpu-cluster-1 -func TestConfigDefaultCluster(t *testing.T){ - cmd := &ConfigCmd{DefaultCluster: "tt-gpu-cluster-1"} // Create config object +func TestConfigDefaultCluster(t *testing.T) { + cmd := &ConfigCmd{DefaultCluster: "tt-gpu-cluster-1"} // Create config object - output := CaptureOutput(func(){ - _ = cmd.Run() + output := CaptureOutput(func() { + _ = cmd.Run(&AppContext{}) }) // fmt.Printf("Captured the output: %s\n", output) if want := "Setting default cluster to: tt-gpu-cluster-1"; !contains(output, want) { - t.Errorf("expected output to contain %q, got %q", want, output) - } + t.Errorf("expected output to contain %q, got %q", want, output) + } } -// Show Config -func TestConfigCmd_Show(t *testing.T){ +// Show Config +func TestConfigCmd_Show(t *testing.T) { cmd := &ConfigCmd{Show: true} - output := CaptureOutput(func(){ - _ = cmd.Run() + output := CaptureOutput(func() { + _ = cmd.Run(&AppContext{}) }) - if want := "Current configuration:"; !contains(output, want){ + if want := "Current configuration:"; !contains(output, want) { t.Errorf("expected output to contain %q, got %q", want, output) } } -// Show Error message if both flags are sent -func TestConfigBothFlagError(t *testing.T){ +// Show Error message if both flags are sent +func TestConfigBothFlagError(t *testing.T) { cmd := &ConfigCmd{DefaultCluster: "tt-gpu-cluster-1", Show: true} - output := CaptureOutput(func(){ - _ = cmd.Run() + output := CaptureOutput(func() { + _ = cmd.Run(&AppContext{}) }) - if want := "Cannot use --show and --default-cluster together"; !contains(output, want){ + if want := "Cannot use --show and --default-cluster together"; !contains(output, want) { t.Errorf("Expected the error message of \"%s\", got %q", want, output) } diff --git a/cli/cmd/job.go b/cli/cmd/job.go index 6fcf5b2..94f487a 100644 --- a/cli/cmd/job.go +++ b/cli/cmd/job.go @@ -6,7 +6,8 @@ type JobCmd struct { // Delete JobDeleteCmd `cmd: "" help: "Delete an existing job"` Status JobStatusCmd `cmd:"" help:"Check the status of a job"` // Cancel CancelCmd `cmd:"" help:"Cancel a running job"` - List ListCmd `cmd:"" help:"List all jobs" default:1` + Logs JobLogsCmd `cmd:"" help:"Read workload logs"` + List ListCmd `cmd:"" help:"List all jobs" default:1` } func (j *JobCmd) Run() error { diff --git a/cli/cmd/job_cancel.go b/cli/cmd/job_cancel.go index 949a885..0cd7072 100644 --- a/cli/cmd/job_cancel.go +++ b/cli/cmd/job_cancel.go @@ -1,72 +1,19 @@ package cmd import ( - "bufio" "fmt" - "os" - "strings" - "time" + "net/url" ) type JobCancelCmd struct { - ID string `arg:"" help:"ID of job you want to cancel"` + ID string `arg:"" help:"Job ID to cancel"` } -func (c *JobCancelCmd) Run(ctx *AppContext) error { - // Same Mock data from job list. - jobs := []Job{ - { - ID: "ID:1", - Name: "docker_container_name_1", - Status: "Running", - GPUType: "AMD", - CreatedAt: time.Now(), - }, - { - ID: "ID:2", - Name: "docker_container_name_2", - Status: "Enqueued", - GPUType: "TT", - CreatedAt: time.Now().Add(-time.Hour * 24), - }, - { - ID: "ID:3", - Name: "docker_container_name_3", - Status: "Running", - GPUType: "TT", - CreatedAt: time.Now().Add(-time.Hour * 24), - }, +func (j *JobCancelCmd) Run(ctx *AppContext) error { + var job Job + if err := ctx.api("POST", "/jobs/"+url.PathEscape(j.ID)+"/cancel", nil, &job); err != nil { + return err } - - // Check if job exists - if !jobExists(jobs, c.ID) { - fmt.Printf("%s does not exist in your jobs.\n", c.ID) - fmt.Printf("Use the command \"job list\" for your list of jobs.") - return nil - } - - fmt.Printf("Are you sure you want to cancel %s? (y/n): \n", c.ID) - - reader := bufio.NewReader(os.Stdin) - input, _ := reader.ReadString('\n') - input = strings.TrimSpace(strings.ToLower(input)) - - if input == "y" || input == "yes" { - fmt.Println("Confirmed, proceeding job cancellation....") - - // Confirmed job cancellation logic - - fmt.Println("Cancelling job with ID:", c.ID) - fmt.Printf("Job cancelled successfully with ID: %s\n", c.ID) - return nil - } else if input == "n" || input == "no" { - fmt.Println("Cancelled.") - return nil - } else { - fmt.Println("Invalid response.") - return nil - } - + fmt.Printf("Job %s: %s\n", job.ID, job.Status) return nil - } diff --git a/cli/cmd/job_cancel_test.go b/cli/cmd/job_cancel_test.go index eff91c6..79da1e2 100644 --- a/cli/cmd/job_cancel_test.go +++ b/cli/cmd/job_cancel_test.go @@ -1,46 +1,25 @@ -package cmd +package cmd import ( + "net/http" + "net/http/httptest" "testing" - "fmt" ) - -// Added job, with no compute type added -func TestJobCancelJobDoesNotExist(t *testing.T){ - // This job should not exist in the dummy - cmd := &JobCancelCmd{ID: "job_12345"} - output := CaptureOutput(func(){ - _ = cmd.Run() - }) - if want := "job_12345 does not exist in your jobs.\nUse the command \"job list\" for your list of jobs."; !contains(output, want){ - t.Errorf("expected output to contain %q, got %q", want, output) - } -} - -// Added job, with compute type -func TestJobCancelValid(t *testing.T){ - // This job should not exist in the dummy - cmd := &JobCancelCmd{ID: "ID:1"} - output := CaptureOutput(func(){ - _ = cmd.Run() +func TestJobCancelCallsAPIWithoutConfirmation(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" || r.URL.Path != "/jobs/cancel-id/cancel" { + t.Errorf("wrong request: %s %s", r.Method, r.URL.Path) + } + _, _ = w.Write([]byte(`{"id":"cancel-id","job_state":"Cancelled"}`)) + })) + defer server.Close() + out := CaptureOutput(func() { + if err := (&JobCancelCmd{ID: "cancel-id"}).Run(&AppContext{APIBaseURL: server.URL}); err != nil { + t.Error(err) + } }) - if want := "Are you sure you want to cancel ID:1? (y/n):"; !contains(output, want){ - t.Errorf("expected output to contain %q, got %q", want, output) + if !contains(out, "Cancelled") || contains(out, "Are you sure") { + t.Fatal(out) } } - - -func TestJobCancelProceed(t *testing.T){ - cmd := &JobCancelCmd{ID: "ID:1"} - // Lowkey, we should refactor this into a - output := CaptureOutput(func(){ - MockInput("y\n", func() { - _ = cmd.Run() - }) - }) - if !contains(output, "Confirmed, proceeding job cancellation...."){ - t.Errorf("expected 'Confirmed, proceeding job cancellation....' but got:\n%s", output) - } - // fmt.Printf("Got the output %s\n", output) -} \ No newline at end of file diff --git a/cli/cmd/job_list.go b/cli/cmd/job_list.go index 9e4787d..0dfec2d 100644 --- a/cli/cmd/job_list.go +++ b/cli/cmd/job_list.go @@ -8,60 +8,42 @@ import ( ) type ListCmd struct { - All bool `help:"List all jobs, including completed and failed ones." short:"a"` + All bool `help:"Include completed, failed, and cancelled jobs" short:"a"` } - type Job struct { - ID string - Name string - Status string - GPUType string - CreatedAt time.Time + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"job_state"` + GPUType string `json:"accelerator"` + CreatedAt time.Time `json:"created"` + Node string `json:"node"` + DeviceCount int `json:"device_count"` + ExitCode *int `json:"exit_code"` + Message string `json:"message"` } -func (l *ListCmd) Run(ctx *AppContext) error { - // Mock data - pull from API in real implementation - jobs := []Job{ - { - ID: "ID:1", - Name: "docker_container_name_1", - Status: "Running", - GPUType: "AMD", - CreatedAt: time.Now(), - }, - { - ID: "ID:2", - Name: "docker_container_name_2", - Status: "Enqueued", - GPUType: "TT", - CreatedAt: time.Now().Add(-time.Hour * 24), - }, - { - ID: "ID:3", - Name: "docker_container_name_3", - Status: "Running", - GPUType: "TT", - CreatedAt: time.Now().Add(-time.Hour * 24), - }, - } +func printJobs(jobs []Job) { w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "Job ID\tName\tStatus\tGPU Type\tCreated At") - fmt.Fprintln(w, "--------------------------------------------------------------") - + fmt.Fprintln(w, "Job ID\tName\tStatus\tCompute\tDevices\tNode\tCreated") for _, job := range jobs { - // Maybe filter based on running? - fmt.Fprintf( - w, - "%s\t%s\t%s\t%s\t%s\n", - job.ID, - job.Name, - job.Status, - job.GPUType, - job.CreatedAt.Format(time.RFC1123), - ) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%s\t%s\n", job.ID, job.Name, job.Status, job.GPUType, job.DeviceCount, job.Node, job.CreatedAt.Format(time.RFC3339)) } + _ = w.Flush() +} - w.Flush() - +func (l *ListCmd) Run(ctx *AppContext) error { + var response struct { + Jobs []Job `json:"jobs"` + } + if err := ctx.api("GET", "/jobs", nil, &response); err != nil { + return err + } + jobs := []Job{} + for _, job := range response.Jobs { + if l.All || job.Status == "Scheduled" || job.Status == "InProgress" { + jobs = append(jobs, job) + } + } + printJobs(jobs) return nil } diff --git a/cli/cmd/job_list_test.go b/cli/cmd/job_list_test.go index ad9b088..070ce98 100644 --- a/cli/cmd/job_list_test.go +++ b/cli/cmd/job_list_test.go @@ -1,18 +1,31 @@ -package cmd +package cmd import ( + "net/http" + "net/http/httptest" "testing" ) -// Will be more specific in the future! -func TestJobList(t *testing.T){ - cmd := &ListCmd{All: true} - output := CaptureOutput(func(){ - _ = cmd.Run() +func TestJobListFiltersCompleted(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"jobs":[{"id":"running-id","name":"work","job_state":"InProgress","accelerator":"nvidia"},{"id":"done-id","name":"old","job_state":"Success","accelerator":"cpu"}]}`)) + })) + defer server.Close() + ctx := &AppContext{APIBaseURL: server.URL} + out := CaptureOutput(func() { + if err := (&ListCmd{}).Run(ctx); err != nil { + t.Error(err) + } }) - // Note the time is dynamic. - want := "Job ID Name Status GPU Type Created At\n--------------------------------------------------------------\nID:1 docker_container_name_1 Running AMD" - if !contains(output, want){ - t.Errorf("expected output to contain %q, got %q", want, output) + if !contains(out, "running-id") || contains(out, "done-id") { + t.Fatal(out) } - } \ No newline at end of file + out = CaptureOutput(func() { + if err := (&ListCmd{All: true}).Run(ctx); err != nil { + t.Error(err) + } + }) + if !contains(out, "done-id") { + t.Fatal(out) + } +} diff --git a/cli/cmd/job_logs.go b/cli/cmd/job_logs.go new file mode 100644 index 0000000..b183510 --- /dev/null +++ b/cli/cmd/job_logs.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "fmt" + "net/url" +) + +type JobLogsCmd struct { + ID string `arg:"" help:"Job ID"` +} + +func (j *JobLogsCmd) Run(ctx *AppContext) error { + var response struct { + Logs string `json:"logs"` + } + if err := ctx.api("GET", "/jobs/"+url.PathEscape(j.ID)+"/logs", nil, &response); err != nil { + return err + } + fmt.Print(response.Logs) + return nil +} diff --git a/cli/cmd/job_status.go b/cli/cmd/job_status.go index 20634c8..9d15a01 100644 --- a/cli/cmd/job_status.go +++ b/cli/cmd/job_status.go @@ -2,46 +2,24 @@ package cmd import ( "fmt" - "os" - "text/tabwriter" - "time" + "net/url" ) type JobStatusCmd struct { - ID string `arg:"" help:"The ID of the job to check the status for"` + ID string `arg:"" help:"Job ID"` } func (j *JobStatusCmd) Run(ctx *AppContext) error { - // Mock data - pull from API in real implementation - jobs := []Job{{ - ID: "ID:1", - Name: "docker_container_name_1", - Status: "Running", - GPUType: "AMD", - CreatedAt: time.Now(), - }} - - job, err := findJobByID(jobs, j.ID) - if err != nil { - fmt.Printf("%s does not exist in your jobs.\n", j.ID) - fmt.Printf("Use the command \"job list\" for your list of jobs.") - return nil + var job Job + if err := ctx.api("GET", "/jobs/"+url.PathEscape(j.ID), nil, &job); err != nil { + return err + } + printJobs([]Job{job}) + if job.ExitCode != nil { + fmt.Println("Exit code:", *job.ExitCode) + } + if job.Message != "" { + fmt.Println(job.Message) } - - println("Checking status for job ID:", j.ID) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "Job ID\tName\tStatus\tGPU Type\tCreated At") - fmt.Fprintln(w, "--------------------------------------------------------------") - - fmt.Fprintf( - w, - "%s\t%s\t%s\t%s\t%s\n", - job.ID, - job.Name, - job.Status, - job.GPUType, - job.CreatedAt.Format(time.RFC1123), - ) - w.Flush() return nil } diff --git a/cli/cmd/job_status_test.go b/cli/cmd/job_status_test.go index d7d82a8..09e1f1d 100644 --- a/cli/cmd/job_status_test.go +++ b/cli/cmd/job_status_test.go @@ -1,29 +1,32 @@ package cmd import ( + "net/http" + "net/http/httptest" "testing" ) -// Added job, with no compute type added -func TestJobStatusJobDoesNotExist(t *testing.T) { - // This job should not exist in the dummy - cmd := &JobStatusCmd{ID: "job_12345"} - output := CaptureOutput(func() { - _ = cmd.Run() +func TestJobStatusPropagatesFailureAndExitCode(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"id":"failed-id","name":"failed","job_state":"Failure","exit_code":17}`)) + })) + defer server.Close() + out := CaptureOutput(func() { + if err := (&JobStatusCmd{ID: "failed-id"}).Run(&AppContext{APIBaseURL: server.URL}); err != nil { + t.Error(err) + } }) - if want := "job_12345 does not exist in your jobs.\nUse the command \"job list\" for your list of jobs."; !contains(output, want) { - t.Errorf("expected output to contain %q, got %q", want, output) + if !contains(out, "Failure") || !contains(out, "Exit code: 17") { + t.Fatal(out) } } - -// Added job, with compute type -func TestJobStatusValid(t *testing.T) { - // This job should not exist in the dummy - cmd := &JobStatusCmd{ID: "ID:1"} - output := CaptureOutput(func() { - _ = cmd.Run() - }) - if want := "docker_container_name_1"; !contains(output, want) { - t.Errorf("expected output to contain %q, got %q", want, output) +func TestJobStatusDoesNotInventSuccessForMissingJob(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + _, _ = w.Write([]byte(`{"error":"job not found"}`)) + })) + defer server.Close() + if err := (&JobStatusCmd{ID: "missing"}).Run(&AppContext{APIBaseURL: server.URL}); err == nil || !contains(err.Error(), "job not found") { + t.Fatal(err) } } diff --git a/cli/cmd/job_submit.go b/cli/cmd/job_submit.go index 22b793f..b513db4 100644 --- a/cli/cmd/job_submit.go +++ b/cli/cmd/job_submit.go @@ -1,61 +1,60 @@ package cmd import ( - "bufio" "fmt" "os" + "path/filepath" "strings" ) type JobSubmitCmd struct { - Script string `arg:"" help:"Path to the job script file to submit"` - Compute string `help:"Type of compute required for the job: AMD|TT|CPU" default:"AMD"` + Script string `arg:"" help:"Path to a Python or shell script"` + Compute string `help:"CPU, NVIDIA, or TT" default:"CPU"` + Devices int `help:"NVIDIA GPU count or Tenstorrent board count" default:"1"` + Image string `help:"Approved container image (optional; defaults by compute type)"` + CPU string `help:"Requested CPU quantity, e.g. 2 or 500m"` + Memory string `help:"Requested memory, e.g. 4Gi"` + Timeout int64 `help:"Maximum execution time in seconds" default:"600"` + Name string `help:"Display name for the job"` } func (j *JobSubmitCmd) Run(ctx *AppContext) error { - // mist job submit