diff --git a/.github/actions/truenas-appliance-runner/action.yml b/.github/actions/truenas-appliance-runner/action.yml new file mode 100644 index 0000000..b00d59b --- /dev/null +++ b/.github/actions/truenas-appliance-runner/action.yml @@ -0,0 +1,126 @@ +name: TrueNAS appliance +description: >- + Boot the TrueNAS appliance attached to this runner under qemu, and hand + the job its address and a credential of its own. The appliance lives + and dies with the runner, so there is nothing to release. + +# The runner comes from a GARM pool on a TrueNAS box whose extra specs +# attach the appliance's disk: +# +# {"disks": ["truenas_appliance:nightly"]} +# +# and whose flavor has room for it beside the job (the appliance's memory, +# 6 GB by default, plus the browser and the build). The pool, the provider +# and the appliance image are iXsystems/github-truenas-runner's (its +# docs/e2e-appliances.md). +# +# These scripts are the one copy: that repository's image build fetches +# them from here, at the last commit that changed this directory, to +# install the appliance image and test-boot it. A change here is a new +# appliance image there. What the two sides share is in lib.sh. +# +# Usage: +# - uses: iXsystems/ux-github-workflows/.github/actions/truenas-appliance-runner@master +# id: appliance +# - run: curl -sk "https://$TN_HOST/api/v2.0/system/info" -u "$TN_USERNAME:$TN_PASSWORD" +# # between tests, back to the baseline in seconds: +# - run: truenas-appliance revert fresh-install +# +# The TN_* variables are the e2e suite's contract, as the lab's claim +# action sets them; truenas-appliance (appliance.sh) is on PATH for +# snapshot, revert, probe, call and console. + +inputs: + baseline: + description: Snapshot taken once the appliance is up, to revert to between tests. Empty takes none. + default: fresh-install + memory-mb: + description: Appliance memory. + default: '6144' + vcpus: + description: Appliance vCPUs. + default: '4' + data-disk-count: + description: Data disks, sparse. + default: '10' + data-disk-gb: + description: Size of each data disk. + default: '10' + ssh: + description: '`true` enables ssh for the admin, on TN_SSH.' + default: 'false' + disk: + description: The appliance disk, if not the one the pool attached (found by its boot-pool). + default: '' + export-env: + description: '`true` writes TN_* to the job environment, for suites that read them from there.' + default: 'true' + +outputs: + host: + description: host:port of the appliance over https. + value: ${{ steps.start.outputs.host }} + host-http: + description: host:port of the appliance over http. + value: ${{ steps.start.outputs.host-http }} + username: + description: Admin user. + value: ${{ steps.start.outputs.username }} + password: + description: Admin password, made for this job and masked. + value: ${{ steps.start.outputs.password }} + ssh: + description: host:port of its ssh, when asked for. + value: ${{ steps.start.outputs.ssh }} + boot-seconds: + description: From qemu starting to the API taking a login. + value: ${{ steps.start.outputs.boot-seconds }} + snapshot-seconds: + description: What taking the baseline took. + value: ${{ steps.start.outputs.snapshot-seconds }} + +runs: + using: composite + steps: + - id: start + shell: bash + env: + APPLIANCE_BASELINE: ${{ inputs.baseline }} + APPLIANCE_MEMORY_MB: ${{ inputs.memory-mb }} + APPLIANCE_VCPUS: ${{ inputs.vcpus }} + APPLIANCE_DATA_DISKS: ${{ inputs.data-disk-count }} + APPLIANCE_DATA_DISK_GB: ${{ inputs.data-disk-gb }} + APPLIANCE_SSH: ${{ inputs.ssh }} + APPLIANCE_DISK_INPUT: ${{ inputs.disk }} + EXPORT_ENV: ${{ inputs.export-env }} + run: | + set -euo pipefail + lib="$GITHUB_ACTION_PATH" + [ -n "$APPLIANCE_DISK_INPUT" ] && export APPLIANCE_DISK="$APPLIANCE_DISK_INPUT" + export APPLIANCE_DIR="$RUNNER_TEMP/truenas-appliance" + # The password is made here, so it is masked before anything can + # print it. + APPLIANCE_PASSWORD="$(head -c 15 /dev/urandom | base64 | tr -d '/+=')" + echo "::add-mask::$APPLIANCE_PASSWORD" + export APPLIANCE_PASSWORD + # On PATH before it starts, so that a failed start's console is + # there for a later step to show. + echo "APPLIANCE_DIR=$APPLIANCE_DIR" >> "$GITHUB_ENV" + mkdir -p "$RUNNER_TEMP/truenas-appliance-bin" + ln -sf "$lib/appliance.sh" "$RUNNER_TEMP/truenas-appliance-bin/truenas-appliance" + echo "$RUNNER_TEMP/truenas-appliance-bin" >> "$GITHUB_PATH" + out="$("$lib/appliance.sh" start)" + get() { sed -n "s/^$1=//p" <<<"$out"; } + { + echo "host=$(get TN_HOST)" + echo "host-http=$(get TN_HOST_HTTP)" + echo "username=$(get TN_USERNAME)" + echo "password=$(get TN_PASSWORD)" + echo "ssh=$(get TN_SSH)" + echo "boot-seconds=$(get APPLIANCE_BOOT_SECONDS)" + echo "snapshot-seconds=$(get APPLIANCE_SNAPSHOT_SECONDS)" + } >> "$GITHUB_OUTPUT" + if [ "$EXPORT_ENV" = true ]; then + grep '^TN_' <<<"$out" >> "$GITHUB_ENV" + fi + echo "Appliance $(get TN_DOMAIN) at $(get TN_HOST): API up in $(get APPLIANCE_BOOT_SECONDS)s, baseline '${APPLIANCE_BASELINE:-none}' in $(get APPLIANCE_SNAPSHOT_SECONDS)s" diff --git a/.github/actions/truenas-appliance-runner/appliance.sh b/.github/actions/truenas-appliance-runner/appliance.sh new file mode 100755 index 0000000..8b4a30f --- /dev/null +++ b/.github/actions/truenas-appliance-runner/appliance.sh @@ -0,0 +1,248 @@ +#!/usr/bin/env bash +# A TrueNAS appliance for this job, booted under qemu inside the runner. +# +# The runner's pool attaches the appliance's disk ("disks" in the pool's +# extra specs: a clone of truenas_appliance: on the box), and the +# appliance lives and dies with the runner: nothing to claim from anyone, +# nothing to release, nothing to leak. See iXsystems/github-truenas-runner's +# docs/e2e-appliances.md. +# +# appliance.sh start boot it, wait for its API, give it this +# claim's admin password, and snapshot it +# as the baseline; KEY=value on stdout +# appliance.sh snapshot save the running appliance, memory and +# disks, as +# appliance.sh revert back to , in seconds, no boot +# appliance.sh probe [baseline] change it, revert, check the change went +# appliance.sh call [params...] +# a middleware API call, as the admin, its +# result as JSON on stdout +# appliance.sh snapshots the names there are +# appliance.sh console [lines] the end of its serial console +# appliance.sh stop +# +# Settings, environment variables, all optional: +# +# APPLIANCE_DISK the appliance's disk: a device or a qcow2/raw file +# (default: the disk the pool attached, found by +# attached_disk) +# APPLIANCE_DIR its state: overlay, data disks, sockets, console +# (default $RUNNER_TEMP/truenas-appliance) +# APPLIANCE_MEMORY_MB 6144 APPLIANCE_VCPUS 4 +# APPLIANCE_DATA_DISKS 10 APPLIANCE_DATA_DISK_GB 10 (sparse) +# APPLIANCE_HTTP_PORT 8080 APPLIANCE_HTTPS_PORT 8443 +# APPLIANCE_SSH_PORT 2222 APPLIANCE_SSH false +# APPLIANCE_BASELINE fresh-install: the snapshot start takes; empty +# takes none +# APPLIANCE_PASSWORD the claim's admin password (default: generated) +# APPLIANCE_BOOT_TIMEOUT 900 seconds +# APPLIANCE_ACCEL kvm; tcg runs without KVM, too slowly for +# anything but testing this script +# +# start prints TN_PROFILE, TN_HOST (host:https port), TN_HOST_HTTP, +# TN_USERNAME, TN_PASSWORD, TN_DOMAIN, TN_BASELINE (the suite's contract, +# as the lab's claim action gives it, less its TN_GUEST_ISO), TN_SSH when +# APPLIANCE_SSH is true, and what it measured: +# APPLIANCE_BOOT_SECONDS, APPLIANCE_SNAPSHOT_SECONDS. + +set -euo pipefail +# shellcheck source=lib.sh +. "$(dirname "$(realpath "$0")")/lib.sh" + +DIR="${APPLIANCE_DIR:-${RUNNER_TEMP:-/tmp}/truenas-appliance}" +QMP="$DIR/qmp.sock" +STATE="$DIR/state.env" + +hmp() { "${TNCTL[@]}" hmp "$QMP" "$*"; } +# resume: a failed loadvm leaves the VM paused, and later loadvms leave it +# so, however they go; cont is a no-op on one that runs. +resume() { "${TNCTL[@]}" qmp "$QMP" cont >/dev/null; } +url() { echo "ws://127.0.0.1:${APPLIANCE_HTTP_PORT:-8080}"; } +running() { [ -f "$DIR/qemu.pid" ] && kill -0 "$(cat "$DIR/qemu.pid")" 2>/dev/null; } +# shellcheck source=/dev/null +state() { [ -r "$STATE" ] || die "no appliance started here ($STATE)"; . "$STATE"; } + +# attached_disk: the appliance disk the runner's pool attached. By its +# serial when the box kept the one the provider gave it; TrueNAS gives +# every VM disk a random serial of its own instead, so otherwise by what is +# on it: the one disk, not the one / is on, that holds a ZFS pool named +# boot-pool (a TrueNAS install). +attached_disk() { + local serial=/dev/disk/by-id/virtio-truenas_appliance + [ -e "$serial" ] && { realpath "$serial"; return; } + lsblk -J -p -o NAME,PKNAME,FSTYPE,LABEL,MOUNTPOINT | python3 -c ' +import json, sys +devs = [] +def walk(ds): + for d in ds: + devs.append(d) + walk(d.get("children") or []) +walk(json.load(sys.stdin)["blockdevices"]) +root = {d["pkname"] for d in devs if d.get("mountpoint") == "/"} +found = sorted({d["pkname"] for d in devs + if d.get("fstype") == "zfs_member" and d.get("label") == "boot-pool" + and d.get("pkname") and d["pkname"] not in root}) +if len(found) != 1: + sys.exit("found %d disks with a TrueNAS boot-pool: %s" % (len(found), " ".join(found) or "none")) +print(found[0]) +' +} + +start() { + local disk="${APPLIANCE_DISK:-}" + local memory="${APPLIANCE_MEMORY_MB:-6144}" vcpus="${APPLIANCE_VCPUS:-4}" + local ndata="${APPLIANCE_DATA_DISKS:-10}" datagb="${APPLIANCE_DATA_DISK_GB:-10}" + local http="${APPLIANCE_HTTP_PORT:-8080}" https="${APPLIANCE_HTTPS_PORT:-8443}" sshp="${APPLIANCE_SSH_PORT:-2222}" + local baseline="${APPLIANCE_BASELINE-fresh-install}" + local password="${APPLIANCE_PASSWORD:-$(head -c 15 /dev/urandom | base64 | tr -d '/+=')}" + local name="appliance-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${GITHUB_JOB:-job}" + + running && die "an appliance is running already ($DIR)" + needs qemu-system-x86_64 qemu-system-x86 qemu-img qemu-utils python3 python3 + ovmf + local accel="${APPLIANCE_ACCEL:-kvm}" cpu=host + if [ "$accel" = kvm ]; then + [ -w /dev/kvm ] || die "no /dev/kvm this user can use: the runner needs nested virtualization (cpu_mode HOST-PASSTHROUGH) and to be in the kvm group" + else + cpu=max + fi + if [ -z "$disk" ] && ! disk="$(attached_disk)"; then + echo "appliance: no appliance disk attached to this runner. Its disks:" >&2 + lsblk -p -o NAME,SIZE,SERIAL,FSTYPE,LABEL,MOUNTPOINT >&2 || true + die "give the runner's pool {\"disks\": [\"truenas_appliance:nightly\"]} in its extra specs, or set APPLIANCE_DISK" + fi + [ -e "$disk" ] || die "no appliance disk at $disk" + disk="$(realpath "$disk")" + # Read only: the appliance writes to an overlay of it. + if [ ! -r "$disk" ]; then + sudo -n chown "$(id -u)" "$disk" || die "$disk is not readable here, and sudo cannot change that" + fi + local format=raw + [ -f "$disk" ] && format="$(qemu-img info --output=json "$disk" | python3 -c 'import json,sys; print(json.load(sys.stdin)["format"])')" + + rm -rf "$DIR" + mkdir -p "$DIR" + qemu-img create -q -f qcow2 -F "$format" -b "$disk" "$DIR/os.qcow2" + vars_qcow2 "$DIR/vars.qcow2" + local drives=() i + for i in $(seq 1 "$ndata"); do + qemu-img create -q -f qcow2 "$DIR/data$i.qcow2" "${datagb}G" + drives+=(-drive "file=$DIR/data$i.qcow2,if=none,id=data$i,format=qcow2,discard=unmap" + -device "virtio-blk-pci,drive=data$i,serial=data$i") + done + + local booted + say "booting $disk as $name: ${memory} MB, $vcpus vCPUs, $ndata data disks of ${datagb}G" + qemu-system-x86_64 -name "$name" -machine "q35,accel=$accel" -cpu "$cpu" -smp "$vcpus" -m "$memory" \ + -smbios "type=1,serial=$name" \ + -drive "if=pflash,format=raw,readonly=on,file=$OVMF_CODE" \ + -drive "if=pflash,format=qcow2,file=$DIR/vars.qcow2" \ + -drive "file=$DIR/os.qcow2,if=none,id=os,format=qcow2,discard=unmap" \ + -device "virtio-blk-pci,drive=os,serial=os,bootindex=0" \ + "${drives[@]}" \ + -nic "user,model=virtio-net-pci$(hostfwd 80 "$http")$(hostfwd 443 "$https")$(hostfwd 22 "$sshp")" \ + -device virtio-rng-pci \ + -qmp "unix:$QMP,server=on,wait=off" \ + -display none -serial "file:$DIR/console.log" \ + -daemonize -pidfile "$DIR/qemu.pid" + + if ! booted=$("${TNCTL[@]}" wait "$(url)" "$APPLIANCE_USER" "$APPLIANCE_TEMPLATE_PASSWORD" \ + --timeout "${APPLIANCE_BOOT_TIMEOUT:-900}"); then + say "--- its console, last 60 lines" + tail -n 60 "$DIR/console.log" >&2 || true + die "the appliance's API never came up" + fi + say "API up ${booted}s after qemu started" + "${TNCTL[@]}" rotate "$(url)" "$APPLIANCE_USER" "$APPLIANCE_TEMPLATE_PASSWORD" "$password" + if [ "${APPLIANCE_SSH:-false}" = true ]; then + "${TNCTL[@]}" enable-ssh "$(url)" "$APPLIANCE_USER" "$password" + say "ssh on 127.0.0.1:$sshp" + fi + { + echo "APPLIANCE_PASSWORD_NOW=$(printf %q "$password")" + echo "APPLIANCE_HTTP_PORT=$http" + } > "$STATE" + + local snapped="" + if [ -n "$baseline" ]; then + local s0 + s0=$(date +%s.%N) + hmp savevm "$baseline" >&2 + snapped=$(awk -v a="$s0" -v b="$(date +%s.%N)" 'BEGIN {printf "%.1f", b - a}') + say "snapshot $baseline taken in ${snapped}s" + fi + + cat <&2 || rc=$? + resume + return "$rc" +} + +# revert: loadvm restores memory and disks as they were, so the API is up +# the moment it returns; its connections from before are gone, and its +# clock is the snapshot's until it catches up. +revert() { + local name="${1:?revert }" + running || die "no appliance running" + state + local t0 + t0=$(date +%s.%N) + if ! hmp loadvm "$name" >&2; then + resume + die "no snapshot $name to revert to; the appliance runs on as it was" + fi + resume + "${TNCTL[@]}" wait "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" --timeout 120 >/dev/null + awk -v a="$t0" -v b="$(date +%s.%N)" 'BEGIN {printf "%.1f\n", b - a}' +} + +# probe : that revert works and what it costs. A group is made +# over the API, the appliance reverted to , and the group must +# be gone; KEY=value on stdout. +probe() { + local baseline="${1:-fresh-install}" took left + state + call() { "${TNCTL[@]}" call "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" "$@"; } + call group.create '{"name": "e2e_probe"}' >/dev/null + [ "$(call group.query '[["name", "=", "e2e_probe"]]' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')" = 1 ] \ + || die "probe: the group made over the API is not there" + took="$(revert "$baseline")" + left="$(call group.query '[["name", "=", "e2e_probe"]]' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')" + [ "$left" = 0 ] || die "probe: after reverting to $baseline the group is still there" + say "revert to $baseline took ${took}s, and undid the change" + echo "APPLIANCE_REVERT_SECONDS=$took" +} + +case "${1:-}" in +start) start ;; +call) shift; [ $# -gt 0 ] || die "call [params...]"; state + "${TNCTL[@]}" call "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" "$@" ;; +probe) shift; probe "$@" ;; +snapshot) shift; snapshot "$@" ;; +revert) shift; revert "$@" ;; +snapshots) running || die "no appliance running"; hmp info snapshots ;; +console) tail -n "${2:-100}" "$DIR/console.log" ;; +stop) if running; then "${TNCTL[@]}" qmp "$QMP" quit >/dev/null || true; fi ;; +*) die "usage: appliance.sh start | snapshot | revert | probe [baseline] | call [params...] | snapshots | console [lines] | stop" ;; +esac diff --git a/.github/actions/truenas-appliance-runner/lib.sh b/.github/actions/truenas-appliance-runner/lib.sh new file mode 100644 index 0000000..c1be3ec --- /dev/null +++ b/.github/actions/truenas-appliance-runner/lib.sh @@ -0,0 +1,74 @@ +# shellcheck shell=bash disable=SC2034 +# Shared by appliance.sh, which boots an appliance in a runner, and by the +# image build in iXsystems/github-truenas-runner, which installs one and +# fetches this from here: how a TrueNAS appliance runs under qemu, and the +# admin it is built with. Sourced, not run. + +# The admin the image is built with, and its password until a claim +# rotates it. Not a secret: the image is booted only by qemu inside a +# runner, its ports bound there alone, and every claim changes the +# password before handing the appliance out. +APPLIANCE_USER=truenas_admin +APPLIANCE_TEMPLATE_PASSWORD=truenas-appliance-template + +APPLIANCE_LIB="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TNCTL=(python3 "$APPLIANCE_LIB/tnctl.py") + +die() { echo "appliance: $*" >&2; exit 1; } +say() { echo "appliance: $*" >&2; } + +# ovmf: the UEFI firmware, as Debian and Ubuntu install it. +ovmf() { + local d try + for try in 1 2; do + for d in /usr/share/OVMF /usr/share/ovmf /usr/share/qemu; do + if [ -r "$d/OVMF_CODE_4M.fd" ] && [ -r "$d/OVMF_VARS_4M.fd" ]; then + OVMF_CODE="$d/OVMF_CODE_4M.fd" OVMF_VARS="$d/OVMF_VARS_4M.fd" + return + fi + done + [ "$try" = 1 ] && needs __no_such_command__ ovmf + done + die "no OVMF firmware (OVMF_CODE_4M.fd), even after installing ovmf" +} + +# needs ...: install what is missing, when this +# runner can (passwordless sudo and apt). +needs() { + local missing=() pkg cmd + while [ $# -gt 0 ]; do + cmd="$1" pkg="$2"; shift 2 + command -v "$cmd" >/dev/null || missing+=("$pkg") + done + [ ${#missing[@]} -eq 0 ] && return + if ! command -v apt-get >/dev/null || ! sudo -n true 2>/dev/null; then + die "missing ${missing[*]}, and no sudo and apt here to install them" + fi + say "installing ${missing[*]}" + sudo -n apt-get update -qq + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends "${missing[@]}" >/dev/null +} + +# vars_qcow2 : a fresh UEFI variable store, as qcow2. savevm snapshots +# every writable disk, and refuses one that is not qcow2: the variable +# store is one. +vars_qcow2() { + qemu-img convert -q -f raw -O qcow2 "$OVMF_VARS" "$1" +} + +# binds: where the appliance's forwarded ports listen. This machine's +# loopback, and Docker's bridge when there is one, so that a container +# reaches it as host.docker.internal; never the runner's network. +binds() { + echo 127.0.0.1 + ip -4 -o addr show dev docker0 2>/dev/null | awk '{sub(/\/.*/, "", $4); print $4}' +} + +# hostfwd : qemu's forward of it, on every bind. +hostfwd() { + local a out="" + for a in $(binds); do + out+=",hostfwd=tcp:$a:$2-:$1" + done + echo "$out" +} diff --git a/.github/actions/truenas-appliance-runner/test_tnctl.py b/.github/actions/truenas-appliance-runner/test_tnctl.py new file mode 100644 index 0000000..ffe7708 --- /dev/null +++ b/.github/actions/truenas-appliance-runner/test_tnctl.py @@ -0,0 +1,195 @@ +"""tnctl against a fake installer, middleware and QMP: python3 +test_tnctl.py. Standard library only, like tnctl.""" + +import base64 +import hashlib +import json +import os +import socket +import struct +import sys +import tempfile +import threading +import unittest +from types import SimpleNamespace + +sys.path.insert(0, os.path.dirname(__file__)) +import tnctl # noqa: E402 + + +class FakeServer: + """A websocket server answering JSON-RPC from handlers[method].""" + + def __init__(self, handlers): + self.handlers = handlers + self.calls = [] + self.paths = [] + self.sock = socket.socket() + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen() + self.url = f"ws://127.0.0.1:{self.sock.getsockname()[1]}" + threading.Thread(target=self.serve, daemon=True).start() + + def serve(self): + while True: + try: + conn, _ = self.sock.accept() + except OSError: + return + threading.Thread(target=self.handle, args=(conn,), daemon=True).start() + + def handle(self, conn): + f = conn.makefile("rb") + request = f.readline().decode() + self.paths.append(request.split()[1]) + key = "" + while (line := f.readline().decode().strip()): + if line.lower().startswith("sec-websocket-key:"): + key = line.split(":", 1)[1].strip() + accept = base64.b64encode(hashlib.sha1((key + tnctl.GUID).encode()).digest()).decode() + conn.sendall(f"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + f"Sec-WebSocket-Accept: {accept}\r\n\r\n".encode()) + + def send(obj, opcode=1): + payload = json.dumps(obj).encode() if opcode == 1 else obj + n = len(payload) + head = bytes([0x80 | opcode]) + head += bytes([n]) if n < 126 else (bytes([126]) + struct.pack("!H", n) if n < 1 << 16 else bytes([127]) + struct.pack("!Q", n)) + conn.sendall(head + payload) + + def read(): + b0, b1 = f.read(2) + n = b1 & 0x7F + if n == 126: + n = struct.unpack("!H", f.read(2))[0] + elif n == 127: + n = struct.unpack("!Q", f.read(8))[0] + mask = f.read(4) + return b0 & 0x0F, bytes(b ^ mask[i % 4] for i, b in enumerate(f.read(n))) + + try: + while True: + opcode, payload = read() + if opcode == 0x8: + return + if opcode != 1: + continue + msg = json.loads(payload) + self.calls.append((msg["method"], msg.get("params", []))) + # A ping before every answer: the client must pong and go on. + send(b"hi", opcode=0x9) + result = self.handlers[msg["method"]](*msg.get("params", []), notify=lambda m, p: send({"jsonrpc": "2.0", "method": m, "params": p})) + if isinstance(result, tnctl.RPCError): + send({"jsonrpc": "2.0", "id": msg["id"], "error": result.error}) + else: + send({"jsonrpc": "2.0", "id": msg["id"], "result": result}) + except (OSError, ValueError): + return + finally: + conn.close() + + def close(self): + self.sock.close() + + +def args(**kw): + return SimpleNamespace(timeout=10, **kw) + + +class TestTnctl(unittest.TestCase): + def test_install(self): + notes = [] + + def install(opts, notify): + notify("installation_progress", [{"progress": 0.5, "message": "half"}]) + notes.append(opts) + return None + + srv = FakeServer({ + "is_adopted": lambda notify: False, + "list_disks": lambda notify: [{"name": "vda"}, {"name": "vdb"}], + "adopt": lambda notify: None, + "install": install, + }) + self.addCleanup(srv.close) + tnctl.cmd_install(args(url=srv.url, disk="")) + self.assertEqual(srv.paths, ["/ws"]) + self.assertEqual(notes, [{"disks": ["vda"], "set_pmbr": True, "authentication": None}]) + + def test_wait_until_ready(self): + ready = iter([False, False, True]) + srv = FakeServer({ + "auth.login_ex": lambda creds, notify: {"response_type": "SUCCESS" if creds["password"] == "pw" else "AUTH_ERR"}, + "system.ready": lambda notify: next(ready), + # Bigger than one short frame length can say, and than two. + "system.info": lambda notify: {"blob": "x" * 70000}, + }) + self.addCleanup(srv.close) + tnctl.cmd_wait(args(url=srv.url, user="truenas_admin", password="pw")) + self.assertEqual([m for m, _ in srv.calls], ["auth.login_ex", "system.ready", "system.ready", "system.ready"]) + self.assertEqual(srv.paths, ["/api/current"]) + c = tnctl.login(srv.url, "truenas_admin", "pw") + self.assertEqual(len(c.call("system.info")["blob"]), 70000) + c.close() + with self.assertRaises(tnctl.RPCError): + tnctl.login(srv.url, "truenas_admin", "wrong") + + def test_errors_and_jobs(self): + states = iter(["RUNNING", "SUCCESS"]) + srv = FakeServer({ + "auth.login_ex": lambda creds, notify: {"response_type": "SUCCESS"}, + "user.query": lambda *a, notify: {"id": 1} if len(a) > 1 else [{"id": 1}], + "user.update": lambda *a, notify: tnctl.RPCError("user.update", {"code": 22, "message": "nope"}), + "service.update": lambda *a, notify: None, + "service.control": lambda *a, notify: 42, + "core.get_jobs": lambda *a, notify: [{"id": 42, "state": next(states), "result": True}], + }) + self.addCleanup(srv.close) + with self.assertRaises(tnctl.RPCError): + tnctl.cmd_rotate(args(url=srv.url, user="truenas_admin", password="pw", new="new")) + c = tnctl.login(srv.url, "u", "p") + self.assertTrue(c.job("service.control", "START", "ssh")) + c.close() + + def test_qmp(self): + path = os.path.join(tempfile.mkdtemp(), "qmp") + srv = socket.socket(socket.AF_UNIX) + srv.bind(path) + srv.listen() + seen = [] + + def serve(): + # One connection per call, as tnctl makes them. + while True: + try: + conn, _ = srv.accept() + except OSError: + return + f = conn.makefile("rwb") + f.write(b'{"QMP": {"version": {}}}\n') + f.flush() + for line in f: + msg = json.loads(line) + seen.append(msg) + f.write(b'{"event": "STOP"}\n') + out = {"return": {}} + if msg["execute"] == "human-monitor-command": + out = {"return": "Error: no snapshot" if "nope" in msg["arguments"]["command-line"] else ""} + f.write(json.dumps(out).encode() + b"\n") + f.flush() + conn.close() + + threading.Thread(target=serve, daemon=True).start() + self.addCleanup(srv.close) + self.assertEqual(tnctl.qmp(path, "human-monitor-command", {"command-line": "savevm fresh-install"}), "") + self.assertEqual([m["execute"] for m in seen], ["qmp_capabilities", "human-monitor-command"]) + # HMP reports its own failures as text: cmd_hmp has to turn one into + # a non-zero exit, or revert would take a missing snapshot as done. + tnctl.cmd_hmp(args(socket=path, line="savevm fresh-install")) + with self.assertRaises(SystemExit) as e: + tnctl.cmd_hmp(args(socket=path, line="loadvm nope")) + self.assertEqual(e.exception.code, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/actions/truenas-appliance-runner/tnctl.py b/.github/actions/truenas-appliance-runner/tnctl.py new file mode 100755 index 0000000..6cc6a2e --- /dev/null +++ b/.github/actions/truenas-appliance-runner/tnctl.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python3 +"""tnctl: drive a TrueNAS appliance that runs under a local qemu. + +Standard library only, so it runs on any runner with a python3: the +installer's and the middleware's JSON-RPC 2.0 over a websocket, and qemu's +QMP over a unix socket. appliance.sh calls it; the image build does too. + + tnctl.py install URL [--disk NAME] + Drive the installer ISO at ws://.../ws: install to the first disk it + lists (or NAME), with the admin left for setup-admin. Returns when the + install is done; the caller powers the VM off and boots the disk. + + tnctl.py setup-admin URL USER PASSWORD [--timeout S] + Once the installed system is up: create its admin. Only a system + with no admin yet allows this, unauthenticated. + + tnctl.py wait URL USER PASSWORD [--timeout S] + Until USER can log in and system.ready is true; prints the seconds. + + tnctl.py rotate URL USER PASSWORD NEW + tnctl.py enable-ssh URL USER PASSWORD + tnctl.py call URL USER PASSWORD METHOD [JSON-PARAMS...] + One call as USER, its result as JSON on stdout. + tnctl.py wait-verify URL USER PASSWORD [--timeout S] + Until the first boot's truenas_verify baseline log exists: middleware + writes it once, after system.ready, and anything changed before it + would be reported as tampering from then on. + + tnctl.py qmp SOCKET COMMAND [JSON-ARGS] + tnctl.py hmp SOCKET "savevm fresh-install" + One QMP command, or a human monitor command through QMP. + +URL is ws://host:port; the path (/ws for the installer, /api/current for +the middleware) is added here. +""" + +import argparse +import base64 +import hashlib +import json +import os +import socket +import struct +import sys +import time +import urllib.parse + +GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + +class Closed(Exception): + """The websocket is gone.""" + + +class WebSocket: + """A websocket client, enough for JSON-RPC: text messages, ping and + pong, close. ws:// only: the appliance is reached through qemu's port + forward on this machine.""" + + def __init__(self, url, timeout=30): + u = urllib.parse.urlsplit(url) + if u.scheme != "ws": + raise ValueError(f"{url}: only ws:// is supported") + self.sock = socket.create_connection((u.hostname, u.port or 80), timeout=timeout) + key = base64.b64encode(os.urandom(16)).decode() + path = (u.path or "/") + (f"?{u.query}" if u.query else "") + self.sock.sendall( + f"GET {path} HTTP/1.1\r\nHost: {u.hostname}:{u.port or 80}\r\n" + "Upgrade: websocket\r\nConnection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode() + ) + head = b"" + while b"\r\n\r\n" not in head: + chunk = self.sock.recv(4096) + if not chunk: + raise Closed(f"{url}: closed during the handshake") + head += chunk + head, self.buf = head.split(b"\r\n\r\n", 1) + lines = head.decode("latin-1").split("\r\n") + status = lines[0].split(" ", 2) + if len(status) < 2 or status[1] != "101": + raise Closed(f"{url}: {lines[0]}") + headers = {k.strip().lower(): v.strip() for k, v in (l.split(":", 1) for l in lines[1:] if ":" in l)} + want = base64.b64encode(hashlib.sha1((key + GUID).encode()).digest()).decode() + if headers.get("sec-websocket-accept") != want: + raise Closed(f"{url}: bad Sec-WebSocket-Accept") + + def _read(self, n): + while len(self.buf) < n: + chunk = self.sock.recv(65536) + if not chunk: + raise Closed("connection closed") + self.buf += chunk + out, self.buf = self.buf[:n], self.buf[n:] + return out + + def _send(self, opcode, payload): + head = bytes([0x80 | opcode]) + n = len(payload) + if n < 126: + head += bytes([0x80 | n]) + elif n < 1 << 16: + head += bytes([0x80 | 126]) + struct.pack("!H", n) + else: + head += bytes([0x80 | 127]) + struct.pack("!Q", n) + mask = os.urandom(4) + self.sock.sendall(head + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))) + + def send(self, text): + self._send(0x1, text.encode()) + + def recv(self, timeout=None): + self.sock.settimeout(timeout) + message = b"" + while True: + b0, b1 = self._read(2) + opcode, n = b0 & 0x0F, b1 & 0x7F + if n == 126: + n = struct.unpack("!H", self._read(2))[0] + elif n == 127: + n = struct.unpack("!Q", self._read(8))[0] + mask = self._read(4) if b1 & 0x80 else None + payload = self._read(n) + if mask: + payload = bytes(b ^ mask[i % 4] for i, b in enumerate(payload)) + if opcode == 0x8: + raise Closed("closed by the server") + if opcode == 0x9: + self._send(0xA, payload) + continue + if opcode == 0xA: + continue + message += payload + if b0 & 0x80: + return message.decode() + + def close(self): + try: + self._send(0x8, b"") + except OSError: + pass + self.sock.close() + + +class RPCError(Exception): + def __init__(self, method, error): + super().__init__(f"{method}: {json.dumps(error)}") + self.error = error + + +class RPC: + """JSON-RPC 2.0 over a websocket; notifications go to on_notify.""" + + def __init__(self, url, timeout=30, on_notify=None): + self.ws = WebSocket(url, timeout) + self.id = 0 + self.on_notify = on_notify or (lambda method, params: None) + + def call(self, method, *params, timeout=120): + self.id += 1 + msg = {"jsonrpc": "2.0", "method": method, "id": self.id} + if params: + msg["params"] = list(params) + self.ws.send(json.dumps(msg)) + deadline = time.monotonic() + timeout + while True: + left = deadline - time.monotonic() + if left <= 0: + raise TimeoutError(f"{method}: no answer in {timeout}s") + data = json.loads(self.ws.recv(left)) + if data.get("id") == self.id: + if "error" in data: + raise RPCError(method, data["error"]) + return data.get("result") + if "method" in data: + self.on_notify(data["method"], data.get("params")) + + def job(self, method, *params, timeout=600): + """A middleware job: its id, then its result once it ends.""" + jid = self.call(method, *params) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + jobs = self.call("core.get_jobs", [["id", "=", jid]]) + if jobs and jobs[0]["state"] == "SUCCESS": + return jobs[0]["result"] + if jobs and jobs[0]["state"] in ("FAILED", "ABORTED"): + raise RPCError(method, jobs[0].get("error")) + time.sleep(1) + raise TimeoutError(f"{method}: job {jid} not done in {timeout}s") + + def close(self): + self.ws.close() + + +def log(*a): + print("tnctl:", *a, file=sys.stderr, flush=True) + + +def api(url): + return url.rstrip("/") + "/api/current" + + +def login(url, user, password, timeout=30): + c = RPC(api(url), timeout) + r = c.call("auth.login_ex", {"mechanism": "PASSWORD_PLAIN", "username": user, "password": password}) + if (r or {}).get("response_type") != "SUCCESS": + c.close() + raise RPCError("auth.login_ex", r) + return c + + +def retry(what, timeout, fn, every=3): + """fn() until it returns, for up to timeout seconds; its last error + is the one raised.""" + deadline = time.monotonic() + timeout + while True: + try: + return fn() + except (OSError, Closed, RPCError, TimeoutError, ValueError) as e: + if time.monotonic() >= deadline: + raise TimeoutError(f"{what}: not in {timeout}s; last: {e}") from e + time.sleep(every) + + +def cmd_install(a): + def progress(method, params): + if method == "installation_progress": + p = (params or [{}])[0] + try: + pct = f"{float(p.get('progress', 0)) * 100:.0f}%" + except (TypeError, ValueError): + pct = "?" + log("install", pct, p.get("message", "")) + + # nginx answers on 80 before the installer behind it does. + c = retry("the installer", a.timeout, lambda: RPC(a.url.rstrip("/") + "/ws", 30, progress), every=5) + try: + if c.call("is_adopted"): + raise SystemExit("the installer is adopted by another client already") + disks = [d["name"] for d in c.call("list_disks") or []] + log("installer sees disks", disks) + disk = a.disk or (disks[0] if disks else None) + if not disk: + raise SystemExit("the installer sees no disk") + c.call("adopt") + log("installing to", disk) + # authentication None: the admin is made after the first boot, by + # setup-admin, as the web UI's first-run page would. + c.call("install", {"disks": [disk], "set_pmbr": True, "authentication": None}, timeout=a.timeout) + log("install done") + finally: + c.close() + + +def cmd_setup_admin(a): + def setup(): + c = RPC(api(a.url), 30) + try: + c.call("user.setup_local_administrator", a.user, a.password, timeout=120) + finally: + c.close() + + retry("the middleware", a.timeout, setup, every=5) + log("admin", a.user, "set up") + + +def cmd_wait(a): + start = time.monotonic() + c = retry("a login", a.timeout, lambda: login(a.url, a.user, a.password), every=2) + def ready(): + if not c.call("system.ready"): + raise ValueError("not ready yet") + + try: + retry("system.ready", max(10, a.timeout - (time.monotonic() - start)), ready, every=2) + finally: + c.close() + print(f"{time.monotonic() - start:.1f}") + + +def cmd_wait_verify(a): + c = login(a.url, a.user, a.password) + try: + path = f"/var/log/audit/truenas_verify.{c.call('system.version')}.log" + retry(path, a.timeout, lambda: c.call("filesystem.stat", path)) + log("truenas_verify baseline written:", path) + finally: + c.close() + + +def cmd_rotate(a): + c = login(a.url, a.user, a.password) + try: + users = c.call("user.query", [["username", "=", a.user]]) + if not users: + raise SystemExit(f"no user {a.user}") + c.call("user.update", users[0]["id"], {"password": a.new}) + finally: + c.close() + + +def cmd_enable_ssh(a): + c = login(a.url, a.user, a.password) + try: + uid = c.call("user.query", [["username", "=", a.user]], {"get": True})["id"] + c.call("user.update", uid, {"ssh_password_enabled": True}) + c.call("service.update", "ssh", {"enable": True}) + c.job("service.control", "START", "ssh", {"silent": False}) + finally: + c.close() + + +def cmd_call(a): + c = login(a.url, a.user, a.password) + try: + print(json.dumps(c.call(a.method, *[json.loads(p) for p in a.params], timeout=a.timeout), indent=2)) + finally: + c.close() + + +def qmp(path, command, arguments=None, timeout=600): + s = socket.socket(socket.AF_UNIX) + s.settimeout(timeout) + s.connect(path) + f = s.makefile("rwb") + + def read(): + while True: + line = f.readline() + if not line: + raise Closed("qmp closed") + msg = json.loads(line) + if "event" not in msg: + return msg + + def send(cmd, args=None): + f.write(json.dumps({"execute": cmd, **({"arguments": args} if args else {})}).encode() + b"\n") + f.flush() + r = read() + if "error" in r: + raise RPCError(cmd, r["error"]) + return r.get("return") + + try: + read() # the greeting + send("qmp_capabilities") + return send(command, arguments) + finally: + s.close() + + +def cmd_qmp(a): + r = qmp(a.socket, a.command, json.loads(a.args) if a.args else None) + print(json.dumps(r, indent=2)) + + +def cmd_hmp(a): + out = qmp(a.socket, "human-monitor-command", {"command-line": a.line}) + # HMP reports its own errors as text, savevm and loadvm included. + if out and out.strip(): + print(out.strip()) + if "rror" in out: + raise SystemExit(1) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + + def p(name, fn, *args, timeout=600): + sp = sub.add_parser(name) + for arg in args: + sp.add_argument(arg) + sp.add_argument("--timeout", type=int, default=timeout) + sp.set_defaults(fn=fn) + return sp + + p("install", cmd_install, "url", timeout=3600).add_argument("--disk", default="") + p("setup-admin", cmd_setup_admin, "url", "user", "password") + p("wait", cmd_wait, "url", "user", "password") + p("wait-verify", cmd_wait_verify, "url", "user", "password", timeout=300) + p("rotate", cmd_rotate, "url", "user", "password", "new") + p("enable-ssh", cmd_enable_ssh, "url", "user", "password") + p("call", cmd_call, "url", "user", "password", "method", timeout=120).add_argument("params", nargs="*") + p("qmp", cmd_qmp, "socket", "command").add_argument("args", nargs="?") + p("hmp", cmd_hmp, "socket", "line") + a = ap.parse_args() + try: + a.fn(a) + except (TimeoutError, RPCError, Closed, OSError) as e: + raise SystemExit(f"tnctl {a.cmd}: {e}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4218673..e2819b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,6 +153,26 @@ jobs: sed -E 's/^\{\{file:(.*)\}\}$/\1/' | sort -u) exit "$missing" + # The runner-side appliance action's scripts. Nothing in this repo runs + # them, and github-truenas-runner's appliance image build fetches them + # from master: a broken one is a failed image build there, or an e2e job + # that cannot boot its appliance. + appliance-runner: + name: Check the appliance runner scripts + runs-on: ubuntu-latest + defaults: + run: + working-directory: .github/actions/truenas-appliance-runner + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: tnctl against a fake installer, middleware and QMP + run: python3 test_tnctl.py + + - name: shellcheck + run: shellcheck -x appliance.sh lib.sh + # Smoke test: this repo calls its own reusable workflow, so a change to # check-member.yml is executed before it can be merged rather than after. # diff --git a/.gitignore b/.gitignore index 214f1ae..e76f916 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ # CI installs js-yaml into the workspace to run scripts/check-input-refs.js node_modules/ +__pycache__/ diff --git a/README.md b/README.md index 4ecb22b..048062c 100644 --- a/README.md +++ b/README.md @@ -523,6 +523,54 @@ What the runner has to provide, the timings, and the lessons behind every rule in the script are in `truenas/webui`'s `e2e/docs/05-ci.md`, which is where this was built and first used. +### `.github/actions/truenas-appliance-runner` + +The other way to give a job an appliance, being prototyped: **the appliance +is inside the runner**. The job runs on an ephemeral GARM runner from +`iXsystems/github-truenas-runner`'s TrueNAS provider, whose pool attaches a +copy-on-write clone of a prebuilt TrueNAS image as a second disk; this +action boots that disk under qemu, inside the runner, rotates its admin +password, snapshots it as `fresh-install`, and exports the suite's `TN_*` +variables as the claim above does, less `TN_GUEST_ISO` (the appliance is an +image, not an ISO), and with `TN_SSH` when `ssh` is `true`. There is nothing to claim and nothing to +release: picking the runner label is the claim, and GARM deleting the +runner is the release. Capacity is the pool's size; jobs queue for a runner +instead of a concurrency group, and sharding is a matrix. + +```yaml +jobs: + e2e: + runs-on: [self-hosted, truenas-appliance] + steps: + - uses: iXsystems/ux-github-workflows/.github/actions/truenas-appliance-runner@master + id: appliance + - run: yarn e2e + # between tests, back to the baseline in seconds, memory included: + - run: truenas-appliance revert fresh-install +``` + +| Input | Default | Notes | +|---|---|---| +| `baseline` | `fresh-install` | Snapshot (`savevm`) taken once it is up; empty takes none | +| `memory-mb`, `vcpus` | `6144`, `4` | The appliance's; the runner's flavor needs room for it and the job | +| `data-disk-count`, `data-disk-gb` | `10`, `10` | Sparse qcow2 files | +| `ssh` | `'false'` | `true` enables ssh for the admin, on `TN_SSH` | +| `disk` | the attached one | Found by its serial, `virtio-truenas_appliance`, where the box keeps it; TrueNAS replaces it, so otherwise the one disk besides the root disk that holds a ZFS `boot-pool` | +| `export-env` | `'true'` | Write `TN_*` to the job environment | + +Outputs: `host`, `host-http`, `username`, `password` (masked), `ssh`, +`boot-seconds`, `snapshot-seconds`. `truenas-appliance` is put on `PATH`: +`snapshot `, `revert `, `call [params...]`, +`snapshots`, `console [lines]`, `probe`. + +These scripts are the one copy. github-truenas-runner's image pipeline +fetches them from here, at the last commit that changed this directory, to +install the appliance image from the nightly ISO and test-boot it the way a +runner will; a change here is therefore a new appliance image there. What +the two must agree on, the admin the image is built with and how qemu boots +it, is `lib.sh`. The design, the pool setup and the numbers to measure are +in github-truenas-runner's `docs/e2e-appliances.md`. + ## Adoption status | Repo | `check-ticket` | `pr-title` | `check-member` | `prepare` | review |