From b219f391041f3efde44e49927bb71f6fe9d05821 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Thu, 15 Oct 2026 12:05:12 +0100 Subject: [PATCH 01/11] Fix Maestro onboarding and route flows Onboarding's Navigating and Battery screens skip themselves when there is nothing to grant (since cb3c90d5d), so tapping their buttons unconditionally failed the flow and every flow after it. Only tap them if the screen shows, and don't expect the Battery screen's Continue after the exemption is granted, since that moves on by itself. Markers.yaml saves C at the current location, where LocationDetails.yaml already saved A, and saveMarker merges a new marker into one at the same coordinates, so C replaces A. Assert that, and build the route in RouteCreation.yaml from B and C. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ex6Fdp5ftuhE1ZEyhZb7WQ --- maestro/Markers.yaml | 9 ++++-- maestro/Onboarding.yaml | 63 ++++++++++++++++++++++++++------------ maestro/RouteCreation.yaml | 10 +++--- 3 files changed, 55 insertions(+), 27 deletions(-) diff --git a/maestro/Markers.yaml b/maestro/Markers.yaml index a0d6e6129..74acde74e 100644 --- a/maestro/Markers.yaml +++ b/maestro/Markers.yaml @@ -1,6 +1,7 @@ appId: org.scottishtecharmy.soundscape # Exercises the Markers screen: the sort controls, editing an existing marker, -# and creating marker "C" at the current location with the New button. +# and creating marker "C" at the current location with the New button, which +# replaces "A" (saved at the same location). # # Depends on markers "A" and "B" having been created by LocationDetails.yaml, so # this flow must run after it in the suite. @@ -63,6 +64,10 @@ appId: org.scottishtecharmy.soundscape - tapOn: id: flexibleAppBarRight -# Saving returns to the Markers screen, which now lists C. +# Saving returns to the Markers screen, which now lists C. A was also saved at +# the current location (LocationDetails.yaml), and saveMarker merges a new +# marker into an existing one at exactly the same coordinates, so C replaces A. - assertVisible: id: "LocationItem-C-.*" +- assertNotVisible: + id: "LocationItem-A-.*" diff --git a/maestro/Onboarding.yaml b/maestro/Onboarding.yaml index 68ef7b6f8..d1d3afab0 100644 --- a/maestro/Onboarding.yaml +++ b/maestro/Onboarding.yaml @@ -31,44 +31,67 @@ appId: org.scottishtecharmy.soundscape env: id: "languageScreenContinueButton" +# The Navigating (permissions) and Battery screens skip themselves when there is +# nothing to grant. launchApp's permissions block above always grants the +# Navigating screen's permissions, and the battery exemption can survive +# clearState, so either screen may never show. The fixed wait gives a skipping +# screen time to move on before checking whether it is still there. - runFlow: when: platform: Android - file: SwipeAndTap.yaml + file: Wait.yaml env: - id: "navigatingScreenContinueButton" + timeout: 2000 -# Battery optimization screen. Tapping "Grant permission" launches a system -# dialog (ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS) which is not a runtime -# permission, so it is not handled by the launchApp permissions block. We wait -# for the system dialog's positive button and tap it, then continue. - runFlow: when: - platform: Android + visible: + id: "navigatingScreenContinueButton" file: SwipeAndTap.yaml env: - id: "batteryOptimizationGrantPermissionButton" + id: "navigatingScreenContinueButton" - runFlow: when: platform: Android file: Wait.yaml env: - timeout: 3000 - -# The system battery-optimisation dialog only appears if the app isn't already -# exempt. Dismiss its "Allow" button if it shows; if it never appears, optional -# lets the flow carry on instead of failing. -- tapOn: - id: "android:id/button1" - optional: true + timeout: 2000 +# Battery optimization screen. Tapping "Grant permission" launches a system +# dialog (ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS) which is not a runtime +# permission, so it is not handled by the launchApp permissions block. We wait +# for the system dialog's positive button and tap it, then continue. - runFlow: when: - platform: Android - file: SwipeAndTap.yaml - env: - id: "batteryOptimizationContinueButton" + visible: + id: "batteryOptimizationGrantPermissionButton" + commands: + - runFlow: + file: SwipeAndTap.yaml + env: + id: "batteryOptimizationGrantPermissionButton" + - runFlow: + file: Wait.yaml + env: + timeout: 3000 + # The system dialog only appears if the app isn't already exempt. + - tapOn: + id: "android:id/button1" + optional: true + - runFlow: + file: Wait.yaml + env: + timeout: 2000 + # Granting the exemption moves on to the next screen by itself; Continue + # is only left to tap if it was refused. + - runFlow: + when: + visible: + id: "batteryOptimizationContinueButton" + file: SwipeAndTap.yaml + env: + id: "batteryOptimizationContinueButton" - runFlow: file: SwipeAndTap.yaml diff --git a/maestro/RouteCreation.yaml b/maestro/RouteCreation.yaml index 6091cf50f..f764037b6 100644 --- a/maestro/RouteCreation.yaml +++ b/maestro/RouteCreation.yaml @@ -2,8 +2,8 @@ appId: org.scottishtecharmy.soundscape # Creates a new route from the Routes screen, adds two existing markers as # waypoints, saves it, and verifies it appears in the routes list. # -# Depends on markers "A" and "B" having been created by LocationDetails.yaml, so -# this flow must run after it in the suite. +# Depends on marker "B" from LocationDetails.yaml and marker "C" from +# Markers.yaml (which replaces "A"), so this flow must run after both. --- - launchApp: clearState: false @@ -28,14 +28,14 @@ appId: org.scottishtecharmy.soundscape - inputText: "Test Route" - pressKey: Enter -# Add the existing markers A and B as waypoints. They are listed by name; the +# Add the existing markers B and C as waypoints. They are listed by name; the # orderId suffix is unknown here so match by name prefix. - tapOn: id: addWaypointsButton -- tapOn: - id: "LocationItem-A-.*" - tapOn: id: "LocationItem-B-.*" +- tapOn: + id: "LocationItem-C-.*" # "Done" in the waypoints dialog (reuses the shared app-bar right button). - tapOn: id: flexibleAppBarRight From b4524e6284c722c366a2e89622dbded82934fef9 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Thu, 15 Oct 2026 12:05:12 +0100 Subject: [PATCH 02/11] Add a local test farm for release builds on physical phones test-farm/farm.py runs one worker per phone plugged into the PC. Each loops: upgrade to the newest release (install -r over existing data, then a PostUpgrade check), a monkey soak, the Maestro suite, then collects crashes, ANRs, memory and battery into farm.db and report.html. Releases come from the upstream build-app.yaml release-apk artifact. Release builds are made testable without debug-only hooks: firebase.test.lab turns analytics off, and shell mock location providers replay a GPX track (held at CI's Edinburgh point while the suite runs). The farm only tests a build it installed itself, and dims the screen and mutes media volume. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ex6Fdp5ftuhE1ZEyhZb7WQ --- test-farm/.gitignore | 8 + test-farm/README.md | 71 +++ test-farm/farm.py | 734 +++++++++++++++++++++++++++++++ test-farm/farm.toml.example | 37 ++ test-farm/flows/PostUpgrade.yaml | 40 ++ 5 files changed, 890 insertions(+) create mode 100644 test-farm/.gitignore create mode 100644 test-farm/README.md create mode 100644 test-farm/farm.py create mode 100644 test-farm/farm.toml.example create mode 100644 test-farm/flows/PostUpgrade.yaml diff --git a/test-farm/.gitignore b/test-farm/.gitignore new file mode 100644 index 000000000..e7f78a8ed --- /dev/null +++ b/test-farm/.gitignore @@ -0,0 +1,8 @@ +farm.toml +farm.db +report.html +report.html.tmp +apks/ +runs/ +__pycache__/ +farm.log diff --git a/test-farm/README.md b/test-farm/README.md new file mode 100644 index 000000000..96b3e39b8 --- /dev/null +++ b/test-farm/README.md @@ -0,0 +1,71 @@ +# Test farm + +`farm.py` runs long-term tests of **release builds** on every Android phone +plugged into this PC. Each phone loops forever: + +1. If a newer release has been built, `adb install -r` it over the existing + app and data, then run `flows/PostUpgrade.yaml` to check the app starts and + the suite's markers and route survived. +2. A `monkey` soak (random input inside the app, fixed seed per run, logged). +3. The Maestro suite in `maestro/Master.yaml` order, one flow at a time. +4. Collect crashes and ANRs (from dropbox), logcat, memory (PSS) and battery. + +Releases come from the upstream `build-app.yaml` workflow's `release-apk` +artifact (the APK that goes to Play), via `gh`. The newest two are cached in +`apks/`, and a freshly attached phone starts one release behind so its first +upgrade happens after its first cycle. Artifacts expire after 90 days, so +leave the farm running (or keep `apks/`) to keep older releases. + +Results: `report.html` (refreshes every minute), `farm.db` (SQLite), and +per-cycle files under `runs///`. + +```bash +cp farm.toml.example farm.toml # optional +python3 test-farm/farm.py # Ctrl-C to stop; removes the mock GPS providers +``` + +## How release builds are made testable + +- **Analytics:** the farm sets `settings put system firebase.test.lab true`, which + makes `MainActivity` use dummy analytics, as it does in Firebase Test Lab. +- **Location:** release builds don't have the debug `REPLAY_GPX` intent, so the + farm adds shell mock providers for `gps` and `network` + (`cmd location providers ...`) and feeds them a GPX track once a second. + This reaches both the fused provider (phones with Play services) and plain + `LocationManager` (AOSP phones). While the Maestro suite runs, the phone is + held at `suite_location` (CI's Edinburgh point), because the flows assert + distance-sorted list order. +- **Maestro:** `testTagsAsResourceId` is set in `Theme.kt` for every build type. + +## Preparing a phone + +- Developer options on, USB debugging on, and on Xiaomi/Oppo-type phones also + "USB debugging (Security settings)" so adb can tap and install. +- Screen lock **None**. Maestro can't get past a PIN. +- Uninstall any existing Soundscape first. The farm only tests a build it + installed itself (it records the install time), so a Play Store or debug + copy leaves the phone **blocked** and untested, unless + `allow_reinstall = true`, which uninstalls the copy and its data. +- Turn off Play Store auto-updates, so Play doesn't replace the farm's build. +- Charge limit (e.g. 80%) where the phone has one. A phone on charge 24/7 + without one will eventually swell its battery. +- Use a powered USB hub. + +The farm sets the screen to minimum brightness and media volume to zero when a +phone is attached and again after every monkey soak. The audio still plays at +zero volume, so the audio path is still tested. + +The farm deliberately leaves the OEM battery savers alone. If one kills the +service, that shows up as a failure, which is a real user problem. + +## Known limits + +- The suite wipes app data (Onboarding uses `clearState`), so a phone's data + only lives from the end of one cycle to the start of the next. That window + is where upgrades happen. +- The monkey can open the notification shade. After each soak the farm turns + off airplane mode, turns on Wi-Fi and collapses the shade. +- Maestro runs on several phones at once. One run lost its driver connection + (`gRPC UNAVAILABLE`) when another phone's session was starting, and that + error is now retried once. If it keeps happening, set + `serialize_maestro = true`. diff --git a/test-farm/farm.py b/test-farm/farm.py new file mode 100644 index 000000000..b5c0a80b3 --- /dev/null +++ b/test-farm/farm.py @@ -0,0 +1,734 @@ +#!/usr/bin/env python3 +"""Soundscape Android test farm. + +Runs long-term tests of release builds on every Android phone plugged into this +PC. One worker thread per phone loops forever: + + upgrade (if a newer release exists) -> PostUpgrade check + monkey soak -> the Maestro suite (maestro/Master.yaml order) + collect crashes/ANRs, memory, battery, logcat + +New releases are picked up from the upstream "Tag and build release" workflow's +release-apk artifact (the same APK that goes to Play) and installed with +`adb install -r` over the phone's existing data, so the real upgrade path is +exercised. Location is driven over adb with a shell mock provider replaying a +GPX track, because release builds don't have the debug REPLAY_GPX intent. + +Results go into farm.db and report.html next to this script. See README.md. +""" + +import argparse +import contextlib +import datetime as dt +import glob +import html +import math +import os +import re +import signal +import sqlite3 +import subprocess +import sys +import threading +import time +import tomllib +import xml.etree.ElementTree as ET + +FARM_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_DIR = os.path.dirname(FARM_DIR) +MAESTRO_DIR = os.path.join(REPO_DIR, "maestro") +POST_UPGRADE_FLOW = os.path.join(FARM_DIR, "flows", "PostUpgrade.yaml") +APK_DIR = os.path.join(FARM_DIR, "apks") +RUNS_DIR = os.path.join(FARM_DIR, "runs") +DB_PATH = os.path.join(FARM_DIR, "farm.db") +REPORT_PATH = os.path.join(FARM_DIR, "report.html") + +PACKAGE = "org.scottishtecharmy.soundscape" +MOCK_PROVIDERS = ("gps", "network") + +DEFAULTS = { + "release_repo": "Scottish-Tech-Army/Soundscape-Android", + "release_workflow": "build-app.yaml", + "release_artifact": "release-apk", + "poll_minutes": 15, + "monkey_events": 5000, + "monkey_throttle_ms": 250, + "flow_timeout_minutes": 15, + "keep_days": 14, + "serialize_maestro": False, + "allow_reinstall": False, + "include_emulators": False, + "max_battery_temp_c": 42, + "replay_speed_mps": 1.4, + "default_location": [55.9474, -4.3058], + # Held while the Maestro suite runs: the flows assert distance-sorted list + # order, so they need the fixed point CI pins with `adb emu geo fix`. + "suite_location": [55.955360, -3.223538], + "default_gpx": "", + "devices": {}, +} + +# Same transient launch failures CI retries once (see run-maestro-tests.yaml). +MAESTRO_RETRY_RE = re.compile( + r"Unable to launch app|device offline|Unknown error|Not able to reach the gRPC server") + +stop_event = threading.Event() +maestro_lock = threading.Lock() +state_lock = threading.Lock() +device_state = {} # serial -> {"state": str, "since": float, "model": str, ...} + + +def log(serial, msg): + stamp = dt.datetime.now().strftime("%H:%M:%S") + print(f"{stamp} [{serial or 'farm'}] {msg}", flush=True) + + +def set_state(serial, state, **extra): + with state_lock: + entry = device_state.setdefault(serial, {}) + entry.update(extra) + entry["state"] = state + entry["since"] = time.time() + log(serial, state) + + +# --------------------------------------------------------------------------- config + + +def load_config(path): + cfg = dict(DEFAULTS) + if path and os.path.exists(path): + with open(path, "rb") as f: + cfg.update(tomllib.load(f)) + base = os.path.dirname(os.path.abspath(path)) + else: + base = FARM_DIR + # GPX paths in the config are relative to the config file. + def resolve(p): + return os.path.normpath(os.path.join(base, p)) if p else "" + cfg["default_gpx"] = resolve(cfg["default_gpx"]) + for dev in cfg["devices"].values(): + if "gpx" in dev: + dev["gpx"] = resolve(dev["gpx"]) + return cfg + + +# --------------------------------------------------------------------------- database + + +class Db: + def __init__(self, path): + self.conn = sqlite3.connect(path, check_same_thread=False) + self.conn.row_factory = sqlite3.Row + self.lock = threading.Lock() + self.conn.executescript(""" + CREATE TABLE IF NOT EXISTS builds( + run_id INTEGER PRIMARY KEY, version_code INTEGER, version_name TEXT, + created_at TEXT, apk_path TEXT); + CREATE TABLE IF NOT EXISTS cycles( + id INTEGER PRIMARY KEY, serial TEXT, model TEXT, android TEXT, + version_code INTEGER, started REAL, ended REAL, status TEXT, + flows_passed INTEGER, flows_failed INTEGER, crashes INTEGER, anrs INTEGER, + pss_kb INTEGER, battery_level INTEGER, battery_temp REAL, dir TEXT); + CREATE TABLE IF NOT EXISTS results( + id INTEGER PRIMARY KEY, cycle_id INTEGER, serial TEXT, ts REAL, + kind TEXT, name TEXT, status TEXT, duration_s REAL, detail TEXT, dir TEXT); + CREATE TABLE IF NOT EXISTS installs( + serial TEXT PRIMARY KEY, last_update TEXT); + CREATE TABLE IF NOT EXISTS events( + id INTEGER PRIMARY KEY, serial TEXT, ts REAL, kind TEXT, + version_code INTEGER, detail TEXT); + """) + + def execute(self, sql, args=()): + with self.lock: + cur = self.conn.execute(sql, args) + self.conn.commit() + return cur.lastrowid + + def query(self, sql, args=()): + with self.lock: + return self.conn.execute(sql, args).fetchall() + + def event(self, serial, kind, version_code, detail): + log(serial, f"EVENT {kind}: {detail.splitlines()[0][:200] if detail else ''}") + self.execute("INSERT INTO events(serial, ts, kind, version_code, detail) VALUES(?,?,?,?,?)", + (serial, time.time(), kind, version_code, detail)) + + def builds(self): + return self.query("SELECT * FROM builds ORDER BY version_code") + + +# --------------------------------------------------------------------------- adb + + +def run(cmd, timeout=120): + try: + return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, + errors="replace") + except subprocess.TimeoutExpired as e: + return subprocess.CompletedProcess(cmd, -1, e.stdout or "", f"timeout after {timeout}s") + + +def adb(serial, *args, timeout=120): + return run(["adb", "-s", serial, *args], timeout=timeout) + + +def shell(serial, command, timeout=120): + return adb(serial, "shell", command, timeout=timeout).stdout.strip() + + +def connected_devices(include_emulators): + out = run(["adb", "devices"]).stdout + serials = [] + for line in out.splitlines()[1:]: + parts = line.split() + if len(parts) == 2 and parts[1] == "device": + if include_emulators or not parts[0].startswith("emulator-"): + serials.append(parts[0]) + return serials + + +def installed_version(serial): + out = shell(serial, f"dumpsys package {PACKAGE} | grep -m1 versionCode") + m = re.search(r"versionCode=(\d+)", out) + return int(m.group(1)) if m else None + + +def find_aapt2(): + sdk = os.environ.get("ANDROID_HOME") or os.path.expanduser("~/Android/Sdk") + tools = sorted(glob.glob(os.path.join(sdk, "build-tools", "*", "aapt2"))) + if not tools: + sys.exit("aapt2 not found: set ANDROID_HOME to your Android SDK") + return tools[-1] + + +# --------------------------------------------------------------------------- releases + + +def release_watcher(cfg, db): + """Caches the APKs of the two newest successful release builds.""" + aapt2 = find_aapt2() + known = {b["run_id"] for b in db.builds()} + while not stop_event.is_set(): + res = run(["gh", "run", "list", "-R", cfg["release_repo"], "-w", cfg["release_workflow"], + "-s", "success", "-L", "2", "--json", "databaseId,createdAt", + "-q", '.[] | "\\(.databaseId) \\(.createdAt)"']) + if res.returncode != 0: + log(None, f"gh run list failed: {res.stderr.strip()}") + for line in res.stdout.split("\n"): + if not line.strip(): + continue + run_id, created = line.split() + if int(run_id) in known: + continue + dest = os.path.join(APK_DIR, f"run-{run_id}") + log(None, f"downloading release build from run {run_id}") + dl = run(["gh", "run", "download", run_id, "-R", cfg["release_repo"], + "-n", cfg["release_artifact"], "-D", dest], timeout=900) + apks = glob.glob(os.path.join(dest, "**", "*.apk"), recursive=True) + if dl.returncode != 0 or not apks: + log(None, f"download of run {run_id} failed: {dl.stderr.strip()}") + continue + badging = run([aapt2, "dump", "badging", apks[0]]).stdout + m = re.search(r"versionCode='(\d+)' versionName='([^']*)'", badging) + if not m: + log(None, f"could not read version from {apks[0]}") + continue + db.execute("INSERT INTO builds VALUES(?,?,?,?,?)", + (int(run_id), int(m.group(1)), m.group(2), created, apks[0])) + known.add(int(run_id)) + log(None, f"new release {m.group(2)} ({m.group(1)}) cached") + stop_event.wait(cfg["poll_minutes"] * 60) + + +# --------------------------------------------------------------------------- location + + +def load_track(path): + root = ET.parse(path).getroot() + points = [(float(p.get("lat")), float(p.get("lon"))) + for p in root.iter() if p.tag.split("}")[-1] in ("trkpt", "rtept")] + if len(points) < 2: + raise ValueError(f"{path}: need at least 2 track points") + return points + + +def haversine(a, b): + lat1, lon1, lat2, lon2 = map(math.radians, (*a, *b)) + h = (math.sin((lat2 - lat1) / 2) ** 2 + + math.cos(lat1) * math.cos(lat2) * math.sin((lon2 - lon1) / 2) ** 2) + return 2 * 6371000 * math.asin(math.sqrt(h)) + + +class LocationFeeder(threading.Thread): + """Feeds mock fixes to the phone once a second over adb. + + Walks the GPX track at a constant speed and turns round at each end rather + than jumping back to the start. With no track, holds one fixed point. + Setting `hold` to a (lat, lon) pins the phone there until it's set to None. + """ + + def __init__(self, serial, gpx, fixed, speed, db, alive): + super().__init__(daemon=True) + self.serial, self.speed, self.db, self.alive = serial, speed, db, alive + self.hold = None + self.points = load_track(gpx) if gpx else [tuple(fixed), tuple(fixed)] + self.cumulative = [0.0] + for a, b in zip(self.points, self.points[1:]): + self.cumulative.append(self.cumulative[-1] + haversine(a, b)) + + def position(self, t): + total = self.cumulative[-1] + if total == 0: + return self.points[0] + d = (self.speed * t) % (2 * total) + if d > total: + d = 2 * total - d + i = max(j for j, c in enumerate(self.cumulative) if c <= d) + if i == len(self.points) - 1: + return self.points[-1] + seg = self.cumulative[i + 1] - self.cumulative[i] + f = (d - self.cumulative[i]) / seg if seg else 0 + (la, lo), (lb, lob) = self.points[i], self.points[i + 1] + return la + (lb - la) * f, lo + (lob - lo) * f + + def run(self): + start = time.time() + while not stop_event.is_set() and self.alive.is_set(): + lat, lon = self.hold or self.position(time.time() - start) + cmd = "; ".join( + f"cmd location providers set-test-provider-location {p} " + f"--location {lat:.7f},{lon:.7f} --accuracy 5" for p in MOCK_PROVIDERS) + out = shell(self.serial, cmd, timeout=10) + if "Exception" in out: + self.db.event(self.serial, "error", None, f"mock location rejected:\n{out}") + return + time.sleep(1) + + +def enable_mock_location(serial): + cmds = ["appops set com.android.shell android:mock_location allow"] + for p in MOCK_PROVIDERS: + cmds += [f"cmd location providers add-test-provider {p}", + f"cmd location providers set-test-provider-enabled {p} true"] + return shell(serial, "; ".join(cmds)) + + +def disable_mock_location(serial): + shell(serial, "; ".join(f"cmd location providers remove-test-provider {p}" + for p in MOCK_PROVIDERS), timeout=20) + + +# --------------------------------------------------------------------------- crashes + + +DROPBOX_TAGS = ("data_app_crash", "data_app_anr", "data_app_native_crash", "SYSTEM_TOMBSTONE") + + +def collect_dropbox(serial, since): + """Returns (kind, text) for every crash/ANR of our app logged at or after `since`. + + `since` is a device-local "YYYY-MM-DD HH:MM:SS" string, which is also the + format dropbox prints entries with, so plain string comparison works. + """ + out = shell(serial, "dumpsys dropbox --print " + " ".join(DROPBOX_TAGS), timeout=60) + found = [] + for entry in re.split(r"^={10,}\s*$", out, flags=re.M): + m = re.search(r"^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d) (\S+)", entry.strip(), re.M) + if m and m.group(1) >= since and PACKAGE in entry: + kind = "anr" if "anr" in m.group(2) else "crash" + found.append((kind, entry.strip())) + return found + + +def pss_kb(serial): + out = shell(serial, f"dumpsys meminfo {PACKAGE}", timeout=60) + m = re.search(r"TOTAL PSS:\s+(\d+)", out) or re.search(r"^\s*TOTAL\s+(\d+)", out, re.M) + return int(m.group(1)) if m else None + + +def battery(serial): + out = shell(serial, "dumpsys battery") + level = re.search(r"level: (\d+)", out) + temp = re.search(r"temperature: (\d+)", out) + return (int(level.group(1)) if level else None, + int(temp.group(1)) / 10 if temp else None) + + +# --------------------------------------------------------------------------- worker + + +def suite_flows(): + """The suite in Master.yaml order, which is the order the flows depend on.""" + with open(os.path.join(MAESTRO_DIR, "Master.yaml")) as f: + return re.findall(r"^- runFlow: (\S+\.yaml)", f.read(), re.M) + + +class Worker(threading.Thread): + def __init__(self, serial, cfg, db): + super().__init__(daemon=True, name=serial) + self.serial, self.cfg, self.db = serial, cfg, db + self.dev_cfg = cfg["devices"].get(serial, {}) + self.alive = threading.Event() + self.alive.set() + self.model = shell(serial, "getprop ro.product.model") + self.android = shell(serial, "getprop ro.build.version.release") + self.last_cycle_ok = False + self.feeder = None + + # ----- setup + + def prepare(self): + s = self.serial + shell(s, "svc power stayon usb") + # MainActivity uses dummy analytics when this is set, which keeps the + # farm's traffic out of the production Firebase numbers. + shell(s, "settings put system firebase.test.lab true") + adb(s, "logcat", "-G", "16M") + self.quieten() + out = enable_mock_location(s) + if "Exception" in out: + self.db.event(s, "error", None, f"could not enable mock location:\n{out}") + else: + gpx = self.dev_cfg.get("gpx", self.cfg["default_gpx"]) + self.feeder = LocationFeeder(s, gpx, self.cfg["default_location"], + self.cfg["replay_speed_mps"], self.db, self.alive) + self.feeder.start() + + def quieten(self): + """Dims the screen right down and turns media volume to zero.""" + s = self.serial + shell(s, "settings put system screen_brightness_mode 0; " + "settings put system screen_brightness 1; " + "cmd media_session volume --stream 3 --set 0") + # Some ROMs (the AOSP Samsung) ignore that. Volume-down keys work there + # because Soundscape is playing, so the keys adjust the media stream. + if "volume is 0 " not in shell(s, "cmd media_session volume --stream 3 --get"): + shell(s, "for i in $(seq 1 16); do input keyevent KEYCODE_VOLUME_DOWN; done") + + def repair(self): + """Undoes things the monkey can do from the notification shade.""" + shell(self.serial, "cmd connectivity airplane-mode disable; svc wifi enable; " + "cmd statusbar collapse; input keyevent KEYCODE_WAKEUP") + self.quieten() + + # ----- install / upgrade + + def last_update(self): + out = shell(self.serial, f"dumpsys package {PACKAGE} | grep -m1 lastUpdateTime") + return out.split("=", 1)[-1].strip() if out else None + + def farm_installed(self): + """True if the app on the phone is the one the farm last installed. + + Anything else (a Play Store install, a debug build, a Play update over + the farm's build) must not be tested as if it were a farm release. + """ + rows = self.db.query("SELECT last_update FROM installs WHERE serial=?", (self.serial,)) + return bool(rows) and rows[0]["last_update"] == self.last_update() + + def install(self, build, upgrade): + s = self.serial + before = installed_version(s) + res = adb(s, "install", "-r", build["apk_path"], timeout=600) + out = res.stdout + res.stderr + ok = res.returncode == 0 and "Success" in out + kind = "upgrade" if upgrade and before else "install" + if ok: + self.db.execute("INSERT OR REPLACE INTO installs VALUES(?,?)", (s, self.last_update())) + self.db.event(s, kind if ok else "error", build["version_code"], + f"{kind} {before} -> {build['version_name']} ({build['version_code']})" + + ("" if ok else "\n" + out)) + return ok + + def ensure_version(self, cycle_dir): + """Installs or upgrades as needed. Returns the version to test, or None.""" + builds = self.db.builds() + if not builds: + return None + latest = builds[-1] + current = installed_version(self.serial) + if current is not None and not self.farm_installed(): + if not self.cfg["allow_reinstall"]: + set_state(self.serial, f"blocked: Soundscape {current} on the phone wasn't " + "installed by the farm; uninstall it by hand") + return None + self.db.event(self.serial, "uninstall", current, + "uninstalling a Soundscape the farm didn't install (allow_reinstall)") + adb(self.serial, "uninstall", PACKAGE) + current = None + if current is None: + # Start one release behind when we can, so the first upgrade + # happens as soon as the first cycle is done. + set_state(self.serial, "installing") + self.install(builds[-2] if len(builds) > 1 else latest, upgrade=False) + elif current < latest["version_code"]: + set_state(self.serial, f"upgrading to {latest['version_name']}") + if self.install(latest, upgrade=True): + note = "" if self.last_cycle_ok else \ + "previous cycle failed, so the suite's markers/route may not have existed" + with self.suite_location(): + self.run_flow(None, POST_UPGRADE_FLOW, cycle_dir, kind="upgrade", note=note) + return installed_version(self.serial) if self.farm_installed() else None + + @contextlib.contextmanager + def suite_location(self): + if self.feeder: + self.feeder.hold = tuple(self.cfg["suite_location"]) + try: + yield + finally: + if self.feeder: + self.feeder.hold = None + + # ----- tests + + def run_flow(self, cycle_id, path, cycle_dir, kind="flow", note=""): + name = os.path.splitext(os.path.basename(path))[0] + out_dir = os.path.join(cycle_dir, name) + os.makedirs(out_dir, exist_ok=True) + report = os.path.join(out_dir, "report.xml") + cmd = [os.path.expanduser("~/.maestro/bin/maestro"), "test", "--device", self.serial, + "--format", "junit", "--output", report, "--test-output-dir", out_dir, + "--debug-output", os.path.join(out_dir, "debug"), "--flatten-debug-output", + "--no-ansi", path] + timeout = self.cfg["flow_timeout_minutes"] * 60 + start = time.time() + for attempt in range(2): + set_state(self.serial, f"running {name}") + if self.cfg["serialize_maestro"]: + with maestro_lock: + res = run(cmd, timeout=timeout) + else: + res = run(cmd, timeout=timeout) + output = res.stdout + res.stderr + try: + with open(report) as f: + output += f.read() + except OSError: + pass + if res.returncode == 0 or not MAESTRO_RETRY_RE.search(output) or attempt: + break + log(self.serial, f"{name}: transient launch failure, retrying once") + with open(os.path.join(out_dir, "maestro.log"), "w") as f: + f.write(res.stdout + res.stderr) + status = "pass" if res.returncode == 0 else "fail" + detail = "\n".join(filter(None, [note, "" if status == "pass" else + (res.stderr or res.stdout)[-2000:]])) + self.db.execute( + "INSERT INTO results(cycle_id, serial, ts, kind, name, status, duration_s, detail, dir) " + "VALUES(?,?,?,?,?,?,?,?,?)", + (cycle_id, self.serial, start, kind, name, status, time.time() - start, detail, out_dir)) + return status == "pass" + + def run_monkey(self, cycle_id, cycle_dir): + set_state(self.serial, "monkey soak") + seed = int(time.time()) % 100000 + start = time.time() + res = adb(self.serial, "shell", "monkey", "-p", PACKAGE, "-s", str(seed), + "--throttle", str(self.cfg["monkey_throttle_ms"]), + "--pct-syskeys", "0", "-v", str(self.cfg["monkey_events"]), + timeout=self.cfg["monkey_events"] * self.cfg["monkey_throttle_ms"] / 1000 + 600) + with open(os.path.join(cycle_dir, "monkey.log"), "w") as f: + f.write(res.stdout + res.stderr) + ok = "Monkey finished" in res.stdout + self.db.execute( + "INSERT INTO results(cycle_id, serial, ts, kind, name, status, duration_s, detail, dir) " + "VALUES(?,?,?,?,?,?,?,?,?)", + (cycle_id, self.serial, start, "monkey", f"monkey seed {seed}", + "pass" if ok else "fail", time.time() - start, + "" if ok else res.stdout[-2000:], cycle_dir)) + self.repair() + return ok + + def cycle(self): + s = self.serial + level, temp = battery(s) + if temp and temp > self.cfg["max_battery_temp_c"]: + set_state(s, f"cooling down ({temp}°C)") + stop_event.wait(600) + return + stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") + cycle_dir = os.path.join(RUNS_DIR, s, stamp) + + version = self.ensure_version(cycle_dir) + if version is None: + if not self.db.builds(): + set_state(s, "waiting for a release download") + stop_event.wait(300) + return + os.makedirs(cycle_dir, exist_ok=True) + + device_since = shell(s, "date '+%Y-%m-%d %H:%M:%S'") + adb(s, "logcat", "-c") + cycle_id = self.db.execute( + "INSERT INTO cycles(serial, model, android, version_code, started, status, dir) " + "VALUES(?,?,?,?,?,?,?)", + (s, self.model, self.android, version, time.time(), "running", cycle_dir)) + + # Monkey first: Onboarding wipes whatever it leaves behind, and the suite + # then ends with the markers and route that PostUpgrade checks for. + self.run_monkey(cycle_id, cycle_dir) + passed = failed = 0 + with self.suite_location(): + for flow in suite_flows(): + if stop_event.is_set(): + break + if self.run_flow(cycle_id, os.path.join(MAESTRO_DIR, flow), cycle_dir): + passed += 1 + else: + failed += 1 + + with open(os.path.join(cycle_dir, "logcat.txt"), "w") as f: + f.write(adb(s, "logcat", "-d", timeout=120).stdout) + problems = collect_dropbox(s, device_since) + for i, (kind, text) in enumerate(problems): + with open(os.path.join(cycle_dir, f"{kind}-{i}.txt"), "w") as f: + f.write(text) + self.db.event(s, kind, version, text) + crashes = sum(1 for k, _ in problems if k == "crash") + anrs = len(problems) - crashes + level, temp = battery(s) + status = "pass" if failed == 0 and not problems else "fail" + self.last_cycle_ok = failed == 0 + self.db.execute( + "UPDATE cycles SET ended=?, status=?, flows_passed=?, flows_failed=?, crashes=?, anrs=?, " + "pss_kb=?, battery_level=?, battery_temp=? WHERE id=?", + (time.time(), status, passed, failed, crashes, anrs, pss_kb(s), level, temp, cycle_id)) + set_state(s, f"cycle {status}: {passed} passed, {failed} failed, " + f"{crashes} crashes, {anrs} ANRs") + + def run(self): + set_state(self.serial, "preparing", model=self.model, android=self.android) + try: + self.prepare() + while not stop_event.is_set(): + if adb(self.serial, "get-state", timeout=15).stdout.strip() != "device": + break + if self.dev_cfg.get("skip"): + set_state(self.serial, "skipped in config") + stop_event.wait(300) + continue + try: + self.cycle() + except Exception as e: # keep the farm going; record and retry + self.db.event(self.serial, "error", None, f"worker exception: {e!r}") + stop_event.wait(60) + finally: + self.alive.clear() + set_state(self.serial, "disconnected") + + +# --------------------------------------------------------------------------- report + + +def fmt_time(ts): + return dt.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M") if ts else "" + + +def write_report(db): + e = html.escape + day_ago = time.time() - 86400 + versions = {b["version_code"]: b["version_name"] for b in db.builds()} + rows = [] + with state_lock: + states = {k: dict(v) for k, v in device_state.items()} + for serial, st in sorted(states.items()): + last = db.query("SELECT * FROM cycles WHERE serial=? AND ended IS NOT NULL " + "ORDER BY id DESC LIMIT 1", (serial,)) + day = db.query("SELECT COUNT(*) n, SUM(status='pass') ok, SUM(crashes) c, SUM(anrs) a " + "FROM cycles WHERE serial=? AND started>? AND ended IS NOT NULL", + (serial, day_ago))[0] + last = last[0] if last else None + ver = last["version_code"] if last else None + rows.append(f"""{e(serial)}{e(st.get('model', ''))} + {e(st.get('android', ''))}{e(versions.get(ver, str(ver or '')))} + {e(st['state'])}
since {fmt_time(st['since'])}
+ {e(last['status']) if last else ''} + {day['ok'] or 0}/{day['n']}{day['c'] or 0}{day['a'] or 0} + {last['pss_kb'] // 1024 if last and last['pss_kb'] else ''} MB + {last['battery_level'] if last else ''}% / {last['battery_temp'] if last else ''}°C""") + failures = "".join( + f"{fmt_time(r['ts'])}{e(r['serial'])}{e(r['kind'])}" + f"{e(r['name'])}files" + f"
{e((r['detail'] or '')[-600:])}
" + for r in db.query("SELECT * FROM results WHERE status!='pass' ORDER BY id DESC LIMIT 50")) + events = "".join( + f"{fmt_time(r['ts'])}{e(r['serial'] or '')}{e(r['kind'])}" + f"{e(versions.get(r['version_code'], str(r['version_code'] or '')))}" + f"
{e((r['detail'] or '')[:600])}
" + for r in db.query("SELECT * FROM events ORDER BY id DESC LIMIT 50")) + builds = ", ".join(f"{e(b['version_name'])} ({b['version_code']})" for b in db.builds()) + page = f""" + +Soundscape Test Farm +

Soundscape test farm

Updated {fmt_time(time.time())}. Cached releases: {builds or 'none yet'}

+

Devices

+ +{''.join(rows)}
SerialModelAndroidAppNowLast cyclePassed 24hCrashes 24hANRs 24hPSSBattery
+

Recent failures

+{failures}
WhenDeviceKindNameDetail
+

Events

+{events}
WhenDeviceKindAppDetail
""" + tmp = REPORT_PATH + ".tmp" + with open(tmp, "w") as f: + f.write(page) + os.replace(tmp, REPORT_PATH) + + +def prune_runs(keep_days): + cutoff = time.time() - keep_days * 86400 + for path in glob.glob(os.path.join(RUNS_DIR, "*", "*")): + if os.path.getmtime(path) < cutoff: + subprocess.run(["rm", "-rf", path]) + + +# --------------------------------------------------------------------------- main + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--config", default=os.path.join(FARM_DIR, "farm.toml")) + args = parser.parse_args() + cfg = load_config(args.config) + os.makedirs(APK_DIR, exist_ok=True) + os.makedirs(RUNS_DIR, exist_ok=True) + db = Db(DB_PATH) + + def handle_signal(*_): + log(None, "stopping...") + stop_event.set() + signal.signal(signal.SIGINT, handle_signal) + signal.signal(signal.SIGTERM, handle_signal) + + threading.Thread(target=release_watcher, args=(cfg, db), daemon=True).start() + workers = {} + last_prune = 0 + while not stop_event.is_set(): + for serial in connected_devices(cfg["include_emulators"]): + if serial not in workers or not workers[serial].is_alive(): + workers[serial] = Worker(serial, cfg, db) + workers[serial].start() + write_report(db) + if time.time() - last_prune > 86400: + prune_runs(cfg["keep_days"]) + last_prune = time.time() + stop_event.wait(30) + + for serial in workers: + disable_mock_location(serial) + write_report(db) + + +if __name__ == "__main__": + main() diff --git a/test-farm/farm.toml.example b/test-farm/farm.toml.example new file mode 100644 index 000000000..49deac45a --- /dev/null +++ b/test-farm/farm.toml.example @@ -0,0 +1,37 @@ +# Copy to farm.toml and edit. Every setting is optional; these are the defaults +# except where noted. Paths are relative to this file. + +# Where release builds come from: the newest two successful runs of this +# workflow, using the APK that goes to Play. +release_repo = "Scottish-Tech-Army/Soundscape-Android" +release_workflow = "build-app.yaml" +release_artifact = "release-apk" +poll_minutes = 15 + +monkey_events = 5000 # several minutes; not every event waits the throttle +monkey_throttle_ms = 250 +flow_timeout_minutes = 15 +keep_days = 14 # per-cycle logs/screenshots under runs/ + +# Run one Maestro flow at a time across all phones, if parallel runs collide. +serialize_maestro = false + +# If a phone has a Soundscape the farm didn't install (Play Store, debug build), +# uninstall it, wiping its data. Off by default: the phone is reported as blocked. +allow_reinstall = false + +include_emulators = false +max_battery_temp_c = 42 # pause the phone for 10 minutes above this + +# Mock location: walk this GPX track at a constant speed, turning round at each +# end. With no track, hold default_location (lat, lon). +replay_speed_mps = 1.4 +default_location = [55.9474, -4.3058] +default_gpx = "../local-pixel.gpx" # (the default is no track) +# Held while the Maestro suite runs; the flows assert distance order. +suite_location = [55.955360, -3.223538] + +# Per-phone overrides, keyed by `adb devices` serial. +# [devices."R5CW12345AB"] +# gpx = "../local-cubot.gpx" +# skip = true diff --git a/test-farm/flows/PostUpgrade.yaml b/test-farm/flows/PostUpgrade.yaml new file mode 100644 index 000000000..d4f83278d --- /dev/null +++ b/test-farm/flows/PostUpgrade.yaml @@ -0,0 +1,40 @@ +appId: org.scottishtecharmy.soundscape +# Run by the test farm straight after `adb install -r` of a newer release over a +# phone that has just finished the suite (see test-farm/farm.py). Checks that +# the upgraded app starts on the old data, and that the markers and the route +# the suite created survived the upgrade. +--- +- launchApp: + clearState: false + +# The release dialog shows on the first launch of a new major.minor version. +- tapOn: + id: "newReleaseDialogDismiss" + optional: true + +- runFlow: + file: ../../maestro/WaitForScreen.yaml + env: + screen: Home +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homeMarkers +# Markers.yaml's C replaced A (same location), so B and C are what's left. +- assertVisible: + id: "LocationItem-B-.*" +- assertVisible: + id: "LocationItem-C-.*" + +- launchApp: + clearState: false +- runFlow: + file: ../../maestro/WaitForScreen.yaml + env: + screen: Home +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homeRoutes +- assertVisible: + text: "Test Route.*" From 2fc63029e52a00a57a8b0fc9029bee62dc4b546f Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Thu, 15 Oct 2026 12:57:16 +0100 Subject: [PATCH 03/11] Test farm: offline maps and search, and an iPhone observer The farm phones' Wi-Fi is unreliable, so every cycle now runs offline. The farm keeps the manifest's current Glasgow extract cached (extractProviderUrl from local.properties), adb-pushes it into the app's files/Download after Onboarding wipes app storage, and flows/OfflineSearch.yaml sets Search mode to Offline. The suite passes with Wi-Fi turned off. USB iPhones are observed through pymobiledevice3, since Maestro can't drive a physical iPhone. --userspace builds the iOS 17+ developer tunnel in-process, so nothing needs root. The worker replays the GPX walk with simulate-location play, keeps a daily Soundscape syslog, and every ios_poll_minutes records TestFlight upgrades, relaunches and restarts, pulls Soundscape crash/hang reports and reads the battery. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ex6Fdp5ftuhE1ZEyhZb7WQ --- test-farm/.gitignore | 1 + test-farm/README.md | 37 +++ test-farm/farm.py | 356 +++++++++++++++++++++++++++-- test-farm/farm.toml.example | 8 + test-farm/flows/OfflineSearch.yaml | 38 +++ 5 files changed, 424 insertions(+), 16 deletions(-) create mode 100644 test-farm/flows/OfflineSearch.yaml diff --git a/test-farm/.gitignore b/test-farm/.gitignore index e7f78a8ed..097cb6f99 100644 --- a/test-farm/.gitignore +++ b/test-farm/.gitignore @@ -6,3 +6,4 @@ apks/ runs/ __pycache__/ farm.log +extracts/ diff --git a/test-farm/README.md b/test-farm/README.md index 96b3e39b8..33ff5b115 100644 --- a/test-farm/README.md +++ b/test-farm/README.md @@ -36,6 +36,43 @@ python3 test-farm/farm.py # Ctrl-C to stop; removes the mock GPS provider held at `suite_location` (CI's Edinburgh point), because the flows assert distance-sorted list order. - **Maestro:** `testTagsAsResourceId` is set in `Theme.kt` for every build type. +- **Offline maps and search:** the phones' Wi-Fi can't be relied on, so the + farm keeps the current manifest extract (`extract_name`, default Glasgow, + which covers the suite point and the GPX walk) in `extracts/`, using + `extractProviderUrl` from `local.properties`. Onboarding wipes app storage, + so after it runs in each cycle the farm `adb push`es the extract into the + app's `files/Download`. That needs no root on the phones tried so far. + It then runs `flows/OfflineSearch.yaml` to set Settings › Search › Search + mode to Offline. Settings has no test tags, so that flow taps English labels + and the phones must be set to English. The suite passes with Wi-Fi off. + +## iPhones + +Maestro can't drive a physical iPhone. It drives iOS through an XCUITest +driver that only Xcode on a Mac can build, sign and launch, and its CLI only +targets simulators. So iPhones are **observed, not driven**. +`pymobiledevice3` (`pipx install pymobiledevice3`) does everything from Linux +without root, because `--userspace` builds the iOS 17+ developer tunnel +in-process. Every `ios_poll_minutes` (10) the farm: + +- mounts the developer disk image (needed after a reboot); +- records a TestFlight version change as an upgrade (TestFlight auto-update + installs releases; the App Store-signed IPA can't be sideloaded); +- relaunches Soundscape if it isn't running, and records a PID change with no + crash report as a restart (killed by iOS or the user); +- pulls new `Soundscape-*` crash and hang reports. bug_type 309 counts as a + crash; anything else is counted with the ANRs; +- reads the battery. + +Throughout, it replays the GPX walk with `simulate-location play`. The app +ignored a single fixed simulated point, so the walk has to move. It also keeps +a daily Soundscape syslog (`--no-debug`, about 66 MB/day) in `runs//`. + +Preparing an iPhone: Developer Mode on, paired with this PC ("Trust"), Soundscape +from TestFlight with automatic updates on, and the battery charge limit set to +80%. Keep it off USB hubs that lose disconnects. One VIA Labs hub here kept a +ghost iPhone connection that hung usbmuxd until it was removed via +`/sys/bus/usb/devices//remove` and usbmuxd was restarted. ## Preparing a phone diff --git a/test-farm/farm.py b/test-farm/farm.py index b5c0a80b3..d63bb6be3 100644 --- a/test-farm/farm.py +++ b/test-farm/farm.py @@ -21,6 +21,8 @@ import contextlib import datetime as dt import glob +import gzip +import json import html import math import os @@ -39,12 +41,17 @@ MAESTRO_DIR = os.path.join(REPO_DIR, "maestro") POST_UPGRADE_FLOW = os.path.join(FARM_DIR, "flows", "PostUpgrade.yaml") APK_DIR = os.path.join(FARM_DIR, "apks") +EXTRACT_DIR = os.path.join(FARM_DIR, "extracts") +OFFLINE_SEARCH_FLOW = os.path.join(FARM_DIR, "flows", "OfflineSearch.yaml") RUNS_DIR = os.path.join(FARM_DIR, "runs") DB_PATH = os.path.join(FARM_DIR, "farm.db") REPORT_PATH = os.path.join(FARM_DIR, "report.html") PACKAGE = "org.scottishtecharmy.soundscape" MOCK_PROVIDERS = ("gps", "network") +# GeoEngine reads extracts from /Download, which defaults to +# the app's external files dir (as CI's push in run-maestro-tests.yaml). +DEVICE_EXTRACT_DIR = f"/storage/emulated/0/Android/data/{PACKAGE}/files/Download" DEFAULTS = { "release_repo": "Scottish-Tech-Army/Soundscape-Android", @@ -58,12 +65,20 @@ "serialize_maestro": False, "allow_reinstall": False, "include_emulators": False, + # iPhones (via pymobiledevice3) are observed, not driven: see IosWorker. + "include_ios": True, + "ios_poll_minutes": 10, "max_battery_temp_c": 42, "replay_speed_mps": 1.4, "default_location": [55.9474, -4.3058], # Held while the Maestro suite runs: the flows assert distance-sorted list # order, so they need the fixed point CI pins with `adb emu geo fix`. "suite_location": [55.955360, -3.223538], + # The offline map pushed to every phone, by its manifest "name". It has to + # cover suite_location and the GPX track. The manifest URL is read from + # local.properties' extractProviderUrl unless set here. + "extract_name": "Glasgow", + "extract_provider_url": "", "default_gpx": "", "devices": {}, } @@ -239,9 +254,67 @@ def release_watcher(cfg, db): (int(run_id), int(m.group(1)), m.group(2), created, apks[0])) known.add(int(run_id)) log(None, f"new release {m.group(2)} ({m.group(1)}) cached") + sync_extract(cfg) stop_event.wait(cfg["poll_minutes"] * 60) +# --------------------------------------------------------------------------- offline map + +current_extract = None # local path of the cached .pmtiles; its sidecar is path + ".geojson" + + +def extract_provider_url(cfg): + if cfg["extract_provider_url"]: + return cfg["extract_provider_url"] + try: + with open(os.path.join(REPO_DIR, "local.properties")) as f: + m = re.search(r"^extractProviderUrl=(.+)$", f.read(), re.M) + return m.group(1).strip() if m else "" + except OSError: + return "" + + +def sync_extract(cfg): + """Keeps the manifest's current extract for cfg["extract_name"] cached locally. + + Downloads only when the manifest names a new file, so the monthly rebuild + is picked up and nothing else is fetched. + """ + global current_extract + base = extract_provider_url(cfg) + if not base: + log(None, "no extractProviderUrl in local.properties; phones will use the network") + return + res = subprocess.run(["curl", "-sfL", f"{base.rstrip('/')}/manifest.geojson.gz"], + capture_output=True, timeout=120) + try: + manifest = json.loads(gzip.decompress(res.stdout)) + feature = next(f for f in manifest["features"] + if f["properties"].get("name") == cfg["extract_name"]) + except Exception as e: # manifest unreachable or no such extract; keep what we have + log(None, f"extract manifest unusable ({e!r}); keeping {current_extract}") + if not current_extract: + cached = sorted(glob.glob(os.path.join(EXTRACT_DIR, "*.pmtiles"))) + current_extract = cached[-1] if cached else None + return + filename = feature["properties"]["filename"] + path = os.path.join(EXTRACT_DIR, filename) + if not os.path.exists(path): + log(None, f"downloading offline map {filename}") + part = path + ".part" + dl = run(["curl", "-sfL", "-o", part, f"{base.rstrip('/')}/{filename}"], timeout=3600) + if dl.returncode != 0: + log(None, f"offline map download failed: {dl.stderr.strip()}") + return + os.replace(part, path) + with open(path + ".geojson", "w") as f: + json.dump(feature, f) + for old in glob.glob(os.path.join(EXTRACT_DIR, "*")): + if not old.startswith(path): + os.remove(old) + current_extract = path + + # --------------------------------------------------------------------------- location @@ -261,23 +334,20 @@ def haversine(a, b): return 2 * 6371000 * math.asin(math.sqrt(h)) -class LocationFeeder(threading.Thread): - """Feeds mock fixes to the phone once a second over adb. +class Track: + """Walks a GPX track at a constant speed, turning round at each end rather + than jumping back to the start. With no track, holds one fixed point.""" - Walks the GPX track at a constant speed and turns round at each end rather - than jumping back to the start. With no track, holds one fixed point. - Setting `hold` to a (lat, lon) pins the phone there until it's set to None. - """ - - def __init__(self, serial, gpx, fixed, speed, db, alive): - super().__init__(daemon=True) - self.serial, self.speed, self.db, self.alive = serial, speed, db, alive - self.hold = None + def __init__(self, gpx, fixed, speed): + self.speed = speed self.points = load_track(gpx) if gpx else [tuple(fixed), tuple(fixed)] self.cumulative = [0.0] for a, b in zip(self.points, self.points[1:]): self.cumulative.append(self.cumulative[-1] + haversine(a, b)) + def round_trip_seconds(self): + return max(60, int(2 * self.cumulative[-1] / self.speed)) + def position(self, t): total = self.cumulative[-1] if total == 0: @@ -293,10 +363,22 @@ def position(self, t): (la, lo), (lb, lob) = self.points[i], self.points[i + 1] return la + (lb - la) * f, lo + (lob - lo) * f + +class LocationFeeder(threading.Thread): + """Feeds the Track's position to an Android phone once a second over adb. + Setting `hold` to a (lat, lon) pins the phone there until it's set to None. + """ + + def __init__(self, serial, gpx, fixed, speed, db, alive): + super().__init__(daemon=True) + self.serial, self.db, self.alive = serial, db, alive + self.hold = None + self.track = Track(gpx, fixed, speed) + def run(self): start = time.time() while not stop_event.is_set() and self.alive.is_set(): - lat, lon = self.hold or self.position(time.time() - start) + lat, lon = self.hold or self.track.position(time.time() - start) cmd = "; ".join( f"cmd location providers set-test-provider-location {p} " f"--location {lat:.7f},{lon:.7f} --accuracy 5" for p in MOCK_PROVIDERS) @@ -407,6 +489,23 @@ def quieten(self): if "volume is 0 " not in shell(s, "cmd media_session volume --stream 3 --get"): shell(s, "for i in $(seq 1 16); do input keyevent KEYCODE_VOLUME_DOWN; done") + def push_extract(self): + """Copies the cached offline map and its metadata sidecar into the app's + extracts directory, named as CI does (without the build-date prefix).""" + if not current_extract: + self.db.event(self.serial, "error", None, "no offline map cached; using the network") + return + set_state(self.serial, "pushing offline map") + name = re.sub(r"^\d{8}-\d{4}-", "", os.path.basename(current_extract)) + shell(self.serial, f"rm -f {DEVICE_EXTRACT_DIR}/*.pmtiles {DEVICE_EXTRACT_DIR}/*.geojson; " + f"mkdir -p {DEVICE_EXTRACT_DIR}") + for src, dest in ((current_extract, name), (current_extract + ".geojson", name + ".geojson")): + res = adb(self.serial, "push", src, f"{DEVICE_EXTRACT_DIR}/{dest}", timeout=900) + if res.returncode != 0: + self.db.event(self.serial, "error", None, + f"offline map push failed:\n{res.stdout}{res.stderr}") + return + def repair(self): """Undoes things the monkey can do from the notification shade.""" shell(self.serial, "cmd connectivity airplane-mode disable; svc wifi enable; " @@ -579,6 +678,14 @@ def cycle(self): passed += 1 else: failed += 1 + if flow == "Onboarding.yaml": + # Onboarding cleared the app's storage and settings, so put the + # offline map back and switch search to offline. + self.push_extract() + if self.run_flow(cycle_id, OFFLINE_SEARCH_FLOW, cycle_dir, kind="setup"): + passed += 1 + else: + failed += 1 with open(os.path.join(cycle_dir, "logcat.txt"), "w") as f: f.write(adb(s, "logcat", "-d", timeout=120).stdout) @@ -620,6 +727,214 @@ def run(self): set_state(self.serial, "disconnected") +# --------------------------------------------------------------------------- iPhone + + +IOS_BUNDLE = "org.scottishtecharmy.soundscape" +PMD3 = os.path.expanduser("~/.local/bin/pymobiledevice3") + + +def pmd3(udid, *args, timeout=120, tunnel=False): + """Runs pymobiledevice3 against one iPhone. Developer services (location, + launch, process list, screenshots) go over the iOS 17+ tunnel, which + --userspace builds in-process, so nothing here needs root.""" + extra = ["--userspace"] if tunnel else [] + return run([PMD3, *args, *extra, "--udid", udid], timeout=timeout) + + +def ios_devices(): + """UDIDs of USB-connected iPhones. --simple skips lockdown, so a phone that + isn't answering can't hang discovery.""" + if not os.path.exists(PMD3): + return [] + res = run([PMD3, "usbmux", "list", "--usb", "--simple"], timeout=20) + try: + return json.loads(res.stdout or "[]") + except json.JSONDecodeError: + return [] + + +class IosWorker(threading.Thread): + """Observes Soundscape on an iPhone. Maestro can't drive a physical iPhone, + so there is no UI testing: the app (installed and upgraded by TestFlight) + is kept running on a replayed GPX walk while the farm records upgrades, + restarts, crash and hang reports, battery and a Soundscape syslog.""" + + def __init__(self, udid, cfg, db): + super().__init__(daemon=True, name=udid) + self.udid, self.cfg, self.db = udid, cfg, db + self.dev_cfg = cfg["devices"].get(udid, {}) + self.dir = os.path.join(RUNS_DIR, udid) + self.player = self.syslog = None + self.syslog_day = None + self.version = self.pid = None + info = {} + res = run([PMD3, "usbmux", "list", "--usb"], timeout=60) + try: + info = next(d for d in json.loads(res.stdout) if d.get("UniqueDeviceID") == udid) + except Exception: + pass + self.model = f"{info.get('DeviceName', 'iPhone')} ({info.get('ProductType', '?')})" + self.os = f"iOS {info.get('ProductVersion', '?')}" + + # ----- background processes + + def write_walk(self): + """One there-and-back of the GPX track as a timestamped GPX at 1 Hz, + which `simulate-location play` replays in real time. A moving track + matters: the app ignored a single fixed simulated point.""" + track = Track(self.dev_cfg.get("gpx", self.cfg["default_gpx"]), + self.cfg["default_location"], self.cfg["replay_speed_mps"]) + t0 = dt.datetime.now(dt.timezone.utc) + pts = [] + for i in range(track.round_trip_seconds()): + lat, lon = track.position(i) + stamp = (t0 + dt.timedelta(seconds=i)).strftime("%Y-%m-%dT%H:%M:%SZ") + pts.append(f'') + path = os.path.join(self.dir, "walk.gpx") + with open(path, "w") as f: + f.write('' + f'{"".join(pts)}') + return path + + def keep_processes_running(self): + if self.player is None or self.player.poll() is not None: + self.player = subprocess.Popen( + [PMD3, "developer", "dvt", "simulate-location", "play", "--userspace", + "--udid", self.udid, self.write_walk()], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + day = dt.date.today().strftime("%Y%m%d") + if self.syslog_day != day or self.syslog is None or self.syslog.poll() is not None: + if self.syslog and self.syslog.poll() is None: + self.syslog.terminate() + self.syslog = subprocess.Popen( + # DEBUG is mostly CoreMotion sensor fusion: ~1.7 GB a day. + [PMD3, "syslog", "live", "--udid", self.udid, "-pn", "Soundscape", "--no-debug", + "-o", os.path.join(self.dir, f"syslog-{day}.txt")], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.syslog_day = day + + def stop_processes(self): + for p in (self.player, self.syslog): + if p and p.poll() is None: + p.terminate() + pmd3(self.udid, "developer", "dvt", "simulate-location", "clear", tunnel=True, timeout=60) + + # ----- one observation + + def installed_version(self): + res = pmd3(self.udid, "apps", "list", timeout=120) + try: + app = json.loads(res.stdout)[IOS_BUNDLE] + return f"{app['CFBundleShortVersionString']} ({app['CFBundleVersion']})" + except Exception: + return None + + def app_pid(self): + res = pmd3(self.udid, "developer", "dvt", "process-id-for-bundle-id", IOS_BUNDLE, + tunnel=True, timeout=60) + m = re.search(r"^\s*(\d+)\s*$", res.stdout, re.M) + return int(m.group(1)) if m and int(m.group(1)) > 0 else None + + def new_reports(self, obs_dir): + """Pulls Soundscape crash/hang reports not seen before. Returns + (kind, name, text); bug_type 309 is a crash, anything else (hangs, + resource reports) is counted as an ANR.""" + res = pmd3(self.udid, "crash", "ls", timeout=120) + names = [n.strip().strip('",') for n in res.stdout.splitlines()] + names = [n for n in names if re.search(r"(^|/)Soundscape[-_.]", n)] + seen = {r["detail"] for r in self.db.query( + "SELECT detail FROM events WHERE serial=? AND kind='ios-report'", (self.udid,))} + found = [] + for name in names: + if name in seen: + continue + base = os.path.basename(name) + os.makedirs(obs_dir, exist_ok=True) + pmd3(self.udid, "crash", "pull", obs_dir, "--match", re.escape(base), timeout=300) + text = "" + for path in glob.glob(os.path.join(obs_dir, "**", base), recursive=True): + with open(path, errors="replace") as f: + text = f.read() + m = re.search(r'"bug_type"\s*:\s*"?(\d+)', text[:2000]) + kind = "crash" if m and m.group(1) == "309" else "anr" + self.db.execute("INSERT INTO events(serial, ts, kind, version_code, detail) " + "VALUES(?,?,?,?,?)", (self.udid, time.time(), "ios-report", None, name)) + found.append((kind, base, f"{base} (bug_type {m.group(1) if m else '?'})\n{text[:4000]}")) + return found + + def battery(self): + res = pmd3(self.udid, "diagnostics", "battery", "single", timeout=60) + level = re.search(r'"CurrentCapacity":\s*(\d+)', res.stdout) + temp = re.search(r'"Temperature":\s*(\d+)', res.stdout) + return (int(level.group(1)) if level else None, + int(temp.group(1)) / 100 if temp else None) + + def observe(self): + u = self.udid + started = time.time() + obs_dir = os.path.join(self.dir, dt.datetime.now().strftime("%Y%m%d-%H%M%S")) + set_state(u, "checking") + # Needed after every reboot; succeeds immediately when already mounted. + mount = pmd3(u, "mounter", "auto-mount", tunnel=True, timeout=300) + if mount.returncode != 0: + self.db.event(u, "error", None, "developer disk image mount failed (is Developer " + "Mode on?)\n" + (mount.stdout + mount.stderr)[-1500:]) + self.keep_processes_running() + + version = self.installed_version() + if version and self.version and version != self.version: + self.db.event(u, "upgrade", None, f"TestFlight upgrade {self.version} -> {version}") + self.version = version or self.version + + problems = self.new_reports(obs_dir) + for kind, name, text in problems: + self.db.event(u, kind, None, text) + pid = self.app_pid() + if pid is None: + if version: + self.db.event(u, "relaunch", None, "Soundscape wasn't running; relaunching it") + pmd3(u, "developer", "dvt", "launch", "--no-kill-existing", IOS_BUNDLE, + tunnel=True, timeout=60) + pid = self.app_pid() + else: + self.db.event(u, "error", None, "Soundscape isn't installed (TestFlight)") + elif self.pid and pid != self.pid and not problems: + self.db.event(u, "restart", None, f"Soundscape restarted (pid {self.pid} -> {pid}) " + "with no crash report: killed by iOS or the user") + self.pid = pid + + level, temp = battery = self.battery() + crashes = sum(1 for k, _, _ in problems if k == "crash") + anrs = len(problems) - crashes + status = "pass" if pid and not problems else "fail" + self.db.execute( + "INSERT INTO cycles(serial, model, android, version_code, started, ended, status, " + "flows_passed, flows_failed, crashes, anrs, battery_level, battery_temp, dir) " + "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + (u, self.model, self.os, None, started, time.time(), status, 0, 0, crashes, anrs, + level, temp, obs_dir)) + set_state(u, f"observing Soundscape {version or '?'} (pid {pid or 'none'}); " + f"{crashes} crashes, {anrs} hang/other reports", + model=self.model, android=self.os, app=version) + + def run(self): + os.makedirs(self.dir, exist_ok=True) + set_state(self.udid, "preparing", model=self.model, android=self.os) + try: + while not stop_event.is_set(): + if self.udid not in ios_devices(): + break + try: + self.observe() + except Exception as e: # keep observing; record and retry + self.db.event(self.udid, "error", None, f"iPhone worker exception: {e!r}") + stop_event.wait(self.cfg["ios_poll_minutes"] * 60) + finally: + self.stop_processes() + set_state(self.udid, "disconnected") + + # --------------------------------------------------------------------------- report @@ -643,7 +958,7 @@ def write_report(db): last = last[0] if last else None ver = last["version_code"] if last else None rows.append(f"""{e(serial)}{e(st.get('model', ''))} - {e(st.get('android', ''))}{e(versions.get(ver, str(ver or '')))} + {e(st.get('android', ''))}{e(versions.get(ver, str(ver or st.get('app', ''))))} {e(st['state'])}
since {fmt_time(st['since'])}
{e(last['status']) if last else ''} {day['ok'] or 0}/{day['n']}{day['c'] or 0}{day['a'] or 0} @@ -673,7 +988,7 @@ def write_report(db): pre{{margin:0;white-space:pre-wrap;font-size:12px;max-width:70ch}} a{{color:inherit}} .wrap{{overflow-x:auto}}

Soundscape test farm

Updated {fmt_time(time.time())}. Cached releases: {builds or 'none yet'}

-

Devices

+

Devices

SerialModelAndroidApp
{''.join(rows)}
SerialModelOSApp NowLast cyclePassed 24hCrashes 24hANRs 24h PSSBattery

Recent failures

@@ -711,6 +1026,8 @@ def handle_signal(*_): signal.signal(signal.SIGINT, handle_signal) signal.signal(signal.SIGTERM, handle_signal) + os.makedirs(EXTRACT_DIR, exist_ok=True) + sync_extract(cfg) # before any phone starts, so the first cycle has it threading.Thread(target=release_watcher, args=(cfg, db), daemon=True).start() workers = {} last_prune = 0 @@ -719,14 +1036,21 @@ def handle_signal(*_): if serial not in workers or not workers[serial].is_alive(): workers[serial] = Worker(serial, cfg, db) workers[serial].start() + for udid in ios_devices() if cfg["include_ios"] else []: + if udid not in workers or not workers[udid].is_alive(): + workers[udid] = IosWorker(udid, cfg, db) + workers[udid].start() write_report(db) if time.time() - last_prune > 86400: prune_runs(cfg["keep_days"]) last_prune = time.time() stop_event.wait(30) - for serial in workers: - disable_mock_location(serial) + for serial, worker in workers.items(): + if isinstance(worker, IosWorker): + worker.stop_processes() + else: + disable_mock_location(serial) write_report(db) diff --git a/test-farm/farm.toml.example b/test-farm/farm.toml.example index 49deac45a..248dea11e 100644 --- a/test-farm/farm.toml.example +++ b/test-farm/farm.toml.example @@ -21,6 +21,8 @@ serialize_maestro = false allow_reinstall = false include_emulators = false +include_ios = true # observe USB iPhones via pymobiledevice3 (see README) +ios_poll_minutes = 10 max_battery_temp_c = 42 # pause the phone for 10 minutes above this # Mock location: walk this GPX track at a constant speed, turning round at each @@ -31,6 +33,12 @@ default_gpx = "../local-pixel.gpx" # (the default is no track) # Held while the Maestro suite runs; the flows assert distance order. suite_location = [55.955360, -3.223538] +# Offline map pushed to every phone, by manifest name; must cover +# suite_location and the GPX track. URL defaults to local.properties' +# extractProviderUrl. +extract_name = "Glasgow" +# extract_provider_url = "" + # Per-phone overrides, keyed by `adb devices` serial. # [devices."R5CW12345AB"] # gpx = "../local-cubot.gpx" diff --git a/test-farm/flows/OfflineSearch.yaml b/test-farm/flows/OfflineSearch.yaml new file mode 100644 index 000000000..f5c13ff65 --- /dev/null +++ b/test-farm/flows/OfflineSearch.yaml @@ -0,0 +1,38 @@ +appId: org.scottishtecharmy.soundscape +# Run by the test farm after Onboarding (which wipes the app's settings) to +# switch Settings > Search > Search mode to Offline, so searches use the +# offline map extract the farm pushes instead of the phones' unreliable Wi-Fi. +# +# The Settings screen has no testTags, so this matches the English labels +# (settings_section / menu_manage_search, settings_section_search_network, +# settings_search_offline): the farm phones must be set to English. +--- +- launchApp: + clearState: false +- runFlow: + file: ../../maestro/WaitForScreen.yaml + env: + screen: Home +- tapOn: + id: topBarMenu +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: menuSettings +- scrollUntilVisible: + element: + text: "Search" + direction: DOWN +- tapOn: + text: "Search" +- scrollUntilVisible: + element: + text: "Search mode" + direction: DOWN +- tapOn: + text: "Search mode" +- tapOn: + text: "Offline" +# The summary under "Search mode" now names the chosen option. +- assertVisible: + text: ".*Offline.*" From cbd5e3d9086bb865cddf6ef0ebdfb87a20dd6115 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Thu, 15 Oct 2026 14:17:15 +0100 Subject: [PATCH 04/11] Test farm: walk with speed and bearing, and harden the Android workers Mock fixes from cmd location providers carry no speed or bearing, so Soundscape treated the phones as standing still and took callout directions from the compass of a phone on a desk. A small mock location helper (test-farm/mock-location, built with plain SDK tools, not part of Soundscape) now sets both on every fix. Its package is deliberately outside org.scottishtecharmy.soundscape: under that prefix Maestro's launchApp stopped granting Soundscape's permissions on the AOSP phone. GPX tracks are thinned to points 15 m apart, because a recording that starts with its owner standing still begins with minutes of on-the-spot GPS jitter, which a constant-speed replay walked in place. The iPhone player now logs to runs//player.log. Workers are harder to knock over: the farm stops its child processes on shutdown (an orphaned Maestro run kept the phone's driver and failed every flow after a restart), a retry after a gRPC driver error reinstalls the driver, a phone on a PIN lock screen is reported as locked instead of failing every flow, and the location feeder survives adb timeouts (their partial output arrives as bytes). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ex6Fdp5ftuhE1ZEyhZb7WQ --- test-farm/README.md | 24 ++- test-farm/farm.py | 188 ++++++++++++++---- test-farm/mock-location/.gitignore | 1 + test-farm/mock-location/AndroidManifest.xml | 18 ++ test-farm/mock-location/build.sh | 22 ++ .../farmlocation/FixReceiver.java | 96 +++++++++ 6 files changed, 308 insertions(+), 41 deletions(-) create mode 100644 test-farm/mock-location/.gitignore create mode 100644 test-farm/mock-location/AndroidManifest.xml create mode 100755 test-farm/mock-location/build.sh create mode 100644 test-farm/mock-location/src/org/scottishtecharmy/farmlocation/FixReceiver.java diff --git a/test-farm/README.md b/test-farm/README.md index 33ff5b115..d02ad8ec8 100644 --- a/test-farm/README.md +++ b/test-farm/README.md @@ -28,13 +28,23 @@ python3 test-farm/farm.py # Ctrl-C to stop; removes the mock GPS provider - **Analytics:** the farm sets `settings put system firebase.test.lab true`, which makes `MainActivity` use dummy analytics, as it does in Firebase Test Lab. -- **Location:** release builds don't have the debug `REPLAY_GPX` intent, so the - farm adds shell mock providers for `gps` and `network` - (`cmd location providers ...`) and feeds them a GPX track once a second. - This reaches both the fused provider (phones with Play services) and plain - `LocationManager` (AOSP phones). While the Maestro suite runs, the phone is - held at `suite_location` (CI's Edinburgh point), because the flows assert - distance-sorted list order. +- **Location:** release builds don't have the debug `REPLAY_GPX` intent, so + the farm installs a small mock location helper (`mock-location/`, built by + its `build.sh` with plain SDK tools; it isn't Soundscape) and grants it the + mock location app-op. It feeds the helper's `gps` and `network` test + providers a fix once a second, with **speed and bearing**. Soundscape only + takes its direction from travel when a fix says the user is moving. + Without them (all that `cmd location providers` can set), callouts follow + the compass of a phone lying on a desk. This reaches both the fused provider + (phones with Play services) and plain `LocationManager` (AOSP phones). The + helper's receiver requires `android.permission.DUMP`, so only adb can feed it. + Tracks are thinned to points 15 m apart, + because a phone recording that starts with its owner standing still + begins with minutes of on-the-spot GPS jitter. While the Maestro suite runs, the phone is + held still (speed 0) at `suite_location` (CI's Edinburgh point), because + the flows assert distance-sorted list order. + iPhones can't get the same fix: `simulate-location` sets latitude and + longitude only. Whether iOS adds a course to simulated fixes is untested. - **Maestro:** `testTagsAsResourceId` is set in `Theme.kt` for every build type. - **Offline maps and search:** the phones' Wi-Fi can't be relied on, so the farm keeps the current manifest extract (`extract_name`, default Glasgow, diff --git a/test-farm/farm.py b/test-farm/farm.py index d63bb6be3..d37de0695 100644 --- a/test-farm/farm.py +++ b/test-farm/farm.py @@ -177,12 +177,40 @@ def builds(self): # --------------------------------------------------------------------------- adb +children = set() # running subprocesses, stopped when the farm stops +children_lock = threading.Lock() + + def run(cmd, timeout=120): + """subprocess.run, except that the farm can stop the process when it stops. + A Maestro run left behind by a stopped farm keeps hold of the phone's + driver, and every flow after a restart then fails with gRPC UNAVAILABLE.""" + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, errors="replace") + with children_lock: + children.add(proc) try: - return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, - errors="replace") - except subprocess.TimeoutExpired as e: - return subprocess.CompletedProcess(cmd, -1, e.stdout or "", f"timeout after {timeout}s") + out, err = proc.communicate(timeout=timeout) + return subprocess.CompletedProcess(cmd, proc.returncode, out, err) + except subprocess.TimeoutExpired: + proc.kill() + out, _ = proc.communicate() + return subprocess.CompletedProcess(cmd, -1, out or "", f"timeout after {timeout}s") + finally: + with children_lock: + children.discard(proc) + + +def stop_children(): + with children_lock: + procs = list(children) + for proc in procs: + proc.terminate() + for proc in procs: + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() def adb(serial, *args, timeout=120): @@ -318,12 +346,23 @@ def sync_extract(cfg): # --------------------------------------------------------------------------- location +TRACK_MIN_STEP_M = 15 + + def load_track(path): + """The track's points, keeping only those at least TRACK_MIN_STEP_M from the + last one kept. A phone recording that starts with its owner standing still + is minutes of GPS jitter (local-pixel.gpx: ~700 m of "path" within 15 m of + the start), which a constant-speed replay would otherwise walk on the spot.""" root = ET.parse(path).getroot() - points = [(float(p.get("lat")), float(p.get("lon"))) - for p in root.iter() if p.tag.split("}")[-1] in ("trkpt", "rtept")] + raw = [(float(p.get("lat")), float(p.get("lon"))) + for p in root.iter() if p.tag.split("}")[-1] in ("trkpt", "rtept")] + points = raw[:1] + for p in raw[1:]: + if haversine(points[-1], p) >= TRACK_MIN_STEP_M: + points.append(p) if len(points) < 2: - raise ValueError(f"{path}: need at least 2 track points") + raise ValueError(f"{path}: need at least 2 track points {TRACK_MIN_STEP_M} m apart") return points @@ -375,31 +414,94 @@ def __init__(self, serial, gpx, fixed, speed, db, alive): self.hold = None self.track = Track(gpx, fixed, speed) + def fix(self, t): + """(lat, lon, speed, bearing) at t seconds. Speed and bearing matter: + Soundscape only takes its direction from travel when a fix says the + user is moving, and otherwise uses the compass of a phone on a desk.""" + if self.hold: + return (*self.hold, 0.0, None) + here, ahead = self.track.position(t), self.track.position(t + 1) + if here == ahead: + return (*here, 0.0, None) + return (*here, self.track.speed, initial_bearing(here, ahead)) + def run(self): start = time.time() + failures = 0 while not stop_event.is_set() and self.alive.is_set(): - lat, lon = self.hold or self.track.position(time.time() - start) - cmd = "; ".join( - f"cmd location providers set-test-provider-location {p} " - f"--location {lat:.7f},{lon:.7f} --accuracy 5" for p in MOCK_PROVIDERS) - out = shell(self.serial, cmd, timeout=10) - if "Exception" in out: - self.db.event(self.serial, "error", None, f"mock location rejected:\n{out}") - return + lat, lon, speed, bearing = self.fix(time.time() - start) + extras = f"--es lat {lat:.7f} --es lon {lon:.7f} --es speed {speed:.2f}" + if bearing is not None: + extras += f" --es bearing {bearing:.1f}" + out = shell(self.serial, f"am broadcast -n {HELPER_PKG}/.FixReceiver " + f"-a {HELPER_PKG}.FIX {extras}", timeout=10) + if 'data="ok"' in out: + failures = 0 + else: + # Keep going: if the feed stops, the phone falls back to its real + # position mid-suite, which breaks flows in confusing ways. + failures += 1 + if failures in (1, 60) or failures % 600 == 0: + self.db.event(self.serial, "error", None, + f"mock location rejected ({failures} in a row):\n{out}") + allow_helper(self.serial) time.sleep(1) -def enable_mock_location(serial): - cmds = ["appops set com.android.shell android:mock_location allow"] - for p in MOCK_PROVIDERS: - cmds += [f"cmd location providers add-test-provider {p}", - f"cmd location providers set-test-provider-enabled {p} true"] - return shell(serial, "; ".join(cmds)) +def initial_bearing(a, b): + lat1, lon1, lat2, lon2 = map(math.radians, (*a, *b)) + y = math.sin(lon2 - lon1) * math.cos(lat2) + x = math.cos(lat1) * math.sin(lat2) - math.sin(lat1) * math.cos(lat2) * math.cos(lon2 - lon1) + return (math.degrees(math.atan2(y, x)) + 360) % 360 + + +# The mock location helper in mock-location/. `cmd location providers` can only +# set a position, so fixes from it have no speed or bearing; the helper sets +# both. It isn't Soundscape, so the farm still tests only release builds. +# Deliberately not under org.scottishtecharmy.soundscape: as +# org.scottishtecharmy.soundscape.farmlocation, Maestro's launchApp stopped +# granting Soundscape's permissions on the AOSP Samsung (it worked again as soon +# as the helper was uninstalled or renamed), and Onboarding failed every cycle. +HELPER_PKG = "org.scottishtecharmy.farmlocation" +HELPER_DIR = os.path.join(FARM_DIR, "mock-location") +HELPER_APK = os.path.join(HELPER_DIR, "build", "farm-location.apk") + + +def build_helper(): + if not os.path.exists(HELPER_APK): + res = run([os.path.join(HELPER_DIR, "build.sh")], timeout=600) + if res.returncode != 0: + sys.exit("mock location helper build failed:\n" + res.stdout + res.stderr) -def disable_mock_location(serial): - shell(serial, "; ".join(f"cmd location providers remove-test-provider {p}" - for p in MOCK_PROVIDERS), timeout=20) +def enable_mock_location(serial): + # Shell test providers from older farm runs would sit beside the helper's. + disable_mock_location(serial, shell_providers=True) + out = adb(serial, "install", "-r", HELPER_APK, timeout=300) + if "Success" not in out.stdout + out.stderr: + return "helper install failed: " + out.stdout + out.stderr + return allow_helper(serial) + + +def allow_helper(serial): + """Grants the helper the mock location app-op. Straight after a fresh + install the grant can be lost (seen on the AOSP Samsung), so it's read back + and repeated until it sticks.""" + for _ in range(5): + shell(serial, f"appops set {HELPER_PKG} android:mock_location allow") + if "allow" in shell(serial, f"appops get {HELPER_PKG} android:mock_location"): + return "" + time.sleep(2) + return "failed to allow mock location for " + HELPER_PKG + + +def disable_mock_location(serial, shell_providers=False): + if shell_providers: + shell(serial, "; ".join(f"cmd location providers remove-test-provider {p}" + for p in MOCK_PROVIDERS), timeout=20) + else: + shell(serial, f"am broadcast -n {HELPER_PKG}/.FixReceiver -a {HELPER_PKG}.STOP", + timeout=20) # --------------------------------------------------------------------------- crashes @@ -470,7 +572,7 @@ def prepare(self): adb(s, "logcat", "-G", "16M") self.quieten() out = enable_mock_location(s) - if "Exception" in out: + if "Exception" in out or "failed" in out: self.db.event(s, "error", None, f"could not enable mock location:\n{out}") else: gpx = self.dev_cfg.get("gpx", self.cfg["default_gpx"]) @@ -602,14 +704,20 @@ def run_flow(self, cycle_id, path, cycle_dir, kind="flow", note=""): else: res = run(cmd, timeout=timeout) output = res.stdout + res.stderr - try: - with open(report) as f: - output += f.read() - except OSError: - pass + # The driver errors that decide a retry only reach the debug log. + for extra in (report, os.path.join(out_dir, "debug", "maestro.log")): + try: + with open(extra, errors="replace") as f: + output += f.read() + except OSError: + pass if res.returncode == 0 or not MAESTRO_RETRY_RE.search(output) or attempt: break log(self.serial, f"{name}: transient launch failure, retrying once") + if "gRPC" in output: + # Maestro's on-device driver is unreachable (seen after a phone + # sat on its lock screen); a fresh install brings it back. + cmd = cmd[:2] + ["--reinstall-driver"] + cmd[2:] with open(os.path.join(out_dir, "maestro.log"), "w") as f: f.write(res.stdout + res.stderr) status = "pass" if res.returncode == 0 else "fail" @@ -651,6 +759,15 @@ def cycle(self): stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") cycle_dir = os.path.join(RUNS_DIR, s, stamp) + if "isKeyguardShowing=true" in shell(s, "dumpsys window | grep isKeyguardShowing"): + shell(s, "input keyevent KEYCODE_WAKEUP; wm dismiss-keyguard") + if "isKeyguardShowing=true" in shell(s, "dumpsys window | grep isKeyguardShowing"): + # A PIN or pattern: Maestro can't get past it, and every flow + # fails in under a second with the driver unreachable. + set_state(s, "locked: set Screen lock to None on this phone") + stop_event.wait(300) + return + version = self.ensure_version(cycle_dir) if version is None: if not self.db.builds(): @@ -799,10 +916,11 @@ def write_walk(self): def keep_processes_running(self): if self.player is None or self.player.poll() is not None: - self.player = subprocess.Popen( - [PMD3, "developer", "dvt", "simulate-location", "play", "--userspace", - "--udid", self.udid, self.write_walk()], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + with open(os.path.join(self.dir, "player.log"), "w") as out: + self.player = subprocess.Popen( + [PMD3, "developer", "dvt", "simulate-location", "play", "--userspace", + "--udid", self.udid, self.write_walk()], + stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT) day = dt.date.today().strftime("%Y%m%d") if self.syslog_day != day or self.syslog is None or self.syslog.poll() is not None: if self.syslog and self.syslog.poll() is None: @@ -1026,6 +1144,7 @@ def handle_signal(*_): signal.signal(signal.SIGINT, handle_signal) signal.signal(signal.SIGTERM, handle_signal) + build_helper() os.makedirs(EXTRACT_DIR, exist_ok=True) sync_extract(cfg) # before any phone starts, so the first cycle has it threading.Thread(target=release_watcher, args=(cfg, db), daemon=True).start() @@ -1046,6 +1165,7 @@ def handle_signal(*_): last_prune = time.time() stop_event.wait(30) + stop_children() for serial, worker in workers.items(): if isinstance(worker, IosWorker): worker.stop_processes() diff --git a/test-farm/mock-location/.gitignore b/test-farm/mock-location/.gitignore new file mode 100644 index 000000000..567609b12 --- /dev/null +++ b/test-farm/mock-location/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/test-farm/mock-location/AndroidManifest.xml b/test-farm/mock-location/AndroidManifest.xml new file mode 100644 index 000000000..89847586d --- /dev/null +++ b/test-farm/mock-location/AndroidManifest.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + diff --git a/test-farm/mock-location/build.sh b/test-farm/mock-location/build.sh new file mode 100755 index 000000000..de15713d5 --- /dev/null +++ b/test-farm/mock-location/build.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Builds the test farm's mock location helper (build/farm-location.apk) straight +# from the SDK tools: one Java class, no resources, no Gradle. Signed with the +# local debug key; farm.py installs it on each Android phone. +set -euo pipefail + +cd "$(dirname "$0")" +SDK="${ANDROID_HOME:-$HOME/Android/Sdk}" +BT="$(ls -d "$SDK"/build-tools/* | sort -V | tail -1)" +JAR="$(ls -d "$SDK"/platforms/android-[0-9]* | sort -V | tail -1)/android.jar" +OUT=build +rm -rf "$OUT" && mkdir -p "$OUT/classes" "$OUT/dex" + +javac -nowarn --release 11 -classpath "$JAR" -d "$OUT/classes" \ + $(find src -name '*.java') +"$BT/d8" --min-api 26 --lib "$JAR" --output "$OUT/dex" $(find "$OUT/classes" -name '*.class') +"$BT/aapt2" link -o "$OUT/unsigned.apk" -I "$JAR" --manifest AndroidManifest.xml +(cd "$OUT/dex" && zip -q ../unsigned.apk classes.dex) +"$BT/zipalign" -f 4 "$OUT/unsigned.apk" "$OUT/aligned.apk" +"$BT/apksigner" sign --ks "$HOME/.android/debug.keystore" --ks-pass pass:android \ + --out "$OUT/farm-location.apk" "$OUT/aligned.apk" +echo "built $OUT/farm-location.apk" diff --git a/test-farm/mock-location/src/org/scottishtecharmy/farmlocation/FixReceiver.java b/test-farm/mock-location/src/org/scottishtecharmy/farmlocation/FixReceiver.java new file mode 100644 index 000000000..209cc5ec1 --- /dev/null +++ b/test-farm/mock-location/src/org/scottishtecharmy/farmlocation/FixReceiver.java @@ -0,0 +1,96 @@ +package org.scottishtecharmy.farmlocation; + +import android.app.Activity; +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.location.Criteria; +import android.location.Location; +import android.location.LocationManager; +import android.os.SystemClock; + +/** + * Sets one mock fix on the gps and network test providers, sent by the test farm + * once a second: + * + * am broadcast -n org.scottishtecharmy.farmlocation/.FixReceiver \ + * -a org.scottishtecharmy.farmlocation.FIX \ + * --es lat 55.9474 --es lon -4.3058 --es speed 1.4 --es bearing 92.5 + * + * speed and bearing are optional. Soundscape only takes its direction from travel + * when a fix says the user is moving, so without them it falls back to the + * phone's compass. Action STOP removes the test providers again. + * + * Needs the mock location app-op: appops set android:mock_location allow + */ +public class FixReceiver extends BroadcastReceiver { + private static final String FIX = "org.scottishtecharmy.farmlocation.FIX"; + private static final String STOP = "org.scottishtecharmy.farmlocation.STOP"; + private static final String[] PROVIDERS = { + LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER + }; + + @Override + @SuppressWarnings("deprecation") + public void onReceive(Context context, Intent intent) { + LocationManager lm = context.getSystemService(LocationManager.class); + try { + if (STOP.equals(intent.getAction())) { + for (String p : PROVIDERS) { + try { + lm.removeTestProvider(p); + } catch (IllegalArgumentException notAdded) { + // nothing to remove + } + } + setResult(Activity.RESULT_OK, "stopped", null); + return; + } + if (!FIX.equals(intent.getAction())) { + setResult(Activity.RESULT_CANCELED, "unknown action", null); + return; + } + double lat = Double.parseDouble(intent.getStringExtra("lat")); + double lon = Double.parseDouble(intent.getStringExtra("lon")); + float accuracy = parse(intent.getStringExtra("accuracy"), 5f); + float speed = parse(intent.getStringExtra("speed"), -1f); + float bearing = parse(intent.getStringExtra("bearing"), -1f); + + for (String p : PROVIDERS) { + try { + lm.addTestProvider(p, false, false, false, false, true, true, true, + Criteria.POWER_LOW, Criteria.ACCURACY_FINE); + } catch (IllegalArgumentException alreadyAdded) { + // a provider from an earlier fix + } + lm.setTestProviderEnabled(p, true); + + Location fix = new Location(p); + fix.setLatitude(lat); + fix.setLongitude(lon); + fix.setAltitude(0); + fix.setAccuracy(accuracy); + fix.setVerticalAccuracyMeters(accuracy); + fix.setTime(System.currentTimeMillis()); + fix.setElapsedRealtimeNanos(SystemClock.elapsedRealtimeNanos()); + if (speed >= 0) { + fix.setSpeed(speed); + fix.setSpeedAccuracyMetersPerSecond(0.3f); + } + if (bearing >= 0) { + fix.setBearing(bearing); + fix.setBearingAccuracyDegrees(10f); + } + lm.setTestProviderLocation(p, fix); + } + setResult(Activity.RESULT_OK, "ok", null); + } catch (RuntimeException e) { + // Most likely a SecurityException: the mock location app-op isn't allowed. + setResult(Activity.RESULT_CANCELED, e.getClass().getSimpleName() + ": " + e.getMessage(), null); + } + } + + private static float parse(String value, float fallback) { + return value == null ? fallback : Float.parseFloat(value); + } +} From 18e32756cc4a3ac4413cf4e261015208e9f34065 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Thu, 15 Oct 2026 14:18:29 +0100 Subject: [PATCH 05/11] Document the test farm in the developer docs Add docs/developers/test-farm.md (what the farm does, how it differs from CI, how release builds are made testable, and the setup traps), link it from the developer index, and point to it from the Maestro docs. Also bring the Maestro flow table up to date: Markers.yaml's C replaces A, and RouteCreation.yaml builds its route from B and C. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ex6Fdp5ftuhE1ZEyhZb7WQ --- docs/developers/developers.md | 1 + docs/developers/maestro-tests.md | 19 +++++++- docs/developers/test-farm.md | 81 ++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 docs/developers/test-farm.md diff --git a/docs/developers/developers.md b/docs/developers/developers.md index 63bdc2ce3..d74173402 100644 --- a/docs/developers/developers.md +++ b/docs/developers/developers.md @@ -66,6 +66,7 @@ If you would like access to the main Soundscape tile provider for development, g * [Branch strategy]({% link developers/branch-strategy.md %}) — which branches are active, where to send PRs, and the upcoming KMP cutover that will replace `main`. * [Build types and analytics]({% link developers/build-types.md %}) — `debug` / `release` / `releaseTest`, the variant-specific source sets that swap in real or no-op analytics, and the `BuildConfig` values read from `local.properties`. * [GitHub actions]({% link developers/actions.md %}) — the CI workflows and the secrets they consume. +* [Test farm]({% link developers/test-farm.md %}) — long-term testing of release builds on physical phones: release upgrades, offline maps, a replayed GPX walk, and crash collection, with iPhones observed rather than driven. * [Unit test example]({% link developers/unit-test-example.md %}) — using host-side unit tests with real map data to prototype and debug geo features. ## Reference diff --git a/docs/developers/maestro-tests.md b/docs/developers/maestro-tests.md index 70b08a2f9..fb03a7db6 100644 --- a/docs/developers/maestro-tests.md +++ b/docs/developers/maestro-tests.md @@ -103,8 +103,8 @@ flows, so they must run in this order: | 2 | `HomePage.yaml` | Exercises the home screen controls (My Location, Around Me, Ahead of Me, menu, sleep). | | 3 | `LocationDetails.yaml` | Creates markers **A** and **B** that later flows rely on. | | 4 | `PlacesNearby.yaml` | Scrolls and opens entries in the Places Nearby list. | -| 5 | `Markers.yaml` | Sorting, editing marker **A**, and creating marker **C** with the New button. | -| 6 | `RouteCreation.yaml` | Builds a route from markers **A** and **B**. | +| 5 | `Markers.yaml` | Sorting, editing marker **A**, and creating marker **C** at the current location with the New button. A was saved there too, and markers at the same coordinates merge, so C replaces A. | +| 6 | `RouteCreation.yaml` | Builds a route from markers **B** and **C**. | | 7 | `FullScreenMap.yaml` | Toggles the full-screen map on the home screen. | `Onboarding.yaml` is the only flow that uses `launchApp: clearState: true`; @@ -426,3 +426,18 @@ always run; a final step re-raises the failure to mark the workflow red. When you add a new flow, add its name (without the `.yaml` extension) to the `for flow in …` list in the correct position so CI runs it. + +## On physical phones: the test farm + +The same flows also run continuously on physical phones against **release +builds** in the [test farm]({% link developers/test-farm.md %}). The farm runs +them in `maestro/Master.yaml` order. It holds the phone at the same Edinburgh +point CI uses, and adds two flows of its own from `test-farm/flows/`: +`OfflineSearch.yaml` after Onboarding, and `PostUpgrade.yaml` after each release +upgrade. Two things matter when changing flows: + +- A flow that only passes on an emulator (for example, by relying on `adb root` + or a debug-only screen) will fail there. +- Changing what the suite leaves behind (markers **B** and **C** and + **Test Route**) needs a matching change to `test-farm/flows/PostUpgrade.yaml`, + which checks they survive an upgrade. diff --git a/docs/developers/test-farm.md b/docs/developers/test-farm.md new file mode 100644 index 000000000..bcf3f8c61 --- /dev/null +++ b/docs/developers/test-farm.md @@ -0,0 +1,81 @@ +--- +title: Test farm +layout: page +parent: Information for developers +has_toc: false +--- + +# Test farm + +The test farm runs long-term tests of **release builds** on physical phones +plugged into a developer's PC. It complements CI: the +[Maestro UI tests]({% link developers/maestro-tests.md %}) in CI run once per +change on emulators, while the farm keeps real phones exercising the app for +days at a time and upgrades them as new releases are built. + +The farm is a single Python script, `test-farm/farm.py`. The setup steps and +every config option are in +[`test-farm/README.md`](https://github.com/Scottish-Tech-Army/Soundscape-Android/blob/main/test-farm/README.md). + +## What it does + +Each Android phone loops through a **cycle**: + +1. If a newer release has been built, it installs it over the existing app and + data (`adb install -r`), then runs `test-farm/flows/PostUpgrade.yaml` to + check that the app starts and the suite's markers and route survived the + upgrade. +2. A `monkey` soak: random input inside the app, logged with its seed so a + failure can be reproduced. +3. The Maestro suite, in `maestro/Master.yaml` order. After `Onboarding.yaml` + it pushes an offline map and runs `test-farm/flows/OfflineSearch.yaml`, so + every flow runs offline. +4. It collects crashes and ANRs (from Android's dropbox), logcat, memory (PSS) + and battery. + +Results go into `test-farm/farm.db`, with a summary page at +`test-farm/report.html` that refreshes every minute. + +iPhones are **observed, not driven**. Maestro can't control a physical iPhone, +because its iOS driver needs Xcode on a Mac. Instead the farm uses +[pymobiledevice3](https://github.com/doronz88/pymobiledevice3) to: +- replay a GPX walk; +- record TestFlight upgrades, and app restarts when the app wasn't running; +- collect Soundscape crash and hang reports, battery level and a Soundscape + syslog. + +## How it differs from CI + +| | CI | Test farm | +|---|---|---| +| Devices | Emulators | Physical phones, including an AOSP phone without Play services | +| Build | Debug | The release APK from the `Tag and build release` workflow | +| Upgrades | Not tested | Each release is installed over the previous one, keeping the data | +| Map data | Glasgow extract pushed as root | The current extract from the manifest, pushed without root, with search set to Offline | +| Location | `adb emu geo fix` | A replayed GPX walk with speed and bearing, held at CI's Edinburgh point while the suite runs | +| Duration | One run | Continuous | + +## Making release builds testable + +The farm doesn't use any debug-only hooks: + +- **Analytics:** setting `firebase.test.lab` makes `MainActivity` use dummy + analytics, as it does in Firebase Test Lab. +- **Location:** a small mock location helper app (`test-farm/mock-location/`, + not part of Soundscape) sets every fix, including speed and bearing. + Soundscape only takes its direction from travel when a fix says the user is + moving. Without speed and bearing it uses the phone's compass, and on a desk + that points nowhere useful. +- **Maestro:** `testTagsAsResourceId` is set for every build type, so the same + flows work on release builds. + +## Things that will catch you out + +- Phones need **Screen lock: None**. Maestro can't get past a PIN, and the farm + reports a locked phone instead of testing it. +- Phones must be in **English**. The Settings screen has no testTags, so + `OfflineSearch.yaml` taps its English labels. +- Remove any Play Store or debug copy of Soundscape first. The farm only tests a + build it installed itself, and reports anything else as blocked. +- Keep phones off USB hubs that lose disconnects. One here kept a ghost iPhone + connection that hung `usbmuxd`. From 2743f27f1cb2b6513f4cd3084cc5689e1030d465 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Thu, 15 Oct 2026 14:35:13 +0100 Subject: [PATCH 06/11] Test farm: start no new processes once stopping A worker could launch its next Maestro flow after shutdown had already stopped the farm's children, leaving an orphaned run holding the phone's driver. run() now refuses to start anything once the farm is stopping. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ex6Fdp5ftuhE1ZEyhZb7WQ --- test-farm/farm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test-farm/farm.py b/test-farm/farm.py index d37de0695..cde50ae6f 100644 --- a/test-farm/farm.py +++ b/test-farm/farm.py @@ -185,9 +185,13 @@ def run(cmd, timeout=120): """subprocess.run, except that the farm can stop the process when it stops. A Maestro run left behind by a stopped farm keeps hold of the phone's driver, and every flow after a restart then fails with gRPC UNAVAILABLE.""" - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, errors="replace") with children_lock: + # Once the farm is stopping, start nothing new: stop_children() may + # already have run, and anything started now would be orphaned. + if stop_event.is_set(): + return subprocess.CompletedProcess(cmd, -1, "", "farm stopping") + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, errors="replace") children.add(proc) try: out, err = proc.communicate(timeout=timeout) From 541ce1944d52ed380f3cf85fd5a55e4f58d10b22 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Fri, 16 Oct 2026 10:00:24 +0100 Subject: [PATCH 07/11] Read the type and distance of roads and addresses in location lists When a location's description starts with its name, as for roads and street addresses, the description replaced the whole content description, so TalkBack and VoiceOver users heard neither the type nor the distance and direction that sighted users see. Keep using the description in place of the name, but always add the type and distance. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ex6Fdp5ftuhE1ZEyhZb7WQ --- .../soundscape/components/LocationItem.kt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/LocationItem.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/LocationItem.kt index b5ec2f117..3269e96de 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/LocationItem.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/LocationItem.kt @@ -125,13 +125,19 @@ fun LocationItem( decoration.index != -1 -> "${decoration.indexDescription} ${decoration.index + 1}. ${item.name}" - else -> item.description?.takeIf { it.startsWith(item.name) } - ?: listOfNotNull( - item.name, + else -> { + // A road or address description already starts with the name, so it + // replaces the name rather than repeating it. The type and distance + // are still read out. + val description = item.description ?: item.street + val namedDescription = description?.takeIf { it.startsWith(item.name) } + listOfNotNull( + namedDescription ?: item.name, item.typeDescription?.additionalText, - item.description ?: item.street, + description.takeIf { namedDescription == null }, distanceStringA11y ).joinToString(", ") + } } if (decoration.editRoute.enabled) { From 9ace1951f4ea1119e118e71f4d46c760697b1a4f Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Fri, 16 Oct 2026 10:00:24 +0100 Subject: [PATCH 08/11] Expose test tags in dialogs and on Settings options Compose dialogs are separate windows outside the theme's testTagsAsResourceId, so Maestro couldn't see their tags. Add a common Modifier.testTagsAsResourceId() (a no-op on iOS) and apply it to the AlertDialogs and the search dialog. Tag the Settings sections, the search mode and show map preferences, list preference options and search results. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ex6Fdp5ftuhE1ZEyhZb7WQ --- .../platform/TestTagsAsResourceId.android.kt | 8 ++++++++ .../soundscape/components/MainSearchBar.kt | 4 +++- .../soundscape/platform/TestTagsAsResourceId.kt | 11 +++++++++++ .../screens/home/home/AudioTourInstructionDialog.kt | 4 +++- .../SharedAdvancedMarkersAndRoutesSettingsScreen.kt | 3 +++ .../screens/home/home/SharedNewReleaseDialog.kt | 3 ++- .../screens/home/locationDetails/OpenInMapApp.kt | 3 +++ .../SharedSaveAndEditMarkerScreen.kt | 3 +++ .../screens/home/settings/SettingsComponents.kt | 8 +++++++- .../screens/home/settings/SharedSettingsScreen.kt | 13 +++++++++++-- .../SharedAddAndEditRouteScreen.kt | 5 +++++ .../soundscape/platform/TestTagsAsResourceId.kt | 5 +++++ 12 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 shared/src/androidMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.android.kt create mode 100644 shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt create mode 100644 shared/src/iosMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt diff --git a/shared/src/androidMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.android.kt b/shared/src/androidMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.android.kt new file mode 100644 index 000000000..61e20162d --- /dev/null +++ b/shared/src/androidMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.android.kt @@ -0,0 +1,8 @@ +package org.scottishtecharmy.soundscape.platform + +import androidx.compose.ui.Modifier +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.semantics.testTagsAsResourceId as testTagsAsResourceIdProperty + +actual fun Modifier.testTagsAsResourceId(): Modifier = + semantics { testTagsAsResourceIdProperty = true } diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/MainSearchBar.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/MainSearchBar.kt index 4fbf2e8cb..5cfa16555 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/MainSearchBar.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/MainSearchBar.kt @@ -59,6 +59,7 @@ import org.scottishtecharmy.soundscape.resources.ui_back_button_title import org.scottishtecharmy.soundscape.platform.FullScreenDialogWindow import org.scottishtecharmy.soundscape.platform.fullScreenDialogProperties import org.scottishtecharmy.soundscape.platform.fullScreenDialogSystemBarsPadding +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.screens.home.data.LocationDescription import org.scottishtecharmy.soundscape.screens.talkbackDescription import org.scottishtecharmy.soundscape.screens.talkbackLive @@ -143,6 +144,7 @@ fun MainSearchBar( Box( modifier = Modifier .fillMaxSize() + .testTagsAsResourceId() .background(colors.background) ) { Column(modifier = Modifier.fillMaxSize().fullScreenDialogSystemBarsPadding()) { @@ -236,7 +238,7 @@ fun MainSearchBar( ) { itemsIndexed(results) { index, item -> Column( - modifier = Modifier.semantics { + modifier = Modifier.testTag("searchResult-$index").semantics { collectionItemInfo = CollectionItemInfo( rowSpan = 1, columnSpan = 1, rowIndex = index, columnIndex = 0, diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt new file mode 100644 index 000000000..e3494806e --- /dev/null +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt @@ -0,0 +1,11 @@ +package org.scottishtecharmy.soundscape.platform + +import androidx.compose.ui.Modifier + +/** + * Exposes the testTags in this subtree as Android resource-ids, so that UiAutomator-based tools + * such as Maestro can find them. The app theme sets this once at its root, but a Dialog is a + * separate window with its own composition root, so content shown in one (the full-screen search, + * a setting's list of choices) has to set it again. Does nothing on iOS. + */ +expect fun Modifier.testTagsAsResourceId(): Modifier diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/AudioTourInstructionDialog.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/AudioTourInstructionDialog.kt index 5fa57ecf0..4a917a44c 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/AudioTourInstructionDialog.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/AudioTourInstructionDialog.kt @@ -10,6 +10,7 @@ import androidx.compose.ui.semantics.paneTitle import androidx.compose.ui.semantics.semantics import org.jetbrains.compose.resources.stringResource import org.scottishtecharmy.soundscape.audio.AudioTourInstruction +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.resources.Res import org.scottishtecharmy.soundscape.resources.menu_audio_tutorial import org.scottishtecharmy.soundscape.resources.tour_continue_button @@ -24,7 +25,8 @@ fun AudioTourInstructionDialog( val dialogTitle = stringResource(Res.string.menu_audio_tutorial) AlertDialog( onDismissRequest = { /* Don't dismiss on outside click for accessibility */ }, - modifier = Modifier.semantics { paneTitle = dialogTitle }, + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId().semantics { paneTitle = dialogTitle }, text = { Text(text = instruction.text) }, diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedAdvancedMarkersAndRoutesSettingsScreen.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedAdvancedMarkersAndRoutesSettingsScreen.kt index e15231d0b..2196d0c89 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedAdvancedMarkersAndRoutesSettingsScreen.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedAdvancedMarkersAndRoutesSettingsScreen.kt @@ -27,6 +27,7 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.platform.testTag import androidx.compose.ui.text.font.FontWeight import org.jetbrains.compose.resources.stringResource +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.resources.Res import org.scottishtecharmy.soundscape.resources.advanced_markers_and_routes_clear_all_alert_message import org.scottishtecharmy.soundscape.resources.advanced_markers_and_routes_clear_all_button @@ -97,6 +98,8 @@ fun SharedAdvancedMarkersAndRoutesSettingsScreen( if (showConfirmationDialog.value) { AlertDialog( + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId(), onDismissRequest = { showConfirmationDialog.value = false }, title = { Text(stringResource(Res.string.settings_reset_dialog_title)) }, text = { Text(stringResource(Res.string.advanced_markers_and_routes_clear_all_alert_message)) }, diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedNewReleaseDialog.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedNewReleaseDialog.kt index d99f0682e..01a7ad91a 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedNewReleaseDialog.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedNewReleaseDialog.kt @@ -15,6 +15,7 @@ import androidx.compose.ui.platform.LocalUriHandler import androidx.compose.ui.platform.testTag import org.jetbrains.compose.resources.stringResource import org.scottishtecharmy.soundscape.platform.appVersionMinorTrimmed +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.preferences.PreferenceKeys import org.scottishtecharmy.soundscape.preferences.PreferencesProvider import org.scottishtecharmy.soundscape.resources.Res @@ -45,7 +46,7 @@ fun SharedNewReleaseDialog( val uriHandler = LocalUriHandler.current AlertDialog( - modifier = Modifier.padding(innerPadding), + modifier = Modifier.testTagsAsResourceId().padding(innerPadding), title = { Text(text = stringResource(Res.string.new_version_info_text)) }, text = { LazyColumn(modifier = Modifier.padding(top = spacing.medium)) { diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/OpenInMapApp.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/OpenInMapApp.kt index 7e6de5975..777517ee2 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/OpenInMapApp.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/OpenInMapApp.kt @@ -36,6 +36,7 @@ import androidx.compose.ui.semantics.customActions import androidx.compose.ui.semantics.semantics import org.jetbrains.compose.resources.stringResource import org.scottishtecharmy.soundscape.platform.isIos +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.preferences.PreferenceDefaults import org.scottishtecharmy.soundscape.preferences.PreferenceKeys import org.scottishtecharmy.soundscape.preferences.PreferencesProvider @@ -189,6 +190,8 @@ private fun MapAppChooserDialog( ) { var rememberChoice by remember { mutableStateOf(initiallyRemember) } AlertDialog( + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId(), onDismissRequest = onDismiss, title = { Text(stringResource(Res.string.open_in_app_dialog_title)) }, text = { diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/SharedSaveAndEditMarkerScreen.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/SharedSaveAndEditMarkerScreen.kt index d865daa9b..bb1d4e18f 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/SharedSaveAndEditMarkerScreen.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/SharedSaveAndEditMarkerScreen.kt @@ -33,6 +33,7 @@ import androidx.compose.ui.platform.testTag import androidx.compose.ui.text.font.FontWeight import org.jetbrains.compose.resources.stringResource import org.scottishtecharmy.soundscape.geojsonparser.geojson.LngLatAlt +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.preferences.PreferenceDefaults import org.scottishtecharmy.soundscape.preferences.PreferenceKeys import org.scottishtecharmy.soundscape.preferences.PreferencesProvider @@ -105,6 +106,8 @@ fun SharedSaveAndEditMarkerScreen( if (showDeleteDialog && onDelete != null) { AlertDialog( + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId(), onDismissRequest = { showDeleteDialog = false }, title = { Text(stringResource(Res.string.settings_reset_dialog_title)) }, text = { Text(stringResource(Res.string.markers_action_delete_alert_message)) }, diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SettingsComponents.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SettingsComponents.kt index 0ff96e67e..fb428a25f 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SettingsComponents.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SettingsComponents.kt @@ -26,9 +26,11 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color import androidx.compose.ui.semantics.Role import androidx.compose.ui.semantics.heading +import androidx.compose.ui.platform.testTag import androidx.compose.ui.semantics.semantics import org.jetbrains.compose.resources.StringResource import org.jetbrains.compose.resources.stringResource +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.resources.Res import org.scottishtecharmy.soundscape.resources.settings_collapse_section import org.scottishtecharmy.soundscape.resources.settings_collapsed @@ -51,6 +53,9 @@ fun ListPreferenceItem( ) { Row( modifier = Modifier + // The choices are shown in a dialog, outside the theme's testTagsAsResourceId. + .testTagsAsResourceId() + .testTag("settingsOption-$value") .background(MaterialTheme.colorScheme.surfaceContainer) .extraSmallPadding() .defaultMinSize(minHeight = spacing.targetSize) @@ -119,9 +124,10 @@ fun ExpandableSectionHeader( expanded: Boolean, onToggle: () -> Unit, textColor: Color, + modifier: Modifier = Modifier, ) { Row( - modifier = Modifier + modifier = modifier .fillMaxWidth() .clickable( role = Role.Button, diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SharedSettingsScreen.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SharedSettingsScreen.kt index 6e91d48fd..f941ae2ad 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SharedSettingsScreen.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SharedSettingsScreen.kt @@ -29,6 +29,7 @@ import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.testTag import androidx.compose.ui.semantics.Role import androidx.compose.ui.semantics.contentDescription import androidx.compose.ui.semantics.semantics @@ -301,6 +302,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "callouts") null else "callouts" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionCallouts"), ) } if (expandedSection.value == "callouts") { @@ -416,6 +418,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "search") null else "search" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionSearch"), ) } if (expandedSection.value == "search") { @@ -423,7 +426,7 @@ fun SharedSettingsScreen( key = PreferenceKeys.GEOCODER_MODE, defaultValue = PreferenceDefaults.GEOCODER_MODE, values = geocoderValues, - modifier = expandedSectionModifier, + modifier = expandedSectionModifier.testTag("settingsSearchMode"), title = { SettingTitle(Res.string.settings_section_search_network, textColor) }, item = { value, currentValue, onClick -> ListPreferenceItem( @@ -485,6 +488,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "accessibility") null else "accessibility" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionAccessibility"), ) } if (expandedSection.value == "accessibility") { @@ -503,7 +507,7 @@ fun SharedSettingsScreen( color = textColor, ) }, - modifier = expandedSectionModifier, + modifier = expandedSectionModifier.testTag("settingsShowMap"), ) } } @@ -524,6 +528,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "storage") null else "storage" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionStorage"), ) } if (expandedSection.value == "storage") { @@ -541,6 +546,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "audio") null else "audio" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionAudio"), ) } if (expandedSection.value == "audio") { @@ -591,6 +597,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "language") null else "language" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionLanguage"), ) } if (expandedSection.value == "language") { @@ -632,6 +639,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "media_controls") null else "media_controls" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionMediaControls"), ) } if (expandedSection.value == "media_controls") { @@ -673,6 +681,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "debug") null else "debug" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionDebug"), ) } if (expandedSection.value == "debug") { diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/markers_routes/screens/addandeditroutescreen/SharedAddAndEditRouteScreen.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/markers_routes/screens/addandeditroutescreen/SharedAddAndEditRouteScreen.kt index d1b6c962f..d9b3aa6d9 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/markers_routes/screens/addandeditroutescreen/SharedAddAndEditRouteScreen.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/markers_routes/screens/addandeditroutescreen/SharedAddAndEditRouteScreen.kt @@ -51,6 +51,7 @@ import org.jetbrains.compose.resources.stringResource import org.scottishtecharmy.soundscape.components.LocationItem import org.scottishtecharmy.soundscape.components.LocationItemDecoration import org.scottishtecharmy.soundscape.geojsonparser.geojson.LngLatAlt +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.resources.Res import org.scottishtecharmy.soundscape.resources.general_alert_cancel import org.scottishtecharmy.soundscape.resources.general_alert_done @@ -160,6 +161,8 @@ fun SharedAddAndEditRouteScreen( if (showDeleteDialog) { AlertDialog( + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId(), onDismissRequest = { showDeleteDialog = false }, title = { Text(stringResource(Res.string.settings_reset_dialog_title)) }, text = { Text(stringResource(Res.string.route_detail_edit_delete_alert_message)) }, @@ -187,6 +190,8 @@ fun SharedAddAndEditRouteScreen( waypointToRemove?.let { waypoint -> AlertDialog( + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId(), onDismissRequest = { waypointToRemove = null }, title = { Text(stringResource(Res.string.settings_reset_dialog_title)) }, text = { diff --git a/shared/src/iosMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt b/shared/src/iosMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt new file mode 100644 index 000000000..8b896aac2 --- /dev/null +++ b/shared/src/iosMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt @@ -0,0 +1,5 @@ +package org.scottishtecharmy.soundscape.platform + +import androidx.compose.ui.Modifier + +actual fun Modifier.testTagsAsResourceId(): Modifier = this From 1af3ce607a7a3522898dd10e423492105e904b26 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Fri, 16 Oct 2026 10:00:24 +0100 Subject: [PATCH 09/11] Test farm: roam the UI, set up once per release, restore after the monkey - Set up once per release (Onboarding and the suite), then run shuffled roaming flows that navigate the UI while the GPS walks. - Keep the app running across flows (LaunchHome brings it to the front and backs out to Home) and record the app process's uptime. - After the monkey, Restore puts back search mode, the map setting and markers B and C and Test Route without restarting the app; the phone is only set up again if that fails. It also completes a set-up whose suite flows failed. - Per-phone show_map, off for the low-memory Cubot. - Wi-Fi off on every phone; the offline map is pushed, or downloaded in-app through a USB proxy (adb reverse) where the shell can't write the app's files. - Per-phone Maestro cache and state dirs, so concurrent runs don't prune each other's logs and hang. - SwipeAndTap starts its swipe at 60% so small screens don't swipe from under the bottom bar; HomePage scrolls to About. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ex6Fdp5ftuhE1ZEyhZb7WQ --- docs/developers/maestro-tests.md | 18 +- docs/developers/test-farm.md | 31 ++- maestro/HomePage.yaml | 7 +- maestro/SwipeAndTap.yaml | 22 +- test-farm/.gitignore | 1 + test-farm/README.md | 75 ++++-- test-farm/farm.py | 301 +++++++++++++++++++++---- test-farm/farm.toml.example | 2 + test-farm/flows/DownloadExtract.yaml | 51 +++++ test-farm/flows/FarmSettings.yaml | 75 ++++++ test-farm/flows/LaunchHome.yaml | 47 ++++ test-farm/flows/OfflineSearch.yaml | 38 ---- test-farm/flows/PostUpgrade.yaml | 22 +- test-farm/flows/Restore.yaml | 125 ++++++++++ test-farm/flows/ShowMap.yaml | 104 +++++++++ test-farm/flows/StopBeacon.yaml | 17 ++ test-farm/flows/roam/Callouts.yaml | 49 ++++ test-farm/flows/roam/MarkerBrowse.yaml | 42 ++++ test-farm/flows/roam/Menus.yaml | 83 +++++++ test-farm/flows/roam/NearbyBeacon.yaml | 47 ++++ test-farm/flows/roam/RouteWalk.yaml | 33 +++ test-farm/flows/roam/Search.yaml | 82 +++++++ test-farm/usb_proxy.py | 98 ++++++++ 23 files changed, 1232 insertions(+), 138 deletions(-) create mode 100644 test-farm/flows/DownloadExtract.yaml create mode 100644 test-farm/flows/FarmSettings.yaml create mode 100644 test-farm/flows/LaunchHome.yaml delete mode 100644 test-farm/flows/OfflineSearch.yaml create mode 100644 test-farm/flows/Restore.yaml create mode 100644 test-farm/flows/ShowMap.yaml create mode 100644 test-farm/flows/StopBeacon.yaml create mode 100644 test-farm/flows/roam/Callouts.yaml create mode 100644 test-farm/flows/roam/MarkerBrowse.yaml create mode 100644 test-farm/flows/roam/Menus.yaml create mode 100644 test-farm/flows/roam/NearbyBeacon.yaml create mode 100644 test-farm/flows/roam/RouteWalk.yaml create mode 100644 test-farm/flows/roam/Search.yaml create mode 100644 test-farm/usb_proxy.py diff --git a/docs/developers/maestro-tests.md b/docs/developers/maestro-tests.md index fb03a7db6..6beacac7e 100644 --- a/docs/developers/maestro-tests.md +++ b/docs/developers/maestro-tests.md @@ -429,15 +429,19 @@ When you add a new flow, add its name (without the `.yaml` extension) to the ## On physical phones: the test farm -The same flows also run continuously on physical phones against **release -builds** in the [test farm]({% link developers/test-farm.md %}). The farm runs -them in `maestro/Master.yaml` order. It holds the phone at the same Edinburgh -point CI uses, and adds two flows of its own from `test-farm/flows/`: -`OfflineSearch.yaml` after Onboarding, and `PostUpgrade.yaml` after each release -upgrade. Two things matter when changing flows: +The same flows also run on physical phones against **release builds** in the +[test farm]({% link developers/test-farm.md %}), once for each release on each +phone, to set it up. The farm runs them in `maestro/Master.yaml` order, holding +the phone at the same Edinburgh point CI uses, with `FarmSettings.yaml` from +`test-farm/flows/` after Onboarding. After that the farm runs its own roaming +flows from `test-farm/flows/roam/` while the phone walks a GPX track, and +`PostUpgrade.yaml` after each release upgrade. Things that matter when +changing flows: - A flow that only passes on an emulator (for example, by relying on `adb root` or a debug-only screen) will fail there. - Changing what the suite leaves behind (markers **B** and **C** and **Test Route**) needs a matching change to `test-farm/flows/PostUpgrade.yaml`, - which checks they survive an upgrade. + which checks they survive an upgrade, and `test-farm/flows/Restore.yaml`, + which recreates them after the farm's monkey soak. The roaming flows use the markers and + the route too. diff --git a/docs/developers/test-farm.md b/docs/developers/test-farm.md index bcf3f8c61..6ce6accf5 100644 --- a/docs/developers/test-farm.md +++ b/docs/developers/test-farm.md @@ -25,13 +25,23 @@ Each Android phone loops through a **cycle**: data (`adb install -r`), then runs `test-farm/flows/PostUpgrade.yaml` to check that the app starts and the suite's markers and route survived the upgrade. -2. A `monkey` soak: random input inside the app, logged with its seed so a - failure can be reproduced. -3. The Maestro suite, in `maestro/Master.yaml` order. After `Onboarding.yaml` - it pushes an offline map and runs `test-farm/flows/OfflineSearch.yaml`, so - every flow runs offline. -4. It collects crashes and ANRs (from Android's dropbox), logcat, memory (PSS) - and battery. +2. Once for each release on each phone, it sets the phone up with the Maestro + suite, in `maestro/Master.yaml` order. After `Onboarding.yaml` it pushes an + offline map and runs `test-farm/flows/FarmSettings.yaml`, which sets search + to Offline. The suite runs with the map off, as in CI; `ShowMap.yaml` then + turns it on for roaming. +3. The roaming flows in `test-farm/flows/roam/`, shuffled, while the phone + walks a GPX track: callouts, beacons on nearby places and search results, + playing a route, the markers, the menus' screens, sleep and wake. They + look around without asserting what's nearby, because that changes as the + phone walks. They bring the app to the front rather than restarting it, + so one process runs for hours across rounds and cycles. +4. A `monkey` soak: random input inside the app, logged with its seed so a + failure can be reproduced. Then `Restore.yaml` puts back the search mode, + the map setting and any markers or route it deleted, without restarting + the app. The phone is only set up again if that fails. +5. It collects crashes and ANRs (from Android's dropbox), logcat, memory (PSS), + battery and the app process's uptime. Results go into `test-farm/farm.db`, with a summary page at `test-farm/report.html` that refreshes every minute. @@ -51,8 +61,9 @@ because its iOS driver needs Xcode on a Mac. Instead the farm uses | Devices | Emulators | Physical phones, including an AOSP phone without Play services | | Build | Debug | The release APK from the `Tag and build release` workflow | | Upgrades | Not tested | Each release is installed over the previous one, keeping the data | -| Map data | Glasgow extract pushed as root | The current extract from the manifest, pushed without root, with search set to Offline | +| Map data | Glasgow extract pushed as root | The current extract from the manifest, pushed without root, with search set to Offline and Wi-Fi turned off | | Location | `adb emu geo fix` | A replayed GPX walk with speed and bearing, held at CI's Edinburgh point while the suite runs | +| Map | Off (Onboarding turns it off) | On | | Duration | One run | Continuous | ## Making release builds testable @@ -73,8 +84,8 @@ The farm doesn't use any debug-only hooks: - Phones need **Screen lock: None**. Maestro can't get past a PIN, and the farm reports a locked phone instead of testing it. -- Phones must be in **English**. The Settings screen has no testTags, so - `OfflineSearch.yaml` taps its English labels. +- Phones must be in **English** for now. Releases before the Settings and + search result testTags are driven by English labels. - Remove any Play Store or debug copy of Soundscape first. The farm only tests a build it installed itself, and reports anything else as blocked. - Keep phones off USB hubs that lose disconnects. One here kept a ghost iPhone diff --git a/maestro/HomePage.yaml b/maestro/HomePage.yaml index 2f34718be..6a82d9043 100644 --- a/maestro/HomePage.yaml +++ b/maestro/HomePage.yaml @@ -36,8 +36,11 @@ appId: org.scottishtecharmy.soundscape id: sleepWakeUpNow - tapOn: id: topBarMenu -- tapOn: - id: menuAboutSoundscape +# Below the bottom of the drawer on small screens (480x854). +- runFlow: + file: SwipeAndTap.yaml + env: + id: menuAboutSoundscape - tapOn: id: appBarLeft - tapOn: diff --git a/maestro/SwipeAndTap.yaml b/maestro/SwipeAndTap.yaml index 49ce0ab78..39c9b1b47 100644 --- a/maestro/SwipeAndTap.yaml +++ b/maestro/SwipeAndTap.yaml @@ -22,6 +22,12 @@ appId: org.scottishtecharmy.soundscape # reverse the nudge to put it back - a partially-clipped target stays visible and # is left as nudged. # +# Swipes that move content up start no lower than 60% of the screen height: on +# a small screen (480x854) the home screen's fixed bottom bar starts at about +# 68%, and a swipe starting on it scrolls nothing. Each step moves 30%, less +# than the scrollable area between the top and bottom bars there (about 43%), +# so a target can't jump from below the fold to above the top in one step. +# # Note: swipe start/end coordinates can't be supplied via ${} variables (Maestro # parses them as literals and throws a number-format error), so the per-direction # coordinates are hard-coded below and selected with `when` conditions. @@ -60,8 +66,8 @@ appId: org.scottishtecharmy.soundscape true: "${output.swipeDir == 'up'}" commands: - swipe: - start: 50%, 80% - end: 50%, 50% + start: 50%, 60% + end: 50%, 30% # 'down' reveals items above (swipe the finger down). - runFlow: when: @@ -85,8 +91,8 @@ appId: org.scottishtecharmy.soundscape true: "${output.swipeFound == 1 && output.swipeDir == 'up'}" commands: - swipe: - start: 50%, 75% - end: 50%, 60% + start: 50%, 55% + end: 50%, 40% - runFlow: when: true: "${output.swipeFound == 1 && output.swipeDir == 'down'}" @@ -106,8 +112,8 @@ appId: org.scottishtecharmy.soundscape id: ${id} commands: - swipe: - start: 50%, 60% - end: 50%, 75% + start: 50%, 40% + end: 50%, 55% - runFlow: when: true: "${output.swipeFound == 1 && output.swipeDir == 'down'}" @@ -115,8 +121,8 @@ appId: org.scottishtecharmy.soundscape id: ${id} commands: - swipe: - start: 50%, 75% - end: 50%, 60% + start: 50%, 55% + end: 50%, 40% - tapOn: id: ${id} diff --git a/test-farm/.gitignore b/test-farm/.gitignore index 097cb6f99..0375d002f 100644 --- a/test-farm/.gitignore +++ b/test-farm/.gitignore @@ -7,3 +7,4 @@ runs/ __pycache__/ farm.log extracts/ +backup-*/ diff --git a/test-farm/README.md b/test-farm/README.md index d02ad8ec8..e3ccd36cd 100644 --- a/test-farm/README.md +++ b/test-farm/README.md @@ -6,9 +6,36 @@ plugged into this PC. Each phone loops forever: 1. If a newer release has been built, `adb install -r` it over the existing app and data, then run `flows/PostUpgrade.yaml` to check the app starts and the suite's markers and route survived. -2. A `monkey` soak (random input inside the app, fixed seed per run, logged). -3. The Maestro suite in `maestro/Master.yaml` order, one flow at a time. -4. Collect crashes and ANRs (from dropbox), logcat, memory (PSS) and battery. +2. **Set up**, once per release on each phone (recorded in the `setups` + table): the Maestro suite in `maestro/Master.yaml` order, one flow at a + time, held at `suite_location`. Onboarding wipes the app, so after it the + farm pushes the offline map and runs `flows/FarmSettings.yaml` (search mode + Offline). After the suite, which expects the map off as in CI, + `flows/ShowMap.yaml` turns the map on for roaming (or leaves it off + for a phone with `show_map = false` in its `[devices]` entry, which the + roaming flows then skip the map steps on). A phone whose Onboarding or + settings fail is set up again next cycle. +3. **Roam**: `roam_rounds` rounds of `flows/roam/*.yaml`, shuffled each round, + while the phone walks its GPX track. These look around (callouts, beacons on + places and search results, the route, markers, menus, sleep) and assert + only that the screens work, not what's nearby. They use the markers and + route the set-up made, and change nothing that later flows rely on. + Each starts with `flows/LaunchHome.yaml`, which brings the app to the + front and backs out to Home without restarting it, so one app process runs + through the rounds, the monkey and following cycles, soaking for hours. +4. A `monkey` soak (random input inside the app, fixed seed per run, logged). + Then `flows/Restore.yaml`, without restarting the app, puts back what the + monkey may have changed and roaming and the next upgrade rely on: search + mode Offline, markers B and C and Test Route (recreated where the suite + made them), and the map on. Only if that fails is the phone set up again + (a `reset` event). +5. Collect crashes and ANRs (from dropbox), logcat, memory (PSS), battery + and how long the app process has been up (`app_uptime_s`; a value that + resets without a crash means something else restarted the app). + +Flow results are recorded by kind: `flow` (set-up suite), `setup` +(FarmSettings, DownloadExtract, ShowMap), `roam`, `monkey`, `restore` and +`upgrade`. Releases come from the upstream `build-app.yaml` workflow's `release-apk` artifact (the APK that goes to Play), via `gh`. The newest two are cached in @@ -50,11 +77,20 @@ python3 test-farm/farm.py # Ctrl-C to stop; removes the mock GPS provider farm keeps the current manifest extract (`extract_name`, default Glasgow, which covers the suite point and the GPX walk) in `extracts/`, using `extractProviderUrl` from `local.properties`. Onboarding wipes app storage, - so after it runs in each cycle the farm `adb push`es the extract into the - app's `files/Download`. That needs no root on the phones tried so far. - It then runs `flows/OfflineSearch.yaml` to set Settings › Search › Search - mode to Offline. Settings has no test tags, so that flow taps English labels - and the phones must be set to English. The suite passes with Wi-Fi off. + so after it runs the farm `adb push`es the extract into the + app's `files/Download`. That needs no root on most phones. Where the + shell can't write to other apps' `Android/data` (the Cubot J10), the app + downloads the extract itself instead (`flows/DownloadExtract.yaml`), through + this PC's network: the farm runs a small HTTP proxy (`usb_proxy.py`), + `adb reverse`s its port onto the phone and sets the phone's global HTTP + proxy to it for the download only. + It then runs `flows/FarmSettings.yaml` to set Settings › Search › Search + mode to Offline. Releases + before the Settings test tags are driven by English labels, so the phones + must be set to English for now. The farm turns **Wi-Fi off** on every + phone when it's attached and after each monkey soak, so everything runs + offline. That also stops Google's fused location (on phones with Play + services) mixing the phone's real Wi-Fi position into the mock fixes. ## iPhones @@ -89,9 +125,12 @@ ghost iPhone connection that hung usbmuxd until it was removed via - Developer options on, USB debugging on, and on Xiaomi/Oppo-type phones also "USB debugging (Security settings)" so adb can tap and install. - Screen lock **None**. Maestro can't get past a PIN. -- Uninstall any existing Soundscape first. The farm only tests a build it - installed itself (it records the install time), so a Play Store or debug - copy leaves the phone **blocked** and untested, unless +- An existing Soundscape is fine if it's a sideloaded release. The farm + only tests a build it installed itself (it records the install time), but + it first tries installing the latest release over a foreign copy. A + release-signed copy accepts that, keeping its data like a user's update, + and the phone is adopted. A Play Store or debug copy is signed differently + and refuses, which leaves the phone **blocked** and untested unless `allow_reinstall = true`, which uninstalls the copy and its data. - Turn off Play Store auto-updates, so Play doesn't replace the farm's build. - Charge limit (e.g. 80%) where the phone has one. A phone on charge 24/7 @@ -107,11 +146,17 @@ service, that shows up as a failure, which is a real user problem. ## Known limits -- The suite wipes app data (Onboarding uses `clearState`), so a phone's data - only lives from the end of one cycle to the start of the next. That window - is where upgrades happen. +- Set-up wipes app data (Onboarding uses `clearState`), once per release. The + data a release upgrades over is whatever the previous release's cycles left. +- The monkey can change other settings (language, voice, callouts). + Restore only puts back search mode, the map and the markers and route. - The monkey can open the notification shade. After each soak the farm turns - off airplane mode, turns on Wi-Fi and collapses the shade. + off airplane mode, turns off Wi-Fi and collapses the shade. +- Each phone's Maestro gets its own `XDG_CACHE_HOME`/`XDG_STATE_HOME` + (`runs//maestro-home/`). Maestro prunes all but its newest few + runs' logs, so with a shared `~/.cache/maestro` a long run lost its log + folder to another phone's run, crashed at exit and hung until the flow + timeout. - Maestro runs on several phones at once. One run lost its driver connection (`gRPC UNAVAILABLE`) when another phone's session was starting, and that error is now retried once. If it keeps happening, set diff --git a/test-farm/farm.py b/test-farm/farm.py index cde50ae6f..e6d84ce18 100644 --- a/test-farm/farm.py +++ b/test-farm/farm.py @@ -26,6 +26,7 @@ import html import math import os +import random import re import signal import sqlite3 @@ -36,13 +37,19 @@ import tomllib import xml.etree.ElementTree as ET +import usb_proxy + FARM_DIR = os.path.dirname(os.path.abspath(__file__)) REPO_DIR = os.path.dirname(FARM_DIR) MAESTRO_DIR = os.path.join(REPO_DIR, "maestro") POST_UPGRADE_FLOW = os.path.join(FARM_DIR, "flows", "PostUpgrade.yaml") APK_DIR = os.path.join(FARM_DIR, "apks") EXTRACT_DIR = os.path.join(FARM_DIR, "extracts") -OFFLINE_SEARCH_FLOW = os.path.join(FARM_DIR, "flows", "OfflineSearch.yaml") +FARM_SETTINGS_FLOW = os.path.join(FARM_DIR, "flows", "FarmSettings.yaml") +DOWNLOAD_EXTRACT_FLOW = os.path.join(FARM_DIR, "flows", "DownloadExtract.yaml") +SHOW_MAP_FLOW = os.path.join(FARM_DIR, "flows", "ShowMap.yaml") +RESTORE_FLOW = os.path.join(FARM_DIR, "flows", "Restore.yaml") +ROAM_DIR = os.path.join(FARM_DIR, "flows", "roam") RUNS_DIR = os.path.join(FARM_DIR, "runs") DB_PATH = os.path.join(FARM_DIR, "farm.db") REPORT_PATH = os.path.join(FARM_DIR, "report.html") @@ -59,6 +66,8 @@ "release_artifact": "release-apk", "poll_minutes": 15, "monkey_events": 5000, + # Rounds of the roaming flows (flows/roam/, shuffled each round) per cycle. + "roam_rounds": 2, "monkey_throttle_ms": 250, "flow_timeout_minutes": 15, "keep_days": 14, @@ -150,10 +159,20 @@ def __init__(self, path): kind TEXT, name TEXT, status TEXT, duration_s REAL, detail TEXT, dir TEXT); CREATE TABLE IF NOT EXISTS installs( serial TEXT PRIMARY KEY, last_update TEXT); + CREATE TABLE IF NOT EXISTS setups( + serial TEXT PRIMARY KEY, version_code INTEGER); CREATE TABLE IF NOT EXISTS events( id INTEGER PRIMARY KEY, serial TEXT, ts REAL, kind TEXT, version_code INTEGER, detail TEXT); """) + try: # added after the table: whether the set-up suite all passed + self.conn.execute("ALTER TABLE setups ADD COLUMN complete INTEGER DEFAULT 1") + except sqlite3.OperationalError: + pass + try: # added after the table: how long the app process had been running + self.conn.execute("ALTER TABLE cycles ADD COLUMN app_uptime_s INTEGER") + except sqlite3.OperationalError: + pass def execute(self, sql, args=()): with self.lock: @@ -181,7 +200,7 @@ def builds(self): children_lock = threading.Lock() -def run(cmd, timeout=120): +def run(cmd, timeout=120, env=None): """subprocess.run, except that the farm can stop the process when it stops. A Maestro run left behind by a stopped farm keeps hold of the phone's driver, and every flow after a restart then fails with gRPC UNAVAILABLE.""" @@ -191,7 +210,8 @@ def run(cmd, timeout=120): if stop_event.is_set(): return subprocess.CompletedProcess(cmd, -1, "", "farm stopping") proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, errors="replace") + text=True, errors="replace", + env={**os.environ, **env} if env else None) children.add(proc) try: out, err = proc.communicate(timeout=timeout) @@ -334,9 +354,10 @@ def sync_extract(cfg): if not os.path.exists(path): log(None, f"downloading offline map {filename}") part = path + ".part" - dl = run(["curl", "-sfL", "-o", part, f"{base.rstrip('/')}/{filename}"], timeout=3600) + dl = run(["curl", "-sSfL", "-o", part, f"{base.rstrip('/')}/{filename}"], timeout=3600) if dl.returncode != 0: - log(None, f"offline map download failed: {dl.stderr.strip()}") + log(None, f"offline map download failed (exit {dl.returncode}): {dl.stderr.strip()}; " + "retrying at the next release poll") return os.replace(part, path) with open(path + ".geojson", "w") as f: @@ -437,7 +458,7 @@ def run(self): extras = f"--es lat {lat:.7f} --es lon {lon:.7f} --es speed {speed:.2f}" if bearing is not None: extras += f" --es bearing {bearing:.1f}" - out = shell(self.serial, f"am broadcast -n {HELPER_PKG}/.FixReceiver " + out = shell(self.serial, f"am broadcast -f 0x10000000 -n {HELPER_PKG}/.FixReceiver " f"-a {HELPER_PKG}.FIX {extras}", timeout=10) if 'data="ok"' in out: failures = 0 @@ -459,6 +480,10 @@ def initial_bearing(a, b): return (math.degrees(math.atan2(y, x)) + 360) % 360 +# Broadcasts to the helper go on the foreground queue (-f 0x10000000, +# FLAG_RECEIVER_FOREGROUND): on the Android 11 A50 the background queue backed +# up behind other apps' broadcasts and a fix took over 15 s to land. +# # The mock location helper in mock-location/. `cmd location providers` can only # set a position, so fixes from it have no speed or bearing; the helper sets # both. It isn't Soundscape, so the farm still tests only release builds. @@ -504,7 +529,7 @@ def disable_mock_location(serial, shell_providers=False): shell(serial, "; ".join(f"cmd location providers remove-test-provider {p}" for p in MOCK_PROVIDERS), timeout=20) else: - shell(serial, f"am broadcast -n {HELPER_PKG}/.FixReceiver -a {HELPER_PKG}.STOP", + shell(serial, f"am broadcast -f 0x10000000 -n {HELPER_PKG}/.FixReceiver -a {HELPER_PKG}.STOP", timeout=20) @@ -536,6 +561,27 @@ def pss_kb(serial): return int(m.group(1)) if m else None +def app_uptime_s(serial): + """Seconds the app's main process has been running, or None if it isn't.""" + pid = shell(serial, f"pidof {PACKAGE}").split() + if not pid: + return None + # ps prints ELAPSED as [[dd-]hh:]mm:ss. + out = shell(serial, f"ps -o ETIME= -p {pid[0]}").strip() + m = re.fullmatch(r"(?:(?:(\d+)-)?(\d+):)?(\d+):(\d+)", out) + if not m: + return None + d, h, mins, secs = (int(g or 0) for g in m.groups()) + return ((d * 24 + h) * 60 + mins) * 60 + secs + + +def format_uptime(seconds): + if seconds is None: + return "app not running" + h, rem = divmod(seconds, 3600) + return f"app up {h}h{rem // 60:02d}m" if h else f"app up {rem // 60}m" + + def battery(serial): out = shell(serial, "dumpsys battery") level = re.search(r"level: (\d+)", out) @@ -553,6 +599,10 @@ def suite_flows(): return re.findall(r"^- runFlow: (\S+\.yaml)", f.read(), re.M) +def roam_flows(): + return sorted(f for f in os.listdir(ROAM_DIR) if f.endswith(".yaml")) + + class Worker(threading.Thread): def __init__(self, serial, cfg, db): super().__init__(daemon=True, name=serial) @@ -564,12 +614,19 @@ def __init__(self, serial, cfg, db): self.android = shell(serial, "getprop ro.build.version.release") self.last_cycle_ok = False self.feeder = None + self.adopt_tried = set() # versions of a foreign install already tried + self.reinstall_driver = False # set after a Maestro run had to be killed # ----- setup def prepare(self): s = self.serial shell(s, "svc power stayon usb") + # Everything the farm tests works offline, and the farm's Wi-Fi is + # unreliable. It also keeps Wi-Fi scans out of Google's fused location, + # which would otherwise mix the phone's real position into the mock one. + shell(s, "svc wifi disable") + self.clear_usb_proxy() # in case the farm stopped mid-download # MainActivity uses dummy analytics when this is set, which keeps the # farm's traffic out of the production Firebase numbers. shell(s, "settings put system firebase.test.lab true") @@ -597,24 +654,49 @@ def quieten(self): def push_extract(self): """Copies the cached offline map and its metadata sidecar into the app's - extracts directory, named as CI does (without the build-date prefix).""" + extracts directory, named as CI does (without the build-date prefix). + Returns False if the phone won't allow it.""" if not current_extract: self.db.event(self.serial, "error", None, "no offline map cached; using the network") - return + return True set_state(self.serial, "pushing offline map") name = re.sub(r"^\d{8}-\d{4}-", "", os.path.basename(current_extract)) shell(self.serial, f"rm -f {DEVICE_EXTRACT_DIR}/*.pmtiles {DEVICE_EXTRACT_DIR}/*.geojson; " f"mkdir -p {DEVICE_EXTRACT_DIR}") for src, dest in ((current_extract, name), (current_extract + ".geojson", name + ".geojson")): res = adb(self.serial, "push", src, f"{DEVICE_EXTRACT_DIR}/{dest}", timeout=900) - if res.returncode != 0: + if res.returncode != 0 and not stop_event.is_set(): + if "Permission denied" in res.stdout + res.stderr: + return False self.db.event(self.serial, "error", None, f"offline map push failed:\n{res.stdout}{res.stderr}") - return + return True + return True + + def download_extract(self, cycle_id, cycle_dir): + """For phones whose shell can't write to the app's Android/data (the + Cubot J10): the app downloads the extract itself from its Offline maps + screen, through this PC's network. usb_proxy's port is reversed onto + the phone and set as its global HTTP proxy for the download only, so + Wi-Fi stays off (the Cubot's barely reaches the access point).""" + s = self.serial + set_state(s, "downloading the offline map over USB") + adb(s, "reverse", f"tcp:{usb_proxy.PORT}", f"tcp:{usb_proxy.PORT}") + shell(s, f"settings put global http_proxy 127.0.0.1:{usb_proxy.PORT}") + try: + return self.run_flow(cycle_id, DOWNLOAD_EXTRACT_FLOW, cycle_dir, kind="setup", + env={"EXTRACT": self.cfg["extract_name"]}, + timeout_minutes=70) + finally: + self.clear_usb_proxy() + + def clear_usb_proxy(self): + shell(self.serial, "settings put global http_proxy :0") + adb(self.serial, "reverse", "--remove", f"tcp:{usb_proxy.PORT}") def repair(self): """Undoes things the monkey can do from the notification shade.""" - shell(self.serial, "cmd connectivity airplane-mode disable; svc wifi enable; " + shell(self.serial, "cmd connectivity airplane-mode disable; svc wifi disable; " "cmd statusbar collapse; input keyevent KEYCODE_WAKEUP") self.quieten() @@ -654,6 +736,18 @@ def ensure_version(self, cycle_dir): return None latest = builds[-1] current = installed_version(self.serial) + if (current is not None and not self.farm_installed() + and current <= latest["version_code"] and current not in self.adopt_tried): + self.adopt_tried.add(current) + # A sideloaded release (same signing key) accepts the latest release + # over the top, keeping its data, just as a user's update would. Play + # Store and debug installs are signed differently and refuse it. + set_state(self.serial, f"adopting Soundscape {current}") + if self.install(latest, upgrade=True): + self.db.event(self.serial, "adopt", latest["version_code"], + f"adopted a sideloaded release {current} by installing " + f"{latest['version_name']} over it") + return installed_version(self.serial) if current is not None and not self.farm_installed(): if not self.cfg["allow_reinstall"]: set_state(self.serial, f"blocked: Soundscape {current} on the phone wasn't " @@ -689,24 +783,50 @@ def suite_location(self): # ----- tests - def run_flow(self, cycle_id, path, cycle_dir, kind="flow", note=""): + def run_flow(self, cycle_id, path, cycle_dir, kind="flow", note="", record=True, env=None, + timeout_minutes=None): + """Runs one Maestro flow and records its result. Returns True or False, + or None if the farm stopped it (which isn't recorded).""" name = os.path.splitext(os.path.basename(path))[0] out_dir = os.path.join(cycle_dir, name) + n = 1 + while os.path.exists(out_dir): # the roaming flows run more than once + n += 1 + out_dir = os.path.join(cycle_dir, f"{name}-{n}") os.makedirs(out_dir, exist_ok=True) report = os.path.join(out_dir, "report.xml") + # Whether the phone roams with the map on (show_map in its [devices] + # entry). ShowMap.yaml sets it, and the roaming flows skip the map + # without it. Suite flows ignore it. + env = {"SHOW_MAP": "true" if self.dev_cfg.get("show_map", True) else "false", + **(env or {})} cmd = [os.path.expanduser("~/.maestro/bin/maestro"), "test", "--device", self.serial, "--format", "junit", "--output", report, "--test-output-dir", out_dir, "--debug-output", os.path.join(out_dir, "debug"), "--flatten-debug-output", - "--no-ansi", path] - timeout = self.cfg["flow_timeout_minutes"] * 60 + "--no-ansi", *[a for k, v in env.items() for a in ("-e", f"{k}={v}")], path] + if self.reinstall_driver: + # A Maestro run killed at the timeout leaves the phone's driver + # unreachable, and the next flows fail at launch until it's + # reinstalled. + cmd = cmd[:2] + ["--reinstall-driver"] + cmd[2:] + self.reinstall_driver = False + timeout = (timeout_minutes or self.cfg["flow_timeout_minutes"]) * 60 + # Each Maestro run keeps its debug log in $XDG_CACHE_HOME/maestro/logs + # (else ~/.cache) and prunes all but the newest few runs' logs there. + # Shared between phones, a long run had its log folder deleted by + # another phone's run; its main thread then died at exit while the + # driver's threads kept the JVM alive, until the flow timeout. + own = os.path.join(RUNS_DIR, self.serial, "maestro-home") + maestro_env = {"XDG_CACHE_HOME": os.path.join(own, "cache"), + "XDG_STATE_HOME": os.path.join(own, "state")} start = time.time() for attempt in range(2): set_state(self.serial, f"running {name}") if self.cfg["serialize_maestro"]: with maestro_lock: - res = run(cmd, timeout=timeout) + res = run(cmd, timeout=timeout, env=maestro_env) else: - res = run(cmd, timeout=timeout) + res = run(cmd, timeout=timeout, env=maestro_env) output = res.stdout + res.stderr # The driver errors that decide a retry only reach the debug log. for extra in (report, os.path.join(out_dir, "debug", "maestro.log")): @@ -718,13 +838,24 @@ def run_flow(self, cycle_id, path, cycle_dir, kind="flow", note=""): if res.returncode == 0 or not MAESTRO_RETRY_RE.search(output) or attempt: break log(self.serial, f"{name}: transient launch failure, retrying once") - if "gRPC" in output: + if "gRPC" in output and "--reinstall-driver" not in cmd: # Maestro's on-device driver is unreachable (seen after a phone # sat on its lock screen); a fresh install brings it back. cmd = cmd[:2] + ["--reinstall-driver"] + cmd[2:] with open(os.path.join(out_dir, "maestro.log"), "w") as f: f.write(res.stdout + res.stderr) + if "timeout after" in res.stderr: + self.reinstall_driver = True + if stop_event.is_set(): + return None status = "pass" if res.returncode == 0 else "fail" + if status == "fail" and "timeout after" in res.stderr and "Flow Passed" in res.stdout: + # Maestro reported the pass and then hung on the way out until the + # flow timeout killed it (see maestro_env above). + status = "pass" + note = "\n".join(filter(None, [note, "Maestro hung after the flow passed; killed"])) + if not record: + return status == "pass" detail = "\n".join(filter(None, [note, "" if status == "pass" else (res.stderr or res.stdout)[-2000:]])) self.db.execute( @@ -753,6 +884,54 @@ def run_monkey(self, cycle_id, cycle_dir): self.repair() return ok + def set_up_version(self): + rows = self.db.query("SELECT version_code FROM setups WHERE serial=?", (self.serial,)) + return rows[0]["version_code"] if rows else None + + def set_up(self, cycle_id, cycle_dir, version, tally): + """Onboards from scratch and builds the state the roaming flows and + PostUpgrade use: runs the Maestro suite in Master.yaml order, held at + suite_location, with the offline map pushed and the farm's settings + applied after Onboarding. Done once per release on each phone.""" + # Onboarding wipes whatever an earlier set-up made, so until this one + # finishes the phone isn't set up (a farm stopped mid-way redoes it). + self.db.execute("DELETE FROM setups WHERE serial=?", (self.serial,)) + failed_before = tally[False] + with self.suite_location(): + for flow in suite_flows(): + if stop_event.is_set(): + return + result = self.run_flow(cycle_id, os.path.join(MAESTRO_DIR, flow), cycle_dir) + if result is not None: + tally[result] += 1 + if flow == "Onboarding.yaml": + if not result: + return # tried again next cycle + # Onboarding cleared the app's storage and settings, so put the + # offline map back and switch search to offline. + if not self.push_extract(): + result = self.download_extract(cycle_id, cycle_dir) + if result is not None: + tally[result] += 1 + if not result: + return + result = self.run_flow(cycle_id, FARM_SETTINGS_FLOW, cycle_dir, kind="setup") + if result is not None: + tally[result] += 1 + if not result: + return + if stop_event.is_set(): + return + # The suite runs with the map off, as in CI; roaming has it on. + result = self.run_flow(cycle_id, SHOW_MAP_FLOW, cycle_dir, kind="setup") + if result is None: + return + tally[result] += 1 + # Recorded even if a later suite flow failed: that's a result, and a + # missing marker or route is caught by the check after the monkey. + self.db.execute("INSERT OR REPLACE INTO setups VALUES(?,?,?)", + (self.serial, version, int(tally[False] == failed_before))) + def cycle(self): s = self.serial level, temp = battery(s) @@ -787,26 +966,62 @@ def cycle(self): "VALUES(?,?,?,?,?,?,?)", (s, self.model, self.android, version, time.time(), "running", cycle_dir)) - # Monkey first: Onboarding wipes whatever it leaves behind, and the suite - # then ends with the markers and route that PostUpgrade checks for. - self.run_monkey(cycle_id, cycle_dir) - passed = failed = 0 - with self.suite_location(): - for flow in suite_flows(): + tally = {True: 0, False: 0} + if self.set_up_version() != version: + self.set_up(cycle_id, cycle_dir, version, tally) + # The roaming flows only look around and don't assert what's nearby, so + # they run on the moving walk rather than held at suite_location. + # Without a finished set-up there's no offline map, settings, markers + # or route to roam with; the monkey still runs and set-up is retried + # next cycle. + flows = roam_flows() if self.set_up_version() == version else [] + complete = self.db.query("SELECT complete FROM setups WHERE serial=?", (s,)) + if flows and not complete[0]["complete"] and not stop_event.is_set(): + # A suite flow failed during set-up; build what it missed first. + with self.suite_location(): + restored = self.run_flow(cycle_id, RESTORE_FLOW, cycle_dir, kind="restore") + if restored is not None: + tally[restored] += 1 + if restored: + self.db.execute("UPDATE setups SET complete=1 WHERE serial=?", (s,)) + for _ in range(self.cfg["roam_rounds"]): + random.shuffle(flows) + for flow in flows: if stop_event.is_set(): break - if self.run_flow(cycle_id, os.path.join(MAESTRO_DIR, flow), cycle_dir): - passed += 1 - else: - failed += 1 - if flow == "Onboarding.yaml": - # Onboarding cleared the app's storage and settings, so put the - # offline map back and switch search to offline. - self.push_extract() - if self.run_flow(cycle_id, OFFLINE_SEARCH_FLOW, cycle_dir, kind="setup"): - passed += 1 - else: - failed += 1 + result = self.run_flow(cycle_id, os.path.join(ROAM_DIR, flow), cycle_dir, + kind="roam") + if result is not None: + tally[result] += 1 + if not stop_event.is_set(): + self.run_monkey(cycle_id, cycle_dir) + # The monkey can delete the markers or route, or change settings, and + # roaming and the next upgrade's PostUpgrade check need them. Restore + # puts them back without restarting the app; only if it can't is the + # phone set up again, now rather than at the start of the next cycle, + # which may begin with an upgrade. + # Restore also runs when a suite flow failed during set-up, and builds + # what that flow didn't; otherwise every roam round fails for want of + # a marker or the route until the next release. A reset only follows + # a set-up that completed: after one that didn't, it would most likely + # fail the same way and onboard again every cycle. + complete = self.db.query("SELECT complete FROM setups WHERE serial=?", (s,)) + if not stop_event.is_set() and self.set_up_version() == version: + with self.suite_location(): + restored = self.run_flow(cycle_id, RESTORE_FLOW, cycle_dir, kind="restore") + if restored is not None: + tally[restored] += 1 + if restored and not complete[0]["complete"]: + self.db.execute("UPDATE setups SET complete=1 WHERE serial=?", (s,)) + if restored is False and complete[0]["complete"]: + self.db.event(s, "reset", version, "couldn't restore the markers, route or " + "settings after the monkey soak; setting up again") + self.set_up(cycle_id, cycle_dir, version, tally) + passed, failed = tally[True], tally[False] + if stop_event.is_set(): + self.db.execute("UPDATE cycles SET ended=?, status=? WHERE id=?", + (time.time(), "stopped", cycle_id)) + return with open(os.path.join(cycle_dir, "logcat.txt"), "w") as f: f.write(adb(s, "logcat", "-d", timeout=120).stdout) @@ -820,12 +1035,14 @@ def cycle(self): level, temp = battery(s) status = "pass" if failed == 0 and not problems else "fail" self.last_cycle_ok = failed == 0 + uptime = app_uptime_s(s) self.db.execute( "UPDATE cycles SET ended=?, status=?, flows_passed=?, flows_failed=?, crashes=?, anrs=?, " - "pss_kb=?, battery_level=?, battery_temp=? WHERE id=?", - (time.time(), status, passed, failed, crashes, anrs, pss_kb(s), level, temp, cycle_id)) + "pss_kb=?, battery_level=?, battery_temp=?, app_uptime_s=? WHERE id=?", + (time.time(), status, passed, failed, crashes, anrs, pss_kb(s), level, temp, uptime, + cycle_id)) set_state(s, f"cycle {status}: {passed} passed, {failed} failed, " - f"{crashes} crashes, {anrs} ANRs") + f"{crashes} crashes, {anrs} ANRs, {format_uptime(uptime)}") def run(self): set_state(self.serial, "preparing", model=self.model, android=self.android) @@ -1125,7 +1342,10 @@ def write_report(db): def prune_runs(keep_days): cutoff = time.time() - keep_days * 86400 - for path in glob.glob(os.path.join(RUNS_DIR, "*", "*")): + # Per-cycle folders (named by timestamp) and iPhone syslogs; not the + # per-phone Maestro home or the iPhone's walk and player log. + for path in (glob.glob(os.path.join(RUNS_DIR, "*", "[0-9]*")) + + glob.glob(os.path.join(RUNS_DIR, "*", "syslog-*"))): if os.path.getmtime(path) < cutoff: subprocess.run(["rm", "-rf", path]) @@ -1152,6 +1372,7 @@ def handle_signal(*_): os.makedirs(EXTRACT_DIR, exist_ok=True) sync_extract(cfg) # before any phone starts, so the first cycle has it threading.Thread(target=release_watcher, args=(cfg, db), daemon=True).start() + usb_proxy.start() # network over USB for in-app map downloads workers = {} last_prune = 0 while not stop_event.is_set(): diff --git a/test-farm/farm.toml.example b/test-farm/farm.toml.example index 248dea11e..7bc7991f2 100644 --- a/test-farm/farm.toml.example +++ b/test-farm/farm.toml.example @@ -10,6 +10,7 @@ poll_minutes = 15 monkey_events = 5000 # several minutes; not every event waits the throttle monkey_throttle_ms = 250 +roam_rounds = 2 # rounds of flows/roam/ (shuffled) per cycle flow_timeout_minutes = 15 keep_days = 14 # per-cycle logs/screenshots under runs/ @@ -43,3 +44,4 @@ extract_name = "Glasgow" # [devices."R5CW12345AB"] # gpx = "../local-cubot.gpx" # skip = true +# show_map = false # keep the map off (low-memory phones); default true diff --git a/test-farm/flows/DownloadExtract.yaml b/test-farm/flows/DownloadExtract.yaml new file mode 100644 index 000000000..abd12a5b1 --- /dev/null +++ b/test-farm/flows/DownloadExtract.yaml @@ -0,0 +1,51 @@ +appId: org.scottishtecharmy.soundscape +# Downloads the offline map ${EXTRACT} from Menu > Offline maps. The farm runs +# this on phones where it can't adb push the extract (the Cubot J10's shell +# can't write to other apps' Android/data), with the phone's HTTP proxy set to +# the farm's usb_proxy over adb reverse, as Wi-Fi is off. The extract's name +# is map data rather than UI text, so it's matched by name. +--- +- runFlow: LaunchHome.yaml +- tapOn: + id: topBarMenu +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: menuOfflineMaps +# The nearby extracts come from the manifest, which needs the network. +- extendedWaitUntil: + visible: + text: ${EXTRACT} + timeout: 60000 +# Onboarding's clearState doesn't always remove a downloaded extract (it +# survived on the Cubot), so it may already be in "Maps already downloaded", +# which is listed first. Its details then offer Delete rather than Download. +- tapOn: + text: ${EXTRACT} + index: 0 +- runFlow: + when: + visible: + id: offlineMapDownload + commands: + - tapOn: + id: offlineMapDownload + # The progress screen shows a percentage until the download finishes + # and the list comes back, now with the extract in the downloaded + # section. 177 MB for Glasgow; minutes to half an hour over USB. + - extendedWaitUntil: + visible: + text: ".*%" + timeout: 30000 + optional: true + - extendedWaitUntil: + notVisible: + text: ".*%" + timeout: 3600000 + - tapOn: + text: ${EXTRACT} + index: 0 +- assertVisible: + id: offlineMapDelete +- back +- back diff --git a/test-farm/flows/FarmSettings.yaml b/test-farm/flows/FarmSettings.yaml new file mode 100644 index 000000000..229b82f78 --- /dev/null +++ b/test-farm/flows/FarmSettings.yaml @@ -0,0 +1,75 @@ +appId: org.scottishtecharmy.soundscape +# Run by the test farm after Onboarding (which wipes the app's settings): +# - Settings > Search > Search mode: Offline, so searches use the offline map +# extract the farm pushes instead of the phones' unreliable Wi-Fi. +# +# The map is turned on separately (ShowMap.yaml) once the set-up suite is done, +# because the suite's flows expect the map off, as in CI. +# +# Releases before the Settings testTags (settingsSection*, settingsSearchMode, +# settingsOption-*) are driven by their English labels +# instead, so the farm phones must be set to English until every release in +# apks/ has the tags. Then the label fallbacks can go. +--- +- runFlow: LaunchHome.yaml +- tapOn: + id: topBarMenu +# Second in the drawer, so never off-screen; SwipeAndTap's nudge scrolled it +# away on the slow Cubot. +- tapOn: + id: menuSettings +- runFlow: + file: ../../maestro/Wait.yaml + env: + timeout: 2000 +# The first section header is always on screen, so it says which kind of +# release this is. +- evalScript: ${output.tagged = false} +- runFlow: + when: + visible: + id: settingsSectionCallouts + commands: + - evalScript: ${output.tagged = true} + +# Search mode: Offline. +- runFlow: + when: + true: ${output.tagged} + commands: + - scrollUntilVisible: + element: + id: settingsSectionSearch + direction: DOWN + - tapOn: + id: settingsSectionSearch + - scrollUntilVisible: + element: + id: settingsSearchMode + direction: DOWN + - tapOn: + id: settingsSearchMode + - tapOn: + id: settingsOption-Offline +- runFlow: + when: + true: ${!output.tagged} + commands: + - scrollUntilVisible: + element: + text: "Search" + direction: DOWN + - tapOn: + text: "Search" + - scrollUntilVisible: + element: + text: "Search mode" + direction: DOWN + - tapOn: + text: "Search mode" + - tapOn: + text: "Offline" + # The summary under "Search mode" now names the chosen option. + - assertVisible: + text: ".*Offline.*" +- back diff --git a/test-farm/flows/LaunchHome.yaml b/test-farm/flows/LaunchHome.yaml new file mode 100644 index 000000000..5cbf0564b --- /dev/null +++ b/test-farm/flows/LaunchHome.yaml @@ -0,0 +1,47 @@ +appId: org.scottishtecharmy.soundscape +# Brings the app to the front and gets it back to Home, dismissing the release +# notes dialog if it shows. It appears on the first launch of a new release +# and can turn up after Onboarding; while it's up, Home's anchor isn't visible +# to Maestro. +# +# The app isn't restarted (stopApp: false), so one process runs through all +# the roam rounds and soaks for hours. The previous flow may have left it on +# any screen, or with the search or a dialog open, so it backs out to Home. +# +# Releases before the dialog's testTags were exposed (a dialog is its own +# window, outside the theme's testTagsAsResourceId) need its English button +# text. +--- +- launchApp: + clearState: false + stopApp: false +- runFlow: + file: ../../maestro/Wait.yaml + env: + timeout: 3000 +- tapOn: + id: newReleaseDialogDismiss + optional: true +- tapOn: + text: "Close Release Notes" + optional: true +- repeat: + while: + notVisible: + id: mainSearchBarCollapsed + times: 6 + commands: + - back + - runFlow: + file: ../../maestro/Wait.yaml + env: + timeout: 1000 +# Backing out of Home closes the activity (the process and service keep +# running), so bring it back if that happened. +- launchApp: + clearState: false + stopApp: false +- runFlow: + file: ../../maestro/WaitForScreen.yaml + env: + screen: Home diff --git a/test-farm/flows/OfflineSearch.yaml b/test-farm/flows/OfflineSearch.yaml deleted file mode 100644 index f5c13ff65..000000000 --- a/test-farm/flows/OfflineSearch.yaml +++ /dev/null @@ -1,38 +0,0 @@ -appId: org.scottishtecharmy.soundscape -# Run by the test farm after Onboarding (which wipes the app's settings) to -# switch Settings > Search > Search mode to Offline, so searches use the -# offline map extract the farm pushes instead of the phones' unreliable Wi-Fi. -# -# The Settings screen has no testTags, so this matches the English labels -# (settings_section / menu_manage_search, settings_section_search_network, -# settings_search_offline): the farm phones must be set to English. ---- -- launchApp: - clearState: false -- runFlow: - file: ../../maestro/WaitForScreen.yaml - env: - screen: Home -- tapOn: - id: topBarMenu -- runFlow: - file: ../../maestro/SwipeAndTap.yaml - env: - id: menuSettings -- scrollUntilVisible: - element: - text: "Search" - direction: DOWN -- tapOn: - text: "Search" -- scrollUntilVisible: - element: - text: "Search mode" - direction: DOWN -- tapOn: - text: "Search mode" -- tapOn: - text: "Offline" -# The summary under "Search mode" now names the chosen option. -- assertVisible: - text: ".*Offline.*" diff --git a/test-farm/flows/PostUpgrade.yaml b/test-farm/flows/PostUpgrade.yaml index d4f83278d..ab8d81278 100644 --- a/test-farm/flows/PostUpgrade.yaml +++ b/test-farm/flows/PostUpgrade.yaml @@ -4,18 +4,9 @@ appId: org.scottishtecharmy.soundscape # the upgraded app starts on the old data, and that the markers and the route # the suite created survived the upgrade. --- -- launchApp: - clearState: false - -# The release dialog shows on the first launch of a new major.minor version. -- tapOn: - id: "newReleaseDialogDismiss" - optional: true - -- runFlow: - file: ../../maestro/WaitForScreen.yaml - env: - screen: Home +# The release dialog shows on the first launch of a new major.minor version; +# LaunchHome dismisses it. +- runFlow: LaunchHome.yaml - runFlow: file: ../../maestro/SwipeAndTap.yaml env: @@ -26,12 +17,7 @@ appId: org.scottishtecharmy.soundscape - assertVisible: id: "LocationItem-C-.*" -- launchApp: - clearState: false -- runFlow: - file: ../../maestro/WaitForScreen.yaml - env: - screen: Home +- runFlow: LaunchHome.yam - runFlow: file: ../../maestro/SwipeAndTap.yaml env: diff --git a/test-farm/flows/Restore.yaml b/test-farm/flows/Restore.yaml new file mode 100644 index 000000000..ca04995ce --- /dev/null +++ b/test-farm/flows/Restore.yaml @@ -0,0 +1,125 @@ +appId: org.scottishtecharmy.soundscape +# Run by the test farm after the monkey soak, held at suite_location, without +# restarting the app. Puts back what the roaming flows and the next upgrade's +# PostUpgrade.yaml rely on and the monkey may have changed: +# - search mode Offline (FarmSettings.yaml); +# - markers B and C and "Test Route" (B -> C), recreated the way the suite +# made them if they've gone; +# - the map as the farm wants it for roaming (SHOW_MAP, the phone's show_map). +# Fails if they can't be restored, and the farm then sets the phone up again. +# +# Recreating is idempotent: B is saved from the nearest place and C at the +# held location, the same spots the suite used, and saving a marker where one +# already is (say one the monkey renamed) updates that marker. +--- +- runFlow: FarmSettings.yaml +# The markers are made with the map off, as in the suite: on a small screen +# the map pushes controls under the bottom bar. +- runFlow: + file: ShowMap.yaml + env: + SHOW: "false" + +# Marker B: the first place nearby. +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homeMarkers +- evalScript: ${output.haveB = false} +- runFlow: + when: + visible: + id: "LocationItem-B-.*" + commands: + - evalScript: ${output.haveB = true} +- runFlow: LaunchHome.yaml +- runFlow: + when: + true: ${!output.haveB} + commands: + - runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homePlacesNearby + direction: down + - tapOn: + id: placesNearby-0 + - tapOn: + id: placesNearby-0 + - tapOn: + id: locationDetailsSaveAsMarker + optional: true + - tapOn: + id: locationDetailsEditMarker + optional: true + - tapOn: + id: name-clearTextField + optional: true + - tapOn: + id: markerName + - inputText: "B" + - tapOn: + id: flexibleAppBarRight + - runFlow: LaunchHome.yaml + +# Marker C: the current (held) location, from the Markers screen's New button. +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homeMarkers +- runFlow: + when: + notVisible: + id: "LocationItem-C-.*" + commands: + - tapOn: + id: appBarRight + - tapOn: + id: addMarkerCurrentLocation + - tapOn: + id: name-clearTextField + optional: true + - tapOn: + id: markerName + - inputText: "C" + - tapOn: + id: flexibleAppBarRight +- assertVisible: + id: "LocationItem-B-.*" +- assertVisible: + id: "LocationItem-C-.*" +- runFlow: LaunchHome.yaml + +# The route, B then C. +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homeRoutes +- runFlow: + when: + notVisible: + text: "Test Route.*" + commands: + - tapOn: + id: appBarRight + - tapOn: + id: routeName + - inputText: "Test Route" + - pressKey: Enter + - tapOn: + id: addWaypointsButton + - tapOn: + id: "LocationItem-B-.*" + - tapOn: + id: "LocationItem-C-.*" + - tapOn: + id: flexibleAppBarRight + - tapOn: + id: flexibleAppBarRight +- assertVisible: + text: "Test Route.*" + +- runFlow: + file: ShowMap.yaml + env: + SHOW: ${SHOW_MAP} diff --git a/test-farm/flows/ShowMap.yaml b/test-farm/flows/ShowMap.yaml new file mode 100644 index 000000000..344b76d48 --- /dev/null +++ b/test-farm/flows/ShowMap.yaml @@ -0,0 +1,104 @@ +appId: org.scottishtecharmy.soundscape +# Sets Settings > Accessibility > Show maps on screen to SHOW ("true" or +# "false"), or if that isn't given to SHOW_MAP, the phone's show_map, which +# the farm passes to every flow (default "true"). Run by the +# test farm after the set-up suite, to turn the map on for roaming, and by +# Restore.yaml. Onboarding.yaml turns the +# map off and the suite's flows expect it off (on a small screen the map +# pushes controls under the bottom bar), but the farm wants the map drawing +# on real phones while it roams. +# +# Releases before the settingsSectionAccessibility and settingsShowMap +# testTags are driven by their English labels. +--- +- runFlow: LaunchHome.yaml +# Whether the map is on is read from Home (its full screen button only shows +# with the map), because Maestro's checked: matches the switch's label as well +# as the switch. +- evalScript: ${output.mapOn = false} +- evalScript: | + ${ + const show = typeof SHOW !== 'undefined' ? SHOW + : typeof SHOW_MAP !== 'undefined' ? SHOW_MAP : 'true'; + output.want = show !== 'false'; + } +- runFlow: + when: + visible: + id: homeFullScreenMap + commands: + - evalScript: ${output.mapOn = true} +- tapOn: + id: topBarMenu +# Second in the drawer, so never off-screen; SwipeAndTap's nudge scrolled it +# away on the slow Cubot. +- tapOn: + id: menuSettings +- runFlow: + file: ../../maestro/Wait.yaml + env: + timeout: 2000 +- evalScript: ${output.tagged = false} +- runFlow: + when: + visible: + id: settingsSectionCallouts + commands: + - evalScript: ${output.tagged = true} +- runFlow: + when: + true: ${output.tagged} + commands: + - scrollUntilVisible: + element: + id: settingsSectionAccessibility + direction: DOWN + - tapOn: + id: settingsSectionAccessibility + - scrollUntilVisible: + element: + id: settingsShowMap + direction: DOWN + - runFlow: + when: + true: ${output.mapOn !== output.want} + commands: + - tapOn: + id: settingsShowMap +- runFlow: + when: + true: ${!output.tagged} + commands: + - scrollUntilVisible: + element: + text: "Accessibility" + direction: DOWN + - tapOn: + text: "Accessibility" + - scrollUntilVisible: + element: + text: "Show maps on screen" + direction: DOWN + - runFlow: + when: + true: ${output.mapOn !== output.want} + commands: + - tapOn: + text: "Show maps on screen" +- back +- runFlow: + file: ../../maestro/WaitForScreen.yaml + env: + screen: Home +- runFlow: + when: + true: ${output.want} + commands: + - assertVisible: + id: homeFullScreenMap +- runFlow: + when: + true: ${!output.want} + commands: + - assertNotVisible: + id: homeFullScreenMap diff --git a/test-farm/flows/StopBeacon.yaml b/test-farm/flows/StopBeacon.yaml new file mode 100644 index 000000000..34223f946 --- /dev/null +++ b/test-farm/flows/StopBeacon.yaml @@ -0,0 +1,17 @@ +appId: org.scottishtecharmy.soundscape +# Stops the audio beacon if one is still playing. A beacon on a nearby place +# ends by itself when the walk reaches it, so there may be nothing to stop. +# The beacon card can sit below the fold, so look further down the home +# screen before deciding it has gone. +--- +- runFlow: + when: + notVisible: + id: routeStop + commands: + - swipe: + start: 50%, 70% + end: 50%, 40% +- tapOn: + id: routeStop + optional: true diff --git a/test-farm/flows/roam/Callouts.yaml b/test-farm/flows/roam/Callouts.yaml new file mode 100644 index 000000000..72bc3d245 --- /dev/null +++ b/test-farm/flows/roam/Callouts.yaml @@ -0,0 +1,49 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow (see test-farm/README.md): the home screen's callout buttons and +# the full screen map, while the farm walks the phone along its GPX track. The +# waits give the callouts time to play and the walk time to move on, so the +# automatic callouts run in between. +--- +- runFlow: ../LaunchHome.yaml + +- tapOn: + id: homeMyLocation +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 15000 +- tapOn: + id: homeAroundMe +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 30000 +- tapOn: + id: homeAheadOfMe +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 30000 +- tapOn: + id: homeNearbyMarkers +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 10000 + +# Watch the map follow the walk for a while, then put the controls back. +# Phones with show_map = false (the Cubot) have no map. +- runFlow: + when: + true: ${SHOW_MAP !== 'false'} + commands: + - tapOn: + id: homeFullScreenMap + - runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 60000 + - tapOn: + id: homeFullScreenMap +- assertVisible: + id: homeMyLocation diff --git a/test-farm/flows/roam/MarkerBrowse.yaml b/test-farm/flows/roam/MarkerBrowse.yaml new file mode 100644 index 000000000..5bc677d05 --- /dev/null +++ b/test-farm/flows/roam/MarkerBrowse.yaml @@ -0,0 +1,42 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow: flips the Markers list's sort order and opens each of the +# suite's markers (B and C) as the walk moves away from them. Changes nothing. +--- +- runFlow: ../LaunchHome.yaml + +- runFlow: + file: ../../../maestro/SwipeAndTap.yaml + env: + id: homeMarkers +- tapOn: + id: SortOption +- tapOn: + id: SortOption + +- tapOn: + id: "LocationItem-B-.*" +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: LocationDetails +- back +- tapOn: + id: "LocationItem-C-.*" +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: LocationDetails +- runFlow: + when: + true: ${SHOW_MAP !== 'false'} + commands: + - tapOn: + id: locationDetailsFullScreenMapFab + - runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 5000 + - tapOn: + id: locationDetailsFullScreenMapFab +- back +- back diff --git a/test-farm/flows/roam/Menus.yaml b/test-farm/flows/roam/Menus.yaml new file mode 100644 index 000000000..7977be0df --- /dev/null +++ b/test-farm/flows/roam/Menus.yaml @@ -0,0 +1,83 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow: looks around the menu's screens without changing anything +# (help topics, offline maps, settings, about) and puts the app to sleep and +# wakes it again. Uses testTags only, so it doesn't depend on the language. +--- +- runFlow: ../LaunchHome.yaml + +# Help: open three topics. +- tapOn: + id: topBarMenu +- tapOn: + id: menuHelpAndTutorials +- tapOn: + id: "helpScreenTopic.*" + index: 0 +- back +- tapOn: + id: "helpScreenTopic.*" + index: 2 +- back +- tapOn: + id: "helpScreenTopic.*" + index: 4 +- back +- back + +# Offline maps: the list of extracts covering the current location. +- tapOn: + id: topBarMenu +- tapOn: + id: menuOfflineMaps +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 5000 +- back + +# Settings: scroll to the bottom and back out without touching anything. +- tapOn: + id: topBarMenu +- tapOn: + id: menuSettings +# Settings has no testTags (WaitForScreen's Settings anchor no longer exists). +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 3000 +- repeat: + times: 4 + commands: + - swipe: + start: 50%, 80% + end: 50%, 30% +- back + +- tapOn: + id: topBarMenu +- runFlow: + file: ../../../maestro/SwipeAndTap.yaml + env: + id: menuAboutSoundscape +- back +# Some screens come back to the open menu drawer rather than Home. +- tapOn: + id: menuDrawerBack + optional: true + +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: Home +- tapOn: + id: topBarSleep +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 30000 +- tapOn: + id: sleepWakeUpNow +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: Home diff --git a/test-farm/flows/roam/NearbyBeacon.yaml b/test-farm/flows/roam/NearbyBeacon.yaml new file mode 100644 index 000000000..f7df626cd --- /dev/null +++ b/test-farm/flows/roam/NearbyBeacon.yaml @@ -0,0 +1,47 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow: picks a place from Places Nearby, looks at its details and map, +# then walks with an audio beacon on it for a minute before stopping it. +# The walk reorders the list, so the flow doesn't care which place it gets. +--- +- runFlow: ../LaunchHome.yaml + +- tapOn: + id: homePlacesNearby +# The first folder is "All", then the places in distance order. +- tapOn: + id: placesNearby-0 +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: PlacesNearby +- tapOn: + id: placesNearby-0 +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: LocationDetails + +- runFlow: + when: + true: ${SHOW_MAP !== 'false'} + commands: + - tapOn: + id: locationDetailsFullScreenMapFab + - runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 5000 + - tapOn: + id: locationDetailsFullScreenMapFab + +- tapOn: + id: locationDetailsStartBeacon +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: Home +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 60000 +- runFlow: ../StopBeacon.yaml diff --git a/test-farm/flows/roam/RouteWalk.yaml b/test-farm/flows/roam/RouteWalk.yaml new file mode 100644 index 000000000..b56795b55 --- /dev/null +++ b/test-farm/flows/roam/RouteWalk.yaml @@ -0,0 +1,33 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow: plays the suite's "Test Route" for two minutes of the walk, +# then stops it. The route's markers are at the suite point, so the beacon is +# a long way off, which is fine for exercising route playback. +--- +- runFlow: ../LaunchHome.yaml + +- runFlow: + file: ../../../maestro/SwipeAndTap.yaml + env: + id: homeRoutes +- tapOn: + id: "LocationItem-Test Route-.*" +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: RouteDetails +- tapOn: + id: routeDetailsStartButton +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: Home +- assertVisible: + id: routeBeaconTitle +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 120000 +- runFlow: + file: ../../../maestro/SwipeAndTap.yaml + env: + id: routeStop diff --git a/test-farm/flows/roam/Search.yaml b/test-farm/flows/roam/Search.yaml new file mode 100644 index 000000000..97060cc52 --- /dev/null +++ b/test-farm/flows/roam/Search.yaml @@ -0,0 +1,82 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow: an offline search for a random common word, then a beacon on +# the first result for half a minute. Search mode was set to Offline when the +# farm set the phone up, so this searches the pushed extract. +--- +- runFlow: ../LaunchHome.yaml + +- evalScript: | + ${ + const words = ['Station', 'Park', 'Church', 'School', 'Road', 'Library', 'Bank', 'Cafe']; + output.searchWord = words[Math.floor(Math.random() * words.length)]; + } +- tapOn: + id: mainSearchBarCollapsed +# The search field takes focus as it opens. +- inputText: ${output.searchWord} +- pressKey: Enter +# Offline search takes a few seconds, longer on slow phones. +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 15000 +# Releases before searchResult-* (the results are in a Dialog, whose testTags +# weren't exposed) need the result found by its text: by a distance in its +# accessibility text, or failing that by the searched word. A road's text is +# just its address, with no distance. Index 1 skips the search field, which +# holds the word too. +- evalScript: ${output.tapped = false} +- runFlow: + when: + visible: + id: searchResult-0 + commands: + - evalScript: ${output.tapped = true} + - tapOn: + id: searchResult-0 +- runFlow: + when: + true: ${!output.tapped} + visible: + text: "(?s).*(metres|km|kilometres|miles|feet).*" + commands: + - evalScript: ${output.tapped = true} + - tapOn: + text: "(?s).*(metres|km|kilometres|miles|feet).*" + index: 0 +- runFlow: + when: + true: ${!output.tapped} + visible: + text: "(?s).*${output.searchWord}.*" + index: 1 + commands: + - evalScript: ${output.tapped = true} + - tapOn: + text: "(?s).*${output.searchWord}.*" + index: 1 +# Nothing called that nearby: the search still ran, so that's not a failure. +- runFlow: + when: + true: ${!output.tapped} + commands: + - back +- runFlow: + when: + true: ${output.tapped} + commands: + - runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: LocationDetails + - tapOn: + id: locationDetailsStartBeacon + - runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: Home + - runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 30000 + - runFlow: ../StopBeacon.yaml diff --git a/test-farm/usb_proxy.py b/test-farm/usb_proxy.py new file mode 100644 index 000000000..a17d543e2 --- /dev/null +++ b/test-farm/usb_proxy.py @@ -0,0 +1,98 @@ +"""A minimal HTTP proxy for giving farm phones network access over USB. + +The farm runs it on this PC, `adb reverse`s its port onto a phone and sets the +phone's global HTTP proxy to 127.0.0.1:, so apps that honour the system +proxy (OkHttp does) reach the internet through the PC with Wi-Fi off. It +handles CONNECT (HTTPS tunnels, which is all the app uses) and plain HTTP. +""" + +import select +import socket +import socketserver +import threading +import time +from urllib.parse import urlsplit + +PORT = 8118 + + +class _Handler(socketserver.StreamRequestHandler): + timeout = 60 + + def handle(self): + line = self.rfile.readline(65537).decode("latin-1") + try: + method, target, version = line.split() + except ValueError: + return + headers = [] + while True: + h = self.rfile.readline(65537) + if h in (b"\r\n", b"\n", b""): + break + headers.append(h) + if method == "CONNECT": + host, _, port = target.rpartition(":") + try: + upstream = socket.create_connection((host, int(port)), timeout=30) + except OSError: + self.wfile.write(b"HTTP/1.1 502 Bad Gateway\r\n\r\n") + return + self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + self.wfile.flush() + leftover = self.rfile.read1(65536) if self.rfile.peek(1) else b"" + if leftover: + upstream.sendall(leftover) + else: + url = urlsplit(target) + try: + upstream = socket.create_connection((url.hostname, url.port or 80), timeout=30) + except OSError: + self.wfile.write(b"HTTP/1.1 502 Bad Gateway\r\n\r\n") + return + path = url.path or "/" + if url.query: + path += "?" + url.query + upstream.sendall(f"{method} {path} {version}\r\n".encode("latin-1")) + upstream.sendall(b"".join(h for h in headers + if not h.lower().startswith(b"proxy-")) + b"\r\n") + self._pipe(self.connection, upstream) + + @staticmethod + def _pipe(a, b): + socks = [a, b] + try: + while True: + ready, _, _ = select.select(socks, [], [], 120) + if not ready: + return + for s in ready: + data = s.recv(65536) + if not data: + return + (b if s is a else a).sendall(data) + except OSError: + pass + finally: + b.close() + + +class _Server(socketserver.ThreadingTCPServer): + daemon_threads = True + allow_reuse_address = True + + +def start(port=PORT): + """Starts the proxy on 127.0.0.1 in a background thread. Retries for a + while if the port is busy, as it is while a farm that's just been + stopped is still shutting down.""" + for attempt in range(30): + try: + server = _Server(("127.0.0.1", port), _Handler) + break + except OSError: + if attempt == 29: + raise + time.sleep(1) + threading.Thread(target=server.serve_forever, daemon=True, name="usb-proxy").start() + return server From fdfed5f3f3a1d07c60acb3d838aa948e79215856 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Fri, 16 Oct 2026 10:08:56 +0100 Subject: [PATCH 10/11] Test farm: collect crashes and ANRs one dropbox tag at a time dumpsys dropbox only prints entries that match every argument, so asking for four tags in one call printed nothing and the farm never recorded a crash or ANR on any phone. The Cubot's monkey ANRs were in dropbox but never reached the farm. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ex6Fdp5ftuhE1ZEyhZb7WQ --- test-farm/farm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test-farm/farm.py b/test-farm/farm.py index e6d84ce18..12d10fd13 100644 --- a/test-farm/farm.py +++ b/test-farm/farm.py @@ -545,7 +545,10 @@ def collect_dropbox(serial, since): `since` is a device-local "YYYY-MM-DD HH:MM:SS" string, which is also the format dropbox prints entries with, so plain string comparison works. """ - out = shell(serial, "dumpsys dropbox --print " + " ".join(DROPBOX_TAGS), timeout=60) + # One tag per call: dumpsys dropbox only prints entries that match every + # argument, so several tags in one call print nothing. + out = "\n".join(shell(serial, f"dumpsys dropbox --print {tag}", timeout=60) + for tag in DROPBOX_TAGS) found = [] for entry in re.split(r"^={10,}\s*$", out, flags=re.M): m = re.search(r"^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d) (\S+)", entry.strip(), re.M) From fbc04cbf87e33a4949f306fc41a7ccb0bdd272c7 Mon Sep 17 00:00:00 2001 From: Dave Craig Date: Fri, 16 Oct 2026 10:48:38 +0100 Subject: [PATCH 11/11] Check the test farm in run-tests A new test-farm job compiles farm.py, builds the mock location helper and runs test-farm/test_farm.py, which needs no phone. It checks: - the example config only uses settings farm.py knows - the flows' runFlow paths into maestro/ resolve - the suite location and launch-failure retries still agree with run-maestro-tests.yaml - the dropbox, uptime, meminfo and battery parsers, and GPX track replay Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ex6Fdp5ftuhE1ZEyhZb7WQ --- .github/workflows/run-tests.yaml | 30 +++++ test-farm/README.md | 9 ++ test-farm/test_farm.py | 216 +++++++++++++++++++++++++++++++ 3 files changed, 255 insertions(+) create mode 100644 test-farm/test_farm.py diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 595d12cd2..bdd9b9502 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -50,6 +50,36 @@ jobs: - name: Run lint over the repo run: ./gradlew lint + # The test farm (test-farm/) runs on phones at home, not here, so this only + # checks what can break without a device: farm.py compiles, its flows' paths + # into maestro/ resolve, the settings it shares with run-maestro-tests.yaml + # still agree, and its adb-output parsers work (test-farm/test_farm.py). It + # also builds the mock location helper. The runner image already has the + # Python (3.11+ for tomllib), JDK and Android SDK these need. + test-farm: + name: Check test farm + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + + - name: Compile farm scripts + run: python3 -m py_compile test-farm/farm.py test-farm/usb_proxy.py + + - name: Unit-test farm.py + run: python3 -m unittest discover -s test-farm -v + + # build.sh signs with ~/.android/debug.keystore, which a fresh runner lacks. + - name: Build mock location helper + run: | + if [ ! -f ~/.android/debug.keystore ]; then + mkdir -p ~/.android + keytool -genkeypair -keystore ~/.android/debug.keystore -storepass android -keypass android \ + -alias androiddebugkey -keyalg RSA -validity 10000 -dname "CN=Android Debug,O=Android,C=US" + fi + test-farm/mock-location/build.sh + # The unit tests and the instrumentation tests are independent of each other, # so they run as separate parallel jobs rather than one long serial job. Both # need the offline map fixtures (the unit tests read them out of the test diff --git a/test-farm/README.md b/test-farm/README.md index e3ccd36cd..644a06497 100644 --- a/test-farm/README.md +++ b/test-farm/README.md @@ -51,6 +51,15 @@ cp farm.toml.example farm.toml # optional python3 test-farm/farm.py # Ctrl-C to stop; removes the mock GPS providers ``` +`test_farm.py` checks the parts that need no phone: the config, the flows' +paths into `maestro/`, the settings shared with `run-maestro-tests.yaml` +(suite location, launch-failure retries) and the parsers of adb output. +`run-tests.yaml` runs it on every PR and builds the mock location helper. + +```bash +python3 -m unittest discover -s test-farm -v +``` + ## How release builds are made testable - **Analytics:** the farm sets `settings put system firebase.test.lab true`, which diff --git a/test-farm/test_farm.py b/test-farm/test_farm.py new file mode 100644 index 000000000..1d8f53c42 --- /dev/null +++ b/test-farm/test_farm.py @@ -0,0 +1,216 @@ +"""Checks farm.py without any phones: run-tests runs these on every PR. + + python3 -m unittest discover -s test-farm -v + +They cover what can break without a device: the config, the wiring between +the farm's flows and maestro/, the settings the farm shares with CI's Maestro +run, and the parsers that read adb output. +""" + +import glob +import os +import re +import tempfile +import tomllib +import unittest + +import farm + +MAESTRO_WORKFLOW = os.path.join(farm.REPO_DIR, ".github", "workflows", "run-maestro-tests.yaml") +FLOW_REF_RE = re.compile(r"^\s*(?:-\s*runFlow:|file:)\s*(\S+\.yaml)\s*$", re.M) + + +class FakeShell: + """Stands in for farm.shell: answers each command from `replies`, keyed by a + substring of the command, and records the commands it was given.""" + + def __init__(self, replies): + self.replies = replies + self.commands = [] + + def __call__(self, serial, command, timeout=120): + self.commands.append(command) + for key, reply in self.replies.items(): + if key in command: + return reply + return "" + + +class ShellTestCase(unittest.TestCase): + def fake_shell(self, replies): + fake = FakeShell(replies) + original = farm.shell + farm.shell = fake + self.addCleanup(setattr, farm, "shell", original) + return fake + + +class ConfigTest(unittest.TestCase): + EXAMPLE = os.path.join(farm.FARM_DIR, "farm.toml.example") + + def test_example_loads(self): + cfg = farm.load_config(self.EXAMPLE) + self.assertEqual(cfg["extract_name"], "Glasgow") + + def test_example_only_uses_known_settings(self): + with open(self.EXAMPLE, "rb") as f: + keys = set(tomllib.load(f)) + self.assertEqual(keys - set(farm.DEFAULTS), set()) + + def test_gpx_paths_are_relative_to_the_config(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "farm.toml") + with open(path, "w") as f: + f.write('default_gpx = "a.gpx"\n[devices.X]\ngpx = "../b.gpx"\n') + cfg = farm.load_config(path) + self.assertEqual(cfg["default_gpx"], os.path.join(d, "a.gpx")) + self.assertEqual(cfg["devices"]["X"]["gpx"], os.path.join(os.path.dirname(d), "b.gpx")) + + +class FlowWiringTest(unittest.TestCase): + def test_suite_flows_exist(self): + flows = farm.suite_flows() + self.assertTrue(flows, "no runFlow lines found in maestro/Master.yaml") + for flow in flows: + self.assertTrue(os.path.isfile(os.path.join(farm.MAESTRO_DIR, flow)), flow) + + def test_roam_flows_exist(self): + self.assertTrue(farm.roam_flows()) + + def test_farm_flows_exist(self): + for path in (farm.POST_UPGRADE_FLOW, farm.FARM_SETTINGS_FLOW, farm.DOWNLOAD_EXTRACT_FLOW, + farm.SHOW_MAP_FLOW, farm.RESTORE_FLOW): + self.assertTrue(os.path.isfile(path), path) + + def test_flow_references_resolve(self): + # Roam flows sit a level deeper than the others, so their paths into + # maestro/ need one more "../". + flows = glob.glob(os.path.join(farm.FARM_DIR, "flows", "**", "*.yaml"), recursive=True) + self.assertTrue(flows) + for flow in flows: + with open(flow) as f: + refs = FLOW_REF_RE.findall(f.read()) + for ref in refs: + target = os.path.normpath(os.path.join(os.path.dirname(flow), ref)) + with self.subTest(flow=os.path.relpath(flow, farm.FARM_DIR), ref=ref): + self.assertTrue(os.path.isfile(target)) + + +class CiDriftTest(unittest.TestCase): + """The farm repeats two things from CI's Maestro run, which must agree.""" + + @classmethod + def setUpClass(cls): + with open(MAESTRO_WORKFLOW) as f: + cls.workflow = f.read() + + def test_suite_location_matches_ci_geo_fix(self): + # The suite's flows assert distance-sorted lists, so the farm has to + # hold the phone at the point CI pins the emulator to. + m = re.search(r"adb emu geo fix (-?[\d.]+) (-?[\d.]+)", self.workflow) + self.assertIsNotNone(m) + lon, lat = float(m.group(1)), float(m.group(2)) + self.assertEqual([lat, lon], farm.DEFAULTS["suite_location"]) + + def test_retry_regex_covers_ci_retries(self): + m = re.search(r'grep -Eq "([^"]+)" "maestro_outputs', self.workflow) + self.assertIsNotNone(m) + for message in m.group(1).split("|"): + with self.subTest(message=message): + self.assertRegex(message, farm.MAESTRO_RETRY_RE) + + +class DropboxTest(ShellTestCase): + @staticmethod + def entry(stamp, tag, body): + return f"========================================\n{stamp} {tag} (text, 100 bytes)\n{body}\n" + + def test_collects_our_crashes_and_anrs_since(self): + pkg = farm.PACKAGE + self.fake_shell({ + "data_app_crash": self.entry("2026-10-01 09:00:00", "data_app_crash", f"Process: {pkg}") + + self.entry("2026-10-02 10:00:00", "data_app_crash", f"Process: {pkg}") + + self.entry("2026-10-02 11:00:00", "data_app_crash", "Process: com.other"), + "data_app_anr": self.entry("2026-10-02 12:00:00", "data_app_anr", f"Process: {pkg}"), + }) + found = farm.collect_dropbox("S", "2026-10-02 00:00:00") + self.assertEqual([kind for kind, _ in found], ["crash", "anr"]) + self.assertIn("2026-10-02 10:00:00", found[0][1]) + + def test_asks_for_one_tag_per_call(self): + # dumpsys dropbox only prints entries matching every argument, so + # several tags in one call print nothing. + fake = self.fake_shell({}) + farm.collect_dropbox("S", "2026-10-02 00:00:00") + self.assertEqual(fake.commands, [f"dumpsys dropbox --print {tag}" for tag in farm.DROPBOX_TAGS]) + + +class DeviceReadingsTest(ShellTestCase): + def test_app_uptime(self): + for etime, seconds in (("03:04", 184), ("02:03:04", 7384), ("1-02:03:04", 93784)): + with self.subTest(etime=etime): + self.fake_shell({"pidof": "1234\n", "ps -o": f" {etime}\n"}) + self.assertEqual(farm.app_uptime_s("S"), seconds) + + def test_app_uptime_not_running(self): + self.fake_shell({"pidof": ""}) + self.assertIsNone(farm.app_uptime_s("S")) + + def test_pss(self): + self.fake_shell({"meminfo": " TOTAL PSS: 123456 TOTAL RSS: 200000"}) + self.assertEqual(farm.pss_kb("S"), 123456) + self.fake_shell({"meminfo": " Native Heap 1000\n TOTAL 98765 1234"}) + self.assertEqual(farm.pss_kb("S"), 98765) + + def test_battery(self): + self.fake_shell({"battery": " level: 80\n temperature: 315\n"}) + self.assertEqual(farm.battery("S"), (80, 31.5)) + self.fake_shell({"battery": ""}) + self.assertEqual(farm.battery("S"), (None, None)) + + +class TrackTest(unittest.TestCase): + START = (55.9474, -4.3058) + + def write_gpx(self, points): + f = tempfile.NamedTemporaryFile("w", suffix=".gpx", delete=False) + self.addCleanup(os.unlink, f.name) + pts = "".join(f'' for lat, lon in points) + f.write('' + f"{pts}") + f.close() + return f.name + + def north(self, metres): + return self.START[0] + metres / 111_195, self.START[1] + + def test_jitter_is_dropped(self): + jitter = [self.north(m) for m in (0, 3, 1, 5, 2)] + points = farm.load_track(self.write_gpx(jitter + [self.north(100), self.north(200)])) + self.assertEqual(len(points), 3) + + def test_too_short_track_is_rejected(self): + with self.assertRaises(ValueError): + farm.load_track(self.write_gpx([self.north(0), self.north(5)])) + + def test_walks_to_the_end_and_back(self): + track = farm.Track(self.write_gpx([self.north(0), self.north(100), self.north(200)]), + None, speed=2.0) + total = track.cumulative[-1] + self.assertAlmostEqual(total, 200, delta=0.5) + self.assertEqual(track.position(0), self.START) + end = track.position(total / 2.0) + self.assertAlmostEqual(end[0], self.north(200)[0], places=6) + # A quarter of the way back is three quarters of the way along. + back = track.position(total * 1.25 / 2.0) + self.assertAlmostEqual(back[0], self.north(150)[0], places=5) + self.assertAlmostEqual(track.position(total / 1.0)[0], self.START[0], places=6) + + def test_fixed_point(self): + track = farm.Track("", self.START, speed=1.4) + for t in (0, 10, 1000): + self.assertEqual(track.position(t), self.START) + + +if __name__ == "__main__": + unittest.main()
WhenDeviceKind