New test-farm for longer term app testing

.github/workflows/run-tests.yaml

@@ -50,6 +50,36 @@ jobs:
       - name: Run lint over the repo
         run: ./gradlew lint
 
+  # The test farm (test-farm/) runs on phones at home, not here, so this only
+  # checks what can break without a device: farm.py compiles, its flows' paths
+  # into maestro/ resolve, the settings it shares with run-maestro-tests.yaml
+  # still agree, and its adb-output parsers work (test-farm/test_farm.py). It
+  # also builds the mock location helper. The runner image already has the
+  # Python (3.11+ for tomllib), JDK and Android SDK these need.
+  test-farm:
+    name: Check test farm
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Compile farm scripts
+        run: python3 -m py_compile test-farm/farm.py test-farm/usb_proxy.py
+
+      - name: Unit-test farm.py
+        run: python3 -m unittest discover -s test-farm -v
+
+      # build.sh signs with ~/.android/debug.keystore, which a fresh runner lacks.
+      - name: Build mock location helper
+        run: |
+          if [ ! -f ~/.android/debug.keystore ]; then
+            mkdir -p ~/.android
+            keytool -genkeypair -keystore ~/.android/debug.keystore -storepass android -keypass android \
+              -alias androiddebugkey -keyalg RSA -validity 10000 -dname "CN=Android Debug,O=Android,C=US"
+          fi
+          test-farm/mock-location/build.sh
+
   # The unit tests and the instrumentation tests are independent of each other,
   # so they run as separate parallel jobs rather than one long serial job. Both
   # need the offline map fixtures (the unit tests read them out of the test

docs/developers/developers.md

@@ -66,6 +66,7 @@ If you would like access to the main Soundscape tile provider for development, g
 * [Branch strategy]({% link developers/branch-strategy.md %}) — which branches are active, where to send PRs, and the upcoming KMP cutover that will replace `main`.
 * [Build types and analytics]({% link developers/build-types.md %}) — `debug` / `release` / `releaseTest`, the variant-specific source sets that swap in real or no-op analytics, and the `BuildConfig` values read from `local.properties`.
 * [GitHub actions]({% link developers/actions.md %}) — the CI workflows and the secrets they consume.
+* [Test farm]({% link developers/test-farm.md %}) — long-term testing of release builds on physical phones: release upgrades, offline maps, a replayed GPX walk, and crash collection, with iPhones observed rather than driven.
 * [Unit test example]({% link developers/unit-test-example.md %}) — using host-side unit tests with real map data to prototype and debug geo features.
 
 ## Reference

docs/developers/maestro-tests.md

@@ -103,8 +103,8 @@ flows, so they must run in this order:
 | 2 | `HomePage.yaml` | Exercises the home screen controls (My Location, Around Me, Ahead of Me, menu, sleep). |
 | 3 | `LocationDetails.yaml` | Creates markers **A** and **B** that later flows rely on. |
 | 4 | `PlacesNearby.yaml` | Scrolls and opens entries in the Places Nearby list. |
-| 5 | `Markers.yaml` | Sorting, editing marker **A**, and creating marker **C** with the New button. |
-| 6 | `RouteCreation.yaml` | Builds a route from markers **A** and **B**. |
+| 5 | `Markers.yaml` | Sorting, editing marker **A**, and creating marker **C** at the current location with the New button. A was saved there too, and markers at the same coordinates merge, so C replaces A. |
+| 6 | `RouteCreation.yaml` | Builds a route from markers **B** and **C**. |
 | 7 | `FullScreenMap.yaml` | Toggles the full-screen map on the home screen. |
 
 `Onboarding.yaml` is the only flow that uses `launchApp: clearState: true`;
@@ -426,3 +426,22 @@ always run; a final step re-raises the failure to mark the workflow red.
 
 When you add a new flow, add its name (without the `.yaml` extension) to the
 `for flow in …` list in the correct position so CI runs it.
+
+## On physical phones: the test farm
+
+The same flows also run on physical phones against **release builds** in the
+[test farm]({% link developers/test-farm.md %}), once for each release on each
+phone, to set it up. The farm runs them in `maestro/Master.yaml` order, holding
+the phone at the same Edinburgh point CI uses, with `FarmSettings.yaml` from
+`test-farm/flows/` after Onboarding. After that the farm runs its own roaming
+flows from `test-farm/flows/roam/` while the phone walks a GPX track, and
+`PostUpgrade.yaml` after each release upgrade. Things that matter when
+changing flows:
+
+- A flow that only passes on an emulator (for example, by relying on `adb root`
+  or a debug-only screen) will fail there.
+- Changing what the suite leaves behind (markers **B** and **C** and
+  **Test Route**) needs a matching change to `test-farm/flows/PostUpgrade.yaml`,
+  which checks they survive an upgrade, and `test-farm/flows/Restore.yaml`,
+  which recreates them after the farm's monkey soak. The roaming flows use the markers and
+  the route too.

docs/developers/test-farm.md

@@ -0,0 +1,92 @@
+---
+title: Test farm
+layout: page
+parent: Information for developers
+has_toc: false
+---
+
+# Test farm
+
+The test farm runs long-term tests of **release builds** on physical phones
+plugged into a developer's PC. It complements CI: the
+[Maestro UI tests]({% link developers/maestro-tests.md %}) in CI run once per
+change on emulators, while the farm keeps real phones exercising the app for
+days at a time and upgrades them as new releases are built.
+
+The farm is a single Python script, `test-farm/farm.py`. The setup steps and
+every config option are in
+[`test-farm/README.md`](https://github.com/Scottish-Tech-Army/Soundscape-Android/blob/main/test-farm/README.md).
+
+## What it does
+
+Each Android phone loops through a **cycle**:
+
+1. If a newer release has been built, it installs it over the existing app and
+   data (`adb install -r`), then runs `test-farm/flows/PostUpgrade.yaml` to
+   check that the app starts and the suite's markers and route survived the
+   upgrade.
+2. Once for each release on each phone, it sets the phone up with the Maestro
+   suite, in `maestro/Master.yaml` order. After `Onboarding.yaml` it pushes an
+   offline map and runs `test-farm/flows/FarmSettings.yaml`, which sets search
+   to Offline. The suite runs with the map off, as in CI; `ShowMap.yaml` then
+   turns it on for roaming.
+3. The roaming flows in `test-farm/flows/roam/`, shuffled, while the phone
+   walks a GPX track: callouts, beacons on nearby places and search results,
+   playing a route, the markers, the menus' screens, sleep and wake. They
+   look around without asserting what's nearby, because that changes as the
+   phone walks. They bring the app to the front rather than restarting it,
+   so one process runs for hours across rounds and cycles.
+4. A `monkey` soak: random input inside the app, logged with its seed so a
+   failure can be reproduced. Then `Restore.yaml` puts back the search mode,
+   the map setting and any markers or route it deleted, without restarting
+   the app. The phone is only set up again if that fails.
+5. It collects crashes and ANRs (from Android's dropbox), logcat, memory (PSS),
+   battery and the app process's uptime.
+
+Results go into `test-farm/farm.db`, with a summary page at
+`test-farm/report.html` that refreshes every minute.
+
+iPhones are **observed, not driven**. Maestro can't control a physical iPhone,
+because its iOS driver needs Xcode on a Mac. Instead the farm uses
+[pymobiledevice3](https://github.com/doronz88/pymobiledevice3) to:
+- replay a GPX walk;
+- record TestFlight upgrades, and app restarts when the app wasn't running;
+- collect Soundscape crash and hang reports, battery level and a Soundscape
+  syslog.
+
+## How it differs from CI
+
+| | CI | Test farm |
+|---|---|---|
+| Devices | Emulators | Physical phones, including an AOSP phone without Play services |
+| Build | Debug | The release APK from the `Tag and build release` workflow |
+| Upgrades | Not tested | Each release is installed over the previous one, keeping the data |
+| Map data | Glasgow extract pushed as root | The current extract from the manifest, pushed without root, with search set to Offline and Wi-Fi turned off |
+| Location | `adb emu geo fix` | A replayed GPX walk with speed and bearing, held at CI's Edinburgh point while the suite runs |
+| Map | Off (Onboarding turns it off) | On |
+| Duration | One run | Continuous |
+
+## Making release builds testable
+
+The farm doesn't use any debug-only hooks:
+
+- **Analytics:** setting `firebase.test.lab` makes `MainActivity` use dummy
+  analytics, as it does in Firebase Test Lab.
+- **Location:** a small mock location helper app (`test-farm/mock-location/`,
+  not part of Soundscape) sets every fix, including speed and bearing.
+  Soundscape only takes its direction from travel when a fix says the user is
+  moving. Without speed and bearing it uses the phone's compass, and on a desk
+  that points nowhere useful.
+- **Maestro:** `testTagsAsResourceId` is set for every build type, so the same
+  flows work on release builds.
+
+## Things that will catch you out
+
+- Phones need **Screen lock: None**. Maestro can't get past a PIN, and the farm
+  reports a locked phone instead of testing it.
+- Phones must be in **English** for now. Releases before the Settings and
+  search result testTags are driven by English labels.
+- Remove any Play Store or debug copy of Soundscape first. The farm only tests a
+  build it installed itself, and reports anything else as blocked.
+- Keep phones off USB hubs that lose disconnects. One here kept a ghost iPhone
+  connection that hung `usbmuxd`.

maestro/HomePage.yaml

@@ -36,8 +36,11 @@ appId: org.scottishtecharmy.soundscape
     id: sleepWakeUpNow
 - tapOn:
     id: topBarMenu
-- tapOn:
-    id: menuAboutSoundscape
+# Below the bottom of the drawer on small screens (480x854).
+- runFlow:
+    file: SwipeAndTap.yaml
+    env:
+      id: menuAboutSoundscape
 - tapOn:
     id: appBarLeft
 - tapOn:

maestro/Markers.yaml

@@ -1,6 +1,7 @@
 appId: org.scottishtecharmy.soundscape
 # Exercises the Markers screen: the sort controls, editing an existing marker,
-# and creating marker "C" at the current location with the New button.
+# and creating marker "C" at the current location with the New button, which
+# replaces "A" (saved at the same location).
 #
 # Depends on markers "A" and "B" having been created by LocationDetails.yaml, so
 # this flow must run after it in the suite.
@@ -63,6 +64,10 @@ appId: org.scottishtecharmy.soundscape
 - tapOn:
     id: flexibleAppBarRight
 
-# Saving returns to the Markers screen, which now lists C.
+# Saving returns to the Markers screen, which now lists C. A was also saved at
+# the current location (LocationDetails.yaml), and saveMarker merges a new
+# marker into an existing one at exactly the same coordinates, so C replaces A.
 - assertVisible:
     id: "LocationItem-C-.*"
+- assertNotVisible:
+    id: "LocationItem-A-.*"

maestro/Onboarding.yaml

@@ -31,44 +31,67 @@ appId: org.scottishtecharmy.soundscape
     env:
       id: "languageScreenContinueButton"
 
+# The Navigating (permissions) and Battery screens skip themselves when there is
+# nothing to grant. launchApp's permissions block above always grants the
+# Navigating screen's permissions, and the battery exemption can survive
+# clearState, so either screen may never show. The fixed wait gives a skipping
+# screen time to move on before checking whether it is still there.
 - runFlow:
     when:
       platform: Android
-    file: SwipeAndTap.yaml
+    file: Wait.yaml
     env:
-      id: "navigatingScreenContinueButton"
+      timeout: 2000
 
-# Battery optimization screen. Tapping "Grant permission" launches a system
-# dialog (ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS) which is not a runtime
-# permission, so it is not handled by the launchApp permissions block. We wait
-# for the system dialog's positive button and tap it, then continue.
 - runFlow:
     when:
-      platform: Android
+      visible:
+        id: "navigatingScreenContinueButton"
     file: SwipeAndTap.yaml
     env:
-      id: "batteryOptimizationGrantPermissionButton"
+      id: "navigatingScreenContinueButton"
 
 - runFlow:
     when:
       platform: Android
     file: Wait.yaml
     env:
-      timeout: 3000
-
-# The system battery-optimisation dialog only appears if the app isn't already
-# exempt. Dismiss its "Allow" button if it shows; if it never appears, optional
-# lets the flow carry on instead of failing.
-- tapOn:
-    id: "android:id/button1"
-    optional: true
+      timeout: 2000
 
+# Battery optimization screen. Tapping "Grant permission" launches a system
+# dialog (ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS) which is not a runtime
+# permission, so it is not handled by the launchApp permissions block. We wait
+# for the system dialog's positive button and tap it, then continue.
 - runFlow:
     when:
-      platform: Android
-    file: SwipeAndTap.yaml
-    env:
-      id: "batteryOptimizationContinueButton"
+      visible:
+        id: "batteryOptimizationGrantPermissionButton"
+    commands:
+      - runFlow:
+          file: SwipeAndTap.yaml
+          env:
+            id: "batteryOptimizationGrantPermissionButton"
+      - runFlow:
+          file: Wait.yaml
+          env:
+            timeout: 3000
+      # The system dialog only appears if the app isn't already exempt.
+      - tapOn:
+          id: "android:id/button1"
+          optional: true
+      - runFlow:
+          file: Wait.yaml
+          env:
+            timeout: 2000
+      # Granting the exemption moves on to the next screen by itself; Continue
+      # is only left to tap if it was refused.
+      - runFlow:
+          when:
+            visible:
+              id: "batteryOptimizationContinueButton"
+          file: SwipeAndTap.yaml
+          env:
+            id: "batteryOptimizationContinueButton"
 
 - runFlow:
     file: SwipeAndTap.yaml

maestro/RouteCreation.yaml

@@ -2,8 +2,8 @@ appId: org.scottishtecharmy.soundscape
 # Creates a new route from the Routes screen, adds two existing markers as
 # waypoints, saves it, and verifies it appears in the routes list.
 #
-# Depends on markers "A" and "B" having been created by LocationDetails.yaml, so
-# this flow must run after it in the suite.
+# Depends on marker "B" from LocationDetails.yaml and marker "C" from
+# Markers.yaml (which replaces "A"), so this flow must run after both.
 ---
 - launchApp:
     clearState: false
@@ -28,14 +28,14 @@ appId: org.scottishtecharmy.soundscape
 - inputText: "Test Route"
 - pressKey: Enter
 
-# Add the existing markers A and B as waypoints. They are listed by name; the
+# Add the existing markers B and C as waypoints. They are listed by name; the
 # orderId suffix is unknown here so match by name prefix.
 - tapOn:
     id: addWaypointsButton
-- tapOn:
-    id: "LocationItem-A-.*"
 - tapOn:
     id: "LocationItem-B-.*"
+- tapOn:
+    id: "LocationItem-C-.*"
 # "Done" in the waypoints dialog (reuses the shared app-bar right button).
 - tapOn:
     id: flexibleAppBarRight

maestro/SwipeAndTap.yaml

@@ -22,6 +22,12 @@ appId: org.scottishtecharmy.soundscape
 # reverse the nudge to put it back - a partially-clipped target stays visible and
 # is left as nudged.
 #
+# Swipes that move content up start no lower than 60% of the screen height: on
+# a small screen (480x854) the home screen's fixed bottom bar starts at about
+# 68%, and a swipe starting on it scrolls nothing. Each step moves 30%, less
+# than the scrollable area between the top and bottom bars there (about 43%),
+# so a target can't jump from below the fold to above the top in one step.
+#
 # Note: swipe start/end coordinates can't be supplied via ${} variables (Maestro
 # parses them as literals and throws a number-format error), so the per-direction
 # coordinates are hard-coded below and selected with `when` conditions.
@@ -60,8 +66,8 @@ appId: org.scottishtecharmy.soundscape
             true: "${output.swipeDir == 'up'}"
           commands:
             - swipe:
-                start: 50%, 80%
-                end: 50%, 50%
+                start: 50%, 60%
+                end: 50%, 30%
       # 'down' reveals items above (swipe the finger down).
       - runFlow:
           when:
@@ -85,8 +91,8 @@ appId: org.scottishtecharmy.soundscape
       true: "${output.swipeFound == 1 && output.swipeDir == 'up'}"
     commands:
       - swipe:
-          start: 50%, 75%
-          end: 50%, 60%
+          start: 50%, 55%
+          end: 50%, 40%
 - runFlow:
     when:
       true: "${output.swipeFound == 1 && output.swipeDir == 'down'}"
@@ -106,17 +112,17 @@ appId: org.scottishtecharmy.soundscape
         id: ${id}
     commands:
       - swipe:
-          start: 50%, 60%
-          end: 50%, 75%
+          start: 50%, 40%
+          end: 50%, 55%
 - runFlow:
     when:
       true: "${output.swipeFound == 1 && output.swipeDir == 'down'}"
       notVisible:
         id: ${id}
     commands:
       - swipe:
-          start: 50%, 75%
-          end: 50%, 60%
+          start: 50%, 55%
+          end: 50%, 40%
 
 - tapOn:
     id: ${id}

shared/src/androidMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.android.kt

@@ -0,0 +1,8 @@
+package org.scottishtecharmy.soundscape.platform
+
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.semantics.semantics
+import androidx.compose.ui.semantics.testTagsAsResourceId as testTagsAsResourceIdProperty
+
+actual fun Modifier.testTagsAsResourceId(): Modifier =
+    semantics { testTagsAsResourceIdProperty = true }