diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 595d12cd2..bdd9b9502 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -50,6 +50,36 @@ jobs: - name: Run lint over the repo run: ./gradlew lint + # The test farm (test-farm/) runs on phones at home, not here, so this only + # checks what can break without a device: farm.py compiles, its flows' paths + # into maestro/ resolve, the settings it shares with run-maestro-tests.yaml + # still agree, and its adb-output parsers work (test-farm/test_farm.py). It + # also builds the mock location helper. The runner image already has the + # Python (3.11+ for tomllib), JDK and Android SDK these need. + test-farm: + name: Check test farm + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + + - name: Compile farm scripts + run: python3 -m py_compile test-farm/farm.py test-farm/usb_proxy.py + + - name: Unit-test farm.py + run: python3 -m unittest discover -s test-farm -v + + # build.sh signs with ~/.android/debug.keystore, which a fresh runner lacks. + - name: Build mock location helper + run: | + if [ ! -f ~/.android/debug.keystore ]; then + mkdir -p ~/.android + keytool -genkeypair -keystore ~/.android/debug.keystore -storepass android -keypass android \ + -alias androiddebugkey -keyalg RSA -validity 10000 -dname "CN=Android Debug,O=Android,C=US" + fi + test-farm/mock-location/build.sh + # The unit tests and the instrumentation tests are independent of each other, # so they run as separate parallel jobs rather than one long serial job. Both # need the offline map fixtures (the unit tests read them out of the test diff --git a/docs/developers/developers.md b/docs/developers/developers.md index 63bdc2ce3..d74173402 100644 --- a/docs/developers/developers.md +++ b/docs/developers/developers.md @@ -66,6 +66,7 @@ If you would like access to the main Soundscape tile provider for development, g * [Branch strategy]({% link developers/branch-strategy.md %}) — which branches are active, where to send PRs, and the upcoming KMP cutover that will replace `main`. * [Build types and analytics]({% link developers/build-types.md %}) — `debug` / `release` / `releaseTest`, the variant-specific source sets that swap in real or no-op analytics, and the `BuildConfig` values read from `local.properties`. * [GitHub actions]({% link developers/actions.md %}) — the CI workflows and the secrets they consume. +* [Test farm]({% link developers/test-farm.md %}) — long-term testing of release builds on physical phones: release upgrades, offline maps, a replayed GPX walk, and crash collection, with iPhones observed rather than driven. * [Unit test example]({% link developers/unit-test-example.md %}) — using host-side unit tests with real map data to prototype and debug geo features. ## Reference diff --git a/docs/developers/maestro-tests.md b/docs/developers/maestro-tests.md index 70b08a2f9..6beacac7e 100644 --- a/docs/developers/maestro-tests.md +++ b/docs/developers/maestro-tests.md @@ -103,8 +103,8 @@ flows, so they must run in this order: | 2 | `HomePage.yaml` | Exercises the home screen controls (My Location, Around Me, Ahead of Me, menu, sleep). | | 3 | `LocationDetails.yaml` | Creates markers **A** and **B** that later flows rely on. | | 4 | `PlacesNearby.yaml` | Scrolls and opens entries in the Places Nearby list. | -| 5 | `Markers.yaml` | Sorting, editing marker **A**, and creating marker **C** with the New button. | -| 6 | `RouteCreation.yaml` | Builds a route from markers **A** and **B**. | +| 5 | `Markers.yaml` | Sorting, editing marker **A**, and creating marker **C** at the current location with the New button. A was saved there too, and markers at the same coordinates merge, so C replaces A. | +| 6 | `RouteCreation.yaml` | Builds a route from markers **B** and **C**. | | 7 | `FullScreenMap.yaml` | Toggles the full-screen map on the home screen. | `Onboarding.yaml` is the only flow that uses `launchApp: clearState: true`; @@ -426,3 +426,22 @@ always run; a final step re-raises the failure to mark the workflow red. When you add a new flow, add its name (without the `.yaml` extension) to the `for flow in …` list in the correct position so CI runs it. + +## On physical phones: the test farm + +The same flows also run on physical phones against **release builds** in the +[test farm]({% link developers/test-farm.md %}), once for each release on each +phone, to set it up. The farm runs them in `maestro/Master.yaml` order, holding +the phone at the same Edinburgh point CI uses, with `FarmSettings.yaml` from +`test-farm/flows/` after Onboarding. After that the farm runs its own roaming +flows from `test-farm/flows/roam/` while the phone walks a GPX track, and +`PostUpgrade.yaml` after each release upgrade. Things that matter when +changing flows: + +- A flow that only passes on an emulator (for example, by relying on `adb root` + or a debug-only screen) will fail there. +- Changing what the suite leaves behind (markers **B** and **C** and + **Test Route**) needs a matching change to `test-farm/flows/PostUpgrade.yaml`, + which checks they survive an upgrade, and `test-farm/flows/Restore.yaml`, + which recreates them after the farm's monkey soak. The roaming flows use the markers and + the route too. diff --git a/docs/developers/test-farm.md b/docs/developers/test-farm.md new file mode 100644 index 000000000..6ce6accf5 --- /dev/null +++ b/docs/developers/test-farm.md @@ -0,0 +1,92 @@ +--- +title: Test farm +layout: page +parent: Information for developers +has_toc: false +--- + +# Test farm + +The test farm runs long-term tests of **release builds** on physical phones +plugged into a developer's PC. It complements CI: the +[Maestro UI tests]({% link developers/maestro-tests.md %}) in CI run once per +change on emulators, while the farm keeps real phones exercising the app for +days at a time and upgrades them as new releases are built. + +The farm is a single Python script, `test-farm/farm.py`. The setup steps and +every config option are in +[`test-farm/README.md`](https://github.com/Scottish-Tech-Army/Soundscape-Android/blob/main/test-farm/README.md). + +## What it does + +Each Android phone loops through a **cycle**: + +1. If a newer release has been built, it installs it over the existing app and + data (`adb install -r`), then runs `test-farm/flows/PostUpgrade.yaml` to + check that the app starts and the suite's markers and route survived the + upgrade. +2. Once for each release on each phone, it sets the phone up with the Maestro + suite, in `maestro/Master.yaml` order. After `Onboarding.yaml` it pushes an + offline map and runs `test-farm/flows/FarmSettings.yaml`, which sets search + to Offline. The suite runs with the map off, as in CI; `ShowMap.yaml` then + turns it on for roaming. +3. The roaming flows in `test-farm/flows/roam/`, shuffled, while the phone + walks a GPX track: callouts, beacons on nearby places and search results, + playing a route, the markers, the menus' screens, sleep and wake. They + look around without asserting what's nearby, because that changes as the + phone walks. They bring the app to the front rather than restarting it, + so one process runs for hours across rounds and cycles. +4. A `monkey` soak: random input inside the app, logged with its seed so a + failure can be reproduced. Then `Restore.yaml` puts back the search mode, + the map setting and any markers or route it deleted, without restarting + the app. The phone is only set up again if that fails. +5. It collects crashes and ANRs (from Android's dropbox), logcat, memory (PSS), + battery and the app process's uptime. + +Results go into `test-farm/farm.db`, with a summary page at +`test-farm/report.html` that refreshes every minute. + +iPhones are **observed, not driven**. Maestro can't control a physical iPhone, +because its iOS driver needs Xcode on a Mac. Instead the farm uses +[pymobiledevice3](https://github.com/doronz88/pymobiledevice3) to: +- replay a GPX walk; +- record TestFlight upgrades, and app restarts when the app wasn't running; +- collect Soundscape crash and hang reports, battery level and a Soundscape + syslog. + +## How it differs from CI + +| | CI | Test farm | +|---|---|---| +| Devices | Emulators | Physical phones, including an AOSP phone without Play services | +| Build | Debug | The release APK from the `Tag and build release` workflow | +| Upgrades | Not tested | Each release is installed over the previous one, keeping the data | +| Map data | Glasgow extract pushed as root | The current extract from the manifest, pushed without root, with search set to Offline and Wi-Fi turned off | +| Location | `adb emu geo fix` | A replayed GPX walk with speed and bearing, held at CI's Edinburgh point while the suite runs | +| Map | Off (Onboarding turns it off) | On | +| Duration | One run | Continuous | + +## Making release builds testable + +The farm doesn't use any debug-only hooks: + +- **Analytics:** setting `firebase.test.lab` makes `MainActivity` use dummy + analytics, as it does in Firebase Test Lab. +- **Location:** a small mock location helper app (`test-farm/mock-location/`, + not part of Soundscape) sets every fix, including speed and bearing. + Soundscape only takes its direction from travel when a fix says the user is + moving. Without speed and bearing it uses the phone's compass, and on a desk + that points nowhere useful. +- **Maestro:** `testTagsAsResourceId` is set for every build type, so the same + flows work on release builds. + +## Things that will catch you out + +- Phones need **Screen lock: None**. Maestro can't get past a PIN, and the farm + reports a locked phone instead of testing it. +- Phones must be in **English** for now. Releases before the Settings and + search result testTags are driven by English labels. +- Remove any Play Store or debug copy of Soundscape first. The farm only tests a + build it installed itself, and reports anything else as blocked. +- Keep phones off USB hubs that lose disconnects. One here kept a ghost iPhone + connection that hung `usbmuxd`. diff --git a/maestro/HomePage.yaml b/maestro/HomePage.yaml index 2f34718be..6a82d9043 100644 --- a/maestro/HomePage.yaml +++ b/maestro/HomePage.yaml @@ -36,8 +36,11 @@ appId: org.scottishtecharmy.soundscape id: sleepWakeUpNow - tapOn: id: topBarMenu -- tapOn: - id: menuAboutSoundscape +# Below the bottom of the drawer on small screens (480x854). +- runFlow: + file: SwipeAndTap.yaml + env: + id: menuAboutSoundscape - tapOn: id: appBarLeft - tapOn: diff --git a/maestro/Markers.yaml b/maestro/Markers.yaml index a0d6e6129..74acde74e 100644 --- a/maestro/Markers.yaml +++ b/maestro/Markers.yaml @@ -1,6 +1,7 @@ appId: org.scottishtecharmy.soundscape # Exercises the Markers screen: the sort controls, editing an existing marker, -# and creating marker "C" at the current location with the New button. +# and creating marker "C" at the current location with the New button, which +# replaces "A" (saved at the same location). # # Depends on markers "A" and "B" having been created by LocationDetails.yaml, so # this flow must run after it in the suite. @@ -63,6 +64,10 @@ appId: org.scottishtecharmy.soundscape - tapOn: id: flexibleAppBarRight -# Saving returns to the Markers screen, which now lists C. +# Saving returns to the Markers screen, which now lists C. A was also saved at +# the current location (LocationDetails.yaml), and saveMarker merges a new +# marker into an existing one at exactly the same coordinates, so C replaces A. - assertVisible: id: "LocationItem-C-.*" +- assertNotVisible: + id: "LocationItem-A-.*" diff --git a/maestro/Onboarding.yaml b/maestro/Onboarding.yaml index 68ef7b6f8..d1d3afab0 100644 --- a/maestro/Onboarding.yaml +++ b/maestro/Onboarding.yaml @@ -31,44 +31,67 @@ appId: org.scottishtecharmy.soundscape env: id: "languageScreenContinueButton" +# The Navigating (permissions) and Battery screens skip themselves when there is +# nothing to grant. launchApp's permissions block above always grants the +# Navigating screen's permissions, and the battery exemption can survive +# clearState, so either screen may never show. The fixed wait gives a skipping +# screen time to move on before checking whether it is still there. - runFlow: when: platform: Android - file: SwipeAndTap.yaml + file: Wait.yaml env: - id: "navigatingScreenContinueButton" + timeout: 2000 -# Battery optimization screen. Tapping "Grant permission" launches a system -# dialog (ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS) which is not a runtime -# permission, so it is not handled by the launchApp permissions block. We wait -# for the system dialog's positive button and tap it, then continue. - runFlow: when: - platform: Android + visible: + id: "navigatingScreenContinueButton" file: SwipeAndTap.yaml env: - id: "batteryOptimizationGrantPermissionButton" + id: "navigatingScreenContinueButton" - runFlow: when: platform: Android file: Wait.yaml env: - timeout: 3000 - -# The system battery-optimisation dialog only appears if the app isn't already -# exempt. Dismiss its "Allow" button if it shows; if it never appears, optional -# lets the flow carry on instead of failing. -- tapOn: - id: "android:id/button1" - optional: true + timeout: 2000 +# Battery optimization screen. Tapping "Grant permission" launches a system +# dialog (ACTION_REQUEST_IGNORE_BATTERY_OPTIMIZATIONS) which is not a runtime +# permission, so it is not handled by the launchApp permissions block. We wait +# for the system dialog's positive button and tap it, then continue. - runFlow: when: - platform: Android - file: SwipeAndTap.yaml - env: - id: "batteryOptimizationContinueButton" + visible: + id: "batteryOptimizationGrantPermissionButton" + commands: + - runFlow: + file: SwipeAndTap.yaml + env: + id: "batteryOptimizationGrantPermissionButton" + - runFlow: + file: Wait.yaml + env: + timeout: 3000 + # The system dialog only appears if the app isn't already exempt. + - tapOn: + id: "android:id/button1" + optional: true + - runFlow: + file: Wait.yaml + env: + timeout: 2000 + # Granting the exemption moves on to the next screen by itself; Continue + # is only left to tap if it was refused. + - runFlow: + when: + visible: + id: "batteryOptimizationContinueButton" + file: SwipeAndTap.yaml + env: + id: "batteryOptimizationContinueButton" - runFlow: file: SwipeAndTap.yaml diff --git a/maestro/RouteCreation.yaml b/maestro/RouteCreation.yaml index 6091cf50f..f764037b6 100644 --- a/maestro/RouteCreation.yaml +++ b/maestro/RouteCreation.yaml @@ -2,8 +2,8 @@ appId: org.scottishtecharmy.soundscape # Creates a new route from the Routes screen, adds two existing markers as # waypoints, saves it, and verifies it appears in the routes list. # -# Depends on markers "A" and "B" having been created by LocationDetails.yaml, so -# this flow must run after it in the suite. +# Depends on marker "B" from LocationDetails.yaml and marker "C" from +# Markers.yaml (which replaces "A"), so this flow must run after both. --- - launchApp: clearState: false @@ -28,14 +28,14 @@ appId: org.scottishtecharmy.soundscape - inputText: "Test Route" - pressKey: Enter -# Add the existing markers A and B as waypoints. They are listed by name; the +# Add the existing markers B and C as waypoints. They are listed by name; the # orderId suffix is unknown here so match by name prefix. - tapOn: id: addWaypointsButton -- tapOn: - id: "LocationItem-A-.*" - tapOn: id: "LocationItem-B-.*" +- tapOn: + id: "LocationItem-C-.*" # "Done" in the waypoints dialog (reuses the shared app-bar right button). - tapOn: id: flexibleAppBarRight diff --git a/maestro/SwipeAndTap.yaml b/maestro/SwipeAndTap.yaml index 49ce0ab78..39c9b1b47 100644 --- a/maestro/SwipeAndTap.yaml +++ b/maestro/SwipeAndTap.yaml @@ -22,6 +22,12 @@ appId: org.scottishtecharmy.soundscape # reverse the nudge to put it back - a partially-clipped target stays visible and # is left as nudged. # +# Swipes that move content up start no lower than 60% of the screen height: on +# a small screen (480x854) the home screen's fixed bottom bar starts at about +# 68%, and a swipe starting on it scrolls nothing. Each step moves 30%, less +# than the scrollable area between the top and bottom bars there (about 43%), +# so a target can't jump from below the fold to above the top in one step. +# # Note: swipe start/end coordinates can't be supplied via ${} variables (Maestro # parses them as literals and throws a number-format error), so the per-direction # coordinates are hard-coded below and selected with `when` conditions. @@ -60,8 +66,8 @@ appId: org.scottishtecharmy.soundscape true: "${output.swipeDir == 'up'}" commands: - swipe: - start: 50%, 80% - end: 50%, 50% + start: 50%, 60% + end: 50%, 30% # 'down' reveals items above (swipe the finger down). - runFlow: when: @@ -85,8 +91,8 @@ appId: org.scottishtecharmy.soundscape true: "${output.swipeFound == 1 && output.swipeDir == 'up'}" commands: - swipe: - start: 50%, 75% - end: 50%, 60% + start: 50%, 55% + end: 50%, 40% - runFlow: when: true: "${output.swipeFound == 1 && output.swipeDir == 'down'}" @@ -106,8 +112,8 @@ appId: org.scottishtecharmy.soundscape id: ${id} commands: - swipe: - start: 50%, 60% - end: 50%, 75% + start: 50%, 40% + end: 50%, 55% - runFlow: when: true: "${output.swipeFound == 1 && output.swipeDir == 'down'}" @@ -115,8 +121,8 @@ appId: org.scottishtecharmy.soundscape id: ${id} commands: - swipe: - start: 50%, 75% - end: 50%, 60% + start: 50%, 55% + end: 50%, 40% - tapOn: id: ${id} diff --git a/shared/src/androidMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.android.kt b/shared/src/androidMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.android.kt new file mode 100644 index 000000000..61e20162d --- /dev/null +++ b/shared/src/androidMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.android.kt @@ -0,0 +1,8 @@ +package org.scottishtecharmy.soundscape.platform + +import androidx.compose.ui.Modifier +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.semantics.testTagsAsResourceId as testTagsAsResourceIdProperty + +actual fun Modifier.testTagsAsResourceId(): Modifier = + semantics { testTagsAsResourceIdProperty = true } diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/LocationItem.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/LocationItem.kt index b5ec2f117..3269e96de 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/LocationItem.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/LocationItem.kt @@ -125,13 +125,19 @@ fun LocationItem( decoration.index != -1 -> "${decoration.indexDescription} ${decoration.index + 1}. ${item.name}" - else -> item.description?.takeIf { it.startsWith(item.name) } - ?: listOfNotNull( - item.name, + else -> { + // A road or address description already starts with the name, so it + // replaces the name rather than repeating it. The type and distance + // are still read out. + val description = item.description ?: item.street + val namedDescription = description?.takeIf { it.startsWith(item.name) } + listOfNotNull( + namedDescription ?: item.name, item.typeDescription?.additionalText, - item.description ?: item.street, + description.takeIf { namedDescription == null }, distanceStringA11y ).joinToString(", ") + } } if (decoration.editRoute.enabled) { diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/MainSearchBar.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/MainSearchBar.kt index 4fbf2e8cb..5cfa16555 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/MainSearchBar.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/components/MainSearchBar.kt @@ -59,6 +59,7 @@ import org.scottishtecharmy.soundscape.resources.ui_back_button_title import org.scottishtecharmy.soundscape.platform.FullScreenDialogWindow import org.scottishtecharmy.soundscape.platform.fullScreenDialogProperties import org.scottishtecharmy.soundscape.platform.fullScreenDialogSystemBarsPadding +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.screens.home.data.LocationDescription import org.scottishtecharmy.soundscape.screens.talkbackDescription import org.scottishtecharmy.soundscape.screens.talkbackLive @@ -143,6 +144,7 @@ fun MainSearchBar( Box( modifier = Modifier .fillMaxSize() + .testTagsAsResourceId() .background(colors.background) ) { Column(modifier = Modifier.fillMaxSize().fullScreenDialogSystemBarsPadding()) { @@ -236,7 +238,7 @@ fun MainSearchBar( ) { itemsIndexed(results) { index, item -> Column( - modifier = Modifier.semantics { + modifier = Modifier.testTag("searchResult-$index").semantics { collectionItemInfo = CollectionItemInfo( rowSpan = 1, columnSpan = 1, rowIndex = index, columnIndex = 0, diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt new file mode 100644 index 000000000..e3494806e --- /dev/null +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt @@ -0,0 +1,11 @@ +package org.scottishtecharmy.soundscape.platform + +import androidx.compose.ui.Modifier + +/** + * Exposes the testTags in this subtree as Android resource-ids, so that UiAutomator-based tools + * such as Maestro can find them. The app theme sets this once at its root, but a Dialog is a + * separate window with its own composition root, so content shown in one (the full-screen search, + * a setting's list of choices) has to set it again. Does nothing on iOS. + */ +expect fun Modifier.testTagsAsResourceId(): Modifier diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/AudioTourInstructionDialog.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/AudioTourInstructionDialog.kt index 5fa57ecf0..4a917a44c 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/AudioTourInstructionDialog.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/AudioTourInstructionDialog.kt @@ -10,6 +10,7 @@ import androidx.compose.ui.semantics.paneTitle import androidx.compose.ui.semantics.semantics import org.jetbrains.compose.resources.stringResource import org.scottishtecharmy.soundscape.audio.AudioTourInstruction +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.resources.Res import org.scottishtecharmy.soundscape.resources.menu_audio_tutorial import org.scottishtecharmy.soundscape.resources.tour_continue_button @@ -24,7 +25,8 @@ fun AudioTourInstructionDialog( val dialogTitle = stringResource(Res.string.menu_audio_tutorial) AlertDialog( onDismissRequest = { /* Don't dismiss on outside click for accessibility */ }, - modifier = Modifier.semantics { paneTitle = dialogTitle }, + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId().semantics { paneTitle = dialogTitle }, text = { Text(text = instruction.text) }, diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedAdvancedMarkersAndRoutesSettingsScreen.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedAdvancedMarkersAndRoutesSettingsScreen.kt index e15231d0b..2196d0c89 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedAdvancedMarkersAndRoutesSettingsScreen.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedAdvancedMarkersAndRoutesSettingsScreen.kt @@ -27,6 +27,7 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.platform.testTag import androidx.compose.ui.text.font.FontWeight import org.jetbrains.compose.resources.stringResource +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.resources.Res import org.scottishtecharmy.soundscape.resources.advanced_markers_and_routes_clear_all_alert_message import org.scottishtecharmy.soundscape.resources.advanced_markers_and_routes_clear_all_button @@ -97,6 +98,8 @@ fun SharedAdvancedMarkersAndRoutesSettingsScreen( if (showConfirmationDialog.value) { AlertDialog( + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId(), onDismissRequest = { showConfirmationDialog.value = false }, title = { Text(stringResource(Res.string.settings_reset_dialog_title)) }, text = { Text(stringResource(Res.string.advanced_markers_and_routes_clear_all_alert_message)) }, diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedNewReleaseDialog.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedNewReleaseDialog.kt index d99f0682e..01a7ad91a 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedNewReleaseDialog.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/home/SharedNewReleaseDialog.kt @@ -15,6 +15,7 @@ import androidx.compose.ui.platform.LocalUriHandler import androidx.compose.ui.platform.testTag import org.jetbrains.compose.resources.stringResource import org.scottishtecharmy.soundscape.platform.appVersionMinorTrimmed +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.preferences.PreferenceKeys import org.scottishtecharmy.soundscape.preferences.PreferencesProvider import org.scottishtecharmy.soundscape.resources.Res @@ -45,7 +46,7 @@ fun SharedNewReleaseDialog( val uriHandler = LocalUriHandler.current AlertDialog( - modifier = Modifier.padding(innerPadding), + modifier = Modifier.testTagsAsResourceId().padding(innerPadding), title = { Text(text = stringResource(Res.string.new_version_info_text)) }, text = { LazyColumn(modifier = Modifier.padding(top = spacing.medium)) { diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/OpenInMapApp.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/OpenInMapApp.kt index 7e6de5975..777517ee2 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/OpenInMapApp.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/OpenInMapApp.kt @@ -36,6 +36,7 @@ import androidx.compose.ui.semantics.customActions import androidx.compose.ui.semantics.semantics import org.jetbrains.compose.resources.stringResource import org.scottishtecharmy.soundscape.platform.isIos +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.preferences.PreferenceDefaults import org.scottishtecharmy.soundscape.preferences.PreferenceKeys import org.scottishtecharmy.soundscape.preferences.PreferencesProvider @@ -189,6 +190,8 @@ private fun MapAppChooserDialog( ) { var rememberChoice by remember { mutableStateOf(initiallyRemember) } AlertDialog( + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId(), onDismissRequest = onDismiss, title = { Text(stringResource(Res.string.open_in_app_dialog_title)) }, text = { diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/SharedSaveAndEditMarkerScreen.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/SharedSaveAndEditMarkerScreen.kt index d865daa9b..bb1d4e18f 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/SharedSaveAndEditMarkerScreen.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/locationDetails/SharedSaveAndEditMarkerScreen.kt @@ -33,6 +33,7 @@ import androidx.compose.ui.platform.testTag import androidx.compose.ui.text.font.FontWeight import org.jetbrains.compose.resources.stringResource import org.scottishtecharmy.soundscape.geojsonparser.geojson.LngLatAlt +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.preferences.PreferenceDefaults import org.scottishtecharmy.soundscape.preferences.PreferenceKeys import org.scottishtecharmy.soundscape.preferences.PreferencesProvider @@ -105,6 +106,8 @@ fun SharedSaveAndEditMarkerScreen( if (showDeleteDialog && onDelete != null) { AlertDialog( + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId(), onDismissRequest = { showDeleteDialog = false }, title = { Text(stringResource(Res.string.settings_reset_dialog_title)) }, text = { Text(stringResource(Res.string.markers_action_delete_alert_message)) }, diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SettingsComponents.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SettingsComponents.kt index 0ff96e67e..fb428a25f 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SettingsComponents.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SettingsComponents.kt @@ -26,9 +26,11 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color import androidx.compose.ui.semantics.Role import androidx.compose.ui.semantics.heading +import androidx.compose.ui.platform.testTag import androidx.compose.ui.semantics.semantics import org.jetbrains.compose.resources.StringResource import org.jetbrains.compose.resources.stringResource +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.resources.Res import org.scottishtecharmy.soundscape.resources.settings_collapse_section import org.scottishtecharmy.soundscape.resources.settings_collapsed @@ -51,6 +53,9 @@ fun ListPreferenceItem( ) { Row( modifier = Modifier + // The choices are shown in a dialog, outside the theme's testTagsAsResourceId. + .testTagsAsResourceId() + .testTag("settingsOption-$value") .background(MaterialTheme.colorScheme.surfaceContainer) .extraSmallPadding() .defaultMinSize(minHeight = spacing.targetSize) @@ -119,9 +124,10 @@ fun ExpandableSectionHeader( expanded: Boolean, onToggle: () -> Unit, textColor: Color, + modifier: Modifier = Modifier, ) { Row( - modifier = Modifier + modifier = modifier .fillMaxWidth() .clickable( role = Role.Button, diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SharedSettingsScreen.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SharedSettingsScreen.kt index 6e91d48fd..f941ae2ad 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SharedSettingsScreen.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/home/settings/SharedSettingsScreen.kt @@ -29,6 +29,7 @@ import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.testTag import androidx.compose.ui.semantics.Role import androidx.compose.ui.semantics.contentDescription import androidx.compose.ui.semantics.semantics @@ -301,6 +302,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "callouts") null else "callouts" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionCallouts"), ) } if (expandedSection.value == "callouts") { @@ -416,6 +418,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "search") null else "search" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionSearch"), ) } if (expandedSection.value == "search") { @@ -423,7 +426,7 @@ fun SharedSettingsScreen( key = PreferenceKeys.GEOCODER_MODE, defaultValue = PreferenceDefaults.GEOCODER_MODE, values = geocoderValues, - modifier = expandedSectionModifier, + modifier = expandedSectionModifier.testTag("settingsSearchMode"), title = { SettingTitle(Res.string.settings_section_search_network, textColor) }, item = { value, currentValue, onClick -> ListPreferenceItem( @@ -485,6 +488,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "accessibility") null else "accessibility" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionAccessibility"), ) } if (expandedSection.value == "accessibility") { @@ -503,7 +507,7 @@ fun SharedSettingsScreen( color = textColor, ) }, - modifier = expandedSectionModifier, + modifier = expandedSectionModifier.testTag("settingsShowMap"), ) } } @@ -524,6 +528,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "storage") null else "storage" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionStorage"), ) } if (expandedSection.value == "storage") { @@ -541,6 +546,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "audio") null else "audio" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionAudio"), ) } if (expandedSection.value == "audio") { @@ -591,6 +597,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "language") null else "language" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionLanguage"), ) } if (expandedSection.value == "language") { @@ -632,6 +639,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "media_controls") null else "media_controls" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionMediaControls"), ) } if (expandedSection.value == "media_controls") { @@ -673,6 +681,7 @@ fun SharedSettingsScreen( if (expandedSection.value == "debug") null else "debug" }, textColor = textColor, + modifier = Modifier.testTag("settingsSectionDebug"), ) } if (expandedSection.value == "debug") { diff --git a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/markers_routes/screens/addandeditroutescreen/SharedAddAndEditRouteScreen.kt b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/markers_routes/screens/addandeditroutescreen/SharedAddAndEditRouteScreen.kt index d1b6c962f..d9b3aa6d9 100644 --- a/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/markers_routes/screens/addandeditroutescreen/SharedAddAndEditRouteScreen.kt +++ b/shared/src/commonMain/kotlin/org/scottishtecharmy/soundscape/screens/markers_routes/screens/addandeditroutescreen/SharedAddAndEditRouteScreen.kt @@ -51,6 +51,7 @@ import org.jetbrains.compose.resources.stringResource import org.scottishtecharmy.soundscape.components.LocationItem import org.scottishtecharmy.soundscape.components.LocationItemDecoration import org.scottishtecharmy.soundscape.geojsonparser.geojson.LngLatAlt +import org.scottishtecharmy.soundscape.platform.testTagsAsResourceId import org.scottishtecharmy.soundscape.resources.Res import org.scottishtecharmy.soundscape.resources.general_alert_cancel import org.scottishtecharmy.soundscape.resources.general_alert_done @@ -160,6 +161,8 @@ fun SharedAddAndEditRouteScreen( if (showDeleteDialog) { AlertDialog( + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId(), onDismissRequest = { showDeleteDialog = false }, title = { Text(stringResource(Res.string.settings_reset_dialog_title)) }, text = { Text(stringResource(Res.string.route_detail_edit_delete_alert_message)) }, @@ -187,6 +190,8 @@ fun SharedAddAndEditRouteScreen( waypointToRemove?.let { waypoint -> AlertDialog( + // A dialog is its own window, outside the theme's testTagsAsResourceId. + modifier = Modifier.testTagsAsResourceId(), onDismissRequest = { waypointToRemove = null }, title = { Text(stringResource(Res.string.settings_reset_dialog_title)) }, text = { diff --git a/shared/src/iosMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt b/shared/src/iosMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt new file mode 100644 index 000000000..8b896aac2 --- /dev/null +++ b/shared/src/iosMain/kotlin/org/scottishtecharmy/soundscape/platform/TestTagsAsResourceId.kt @@ -0,0 +1,5 @@ +package org.scottishtecharmy.soundscape.platform + +import androidx.compose.ui.Modifier + +actual fun Modifier.testTagsAsResourceId(): Modifier = this diff --git a/test-farm/.gitignore b/test-farm/.gitignore new file mode 100644 index 000000000..0375d002f --- /dev/null +++ b/test-farm/.gitignore @@ -0,0 +1,10 @@ +farm.toml +farm.db +report.html +report.html.tmp +apks/ +runs/ +__pycache__/ +farm.log +extracts/ +backup-*/ diff --git a/test-farm/README.md b/test-farm/README.md new file mode 100644 index 000000000..644a06497 --- /dev/null +++ b/test-farm/README.md @@ -0,0 +1,172 @@ +# Test farm + +`farm.py` runs long-term tests of **release builds** on every Android phone +plugged into this PC. Each phone loops forever: + +1. If a newer release has been built, `adb install -r` it over the existing + app and data, then run `flows/PostUpgrade.yaml` to check the app starts and + the suite's markers and route survived. +2. **Set up**, once per release on each phone (recorded in the `setups` + table): the Maestro suite in `maestro/Master.yaml` order, one flow at a + time, held at `suite_location`. Onboarding wipes the app, so after it the + farm pushes the offline map and runs `flows/FarmSettings.yaml` (search mode + Offline). After the suite, which expects the map off as in CI, + `flows/ShowMap.yaml` turns the map on for roaming (or leaves it off + for a phone with `show_map = false` in its `[devices]` entry, which the + roaming flows then skip the map steps on). A phone whose Onboarding or + settings fail is set up again next cycle. +3. **Roam**: `roam_rounds` rounds of `flows/roam/*.yaml`, shuffled each round, + while the phone walks its GPX track. These look around (callouts, beacons on + places and search results, the route, markers, menus, sleep) and assert + only that the screens work, not what's nearby. They use the markers and + route the set-up made, and change nothing that later flows rely on. + Each starts with `flows/LaunchHome.yaml`, which brings the app to the + front and backs out to Home without restarting it, so one app process runs + through the rounds, the monkey and following cycles, soaking for hours. +4. A `monkey` soak (random input inside the app, fixed seed per run, logged). + Then `flows/Restore.yaml`, without restarting the app, puts back what the + monkey may have changed and roaming and the next upgrade rely on: search + mode Offline, markers B and C and Test Route (recreated where the suite + made them), and the map on. Only if that fails is the phone set up again + (a `reset` event). +5. Collect crashes and ANRs (from dropbox), logcat, memory (PSS), battery + and how long the app process has been up (`app_uptime_s`; a value that + resets without a crash means something else restarted the app). + +Flow results are recorded by kind: `flow` (set-up suite), `setup` +(FarmSettings, DownloadExtract, ShowMap), `roam`, `monkey`, `restore` and +`upgrade`. + +Releases come from the upstream `build-app.yaml` workflow's `release-apk` +artifact (the APK that goes to Play), via `gh`. The newest two are cached in +`apks/`, and a freshly attached phone starts one release behind so its first +upgrade happens after its first cycle. Artifacts expire after 90 days, so +leave the farm running (or keep `apks/`) to keep older releases. + +Results: `report.html` (refreshes every minute), `farm.db` (SQLite), and +per-cycle files under `runs///`. + +```bash +cp farm.toml.example farm.toml # optional +python3 test-farm/farm.py # Ctrl-C to stop; removes the mock GPS providers +``` + +`test_farm.py` checks the parts that need no phone: the config, the flows' +paths into `maestro/`, the settings shared with `run-maestro-tests.yaml` +(suite location, launch-failure retries) and the parsers of adb output. +`run-tests.yaml` runs it on every PR and builds the mock location helper. + +```bash +python3 -m unittest discover -s test-farm -v +``` + +## How release builds are made testable + +- **Analytics:** the farm sets `settings put system firebase.test.lab true`, which + makes `MainActivity` use dummy analytics, as it does in Firebase Test Lab. +- **Location:** release builds don't have the debug `REPLAY_GPX` intent, so + the farm installs a small mock location helper (`mock-location/`, built by + its `build.sh` with plain SDK tools; it isn't Soundscape) and grants it the + mock location app-op. It feeds the helper's `gps` and `network` test + providers a fix once a second, with **speed and bearing**. Soundscape only + takes its direction from travel when a fix says the user is moving. + Without them (all that `cmd location providers` can set), callouts follow + the compass of a phone lying on a desk. This reaches both the fused provider + (phones with Play services) and plain `LocationManager` (AOSP phones). The + helper's receiver requires `android.permission.DUMP`, so only adb can feed it. + Tracks are thinned to points 15 m apart, + because a phone recording that starts with its owner standing still + begins with minutes of on-the-spot GPS jitter. While the Maestro suite runs, the phone is + held still (speed 0) at `suite_location` (CI's Edinburgh point), because + the flows assert distance-sorted list order. + iPhones can't get the same fix: `simulate-location` sets latitude and + longitude only. Whether iOS adds a course to simulated fixes is untested. +- **Maestro:** `testTagsAsResourceId` is set in `Theme.kt` for every build type. +- **Offline maps and search:** the phones' Wi-Fi can't be relied on, so the + farm keeps the current manifest extract (`extract_name`, default Glasgow, + which covers the suite point and the GPX walk) in `extracts/`, using + `extractProviderUrl` from `local.properties`. Onboarding wipes app storage, + so after it runs the farm `adb push`es the extract into the + app's `files/Download`. That needs no root on most phones. Where the + shell can't write to other apps' `Android/data` (the Cubot J10), the app + downloads the extract itself instead (`flows/DownloadExtract.yaml`), through + this PC's network: the farm runs a small HTTP proxy (`usb_proxy.py`), + `adb reverse`s its port onto the phone and sets the phone's global HTTP + proxy to it for the download only. + It then runs `flows/FarmSettings.yaml` to set Settings › Search › Search + mode to Offline. Releases + before the Settings test tags are driven by English labels, so the phones + must be set to English for now. The farm turns **Wi-Fi off** on every + phone when it's attached and after each monkey soak, so everything runs + offline. That also stops Google's fused location (on phones with Play + services) mixing the phone's real Wi-Fi position into the mock fixes. + +## iPhones + +Maestro can't drive a physical iPhone. It drives iOS through an XCUITest +driver that only Xcode on a Mac can build, sign and launch, and its CLI only +targets simulators. So iPhones are **observed, not driven**. +`pymobiledevice3` (`pipx install pymobiledevice3`) does everything from Linux +without root, because `--userspace` builds the iOS 17+ developer tunnel +in-process. Every `ios_poll_minutes` (10) the farm: + +- mounts the developer disk image (needed after a reboot); +- records a TestFlight version change as an upgrade (TestFlight auto-update + installs releases; the App Store-signed IPA can't be sideloaded); +- relaunches Soundscape if it isn't running, and records a PID change with no + crash report as a restart (killed by iOS or the user); +- pulls new `Soundscape-*` crash and hang reports. bug_type 309 counts as a + crash; anything else is counted with the ANRs; +- reads the battery. + +Throughout, it replays the GPX walk with `simulate-location play`. The app +ignored a single fixed simulated point, so the walk has to move. It also keeps +a daily Soundscape syslog (`--no-debug`, about 66 MB/day) in `runs//`. + +Preparing an iPhone: Developer Mode on, paired with this PC ("Trust"), Soundscape +from TestFlight with automatic updates on, and the battery charge limit set to +80%. Keep it off USB hubs that lose disconnects. One VIA Labs hub here kept a +ghost iPhone connection that hung usbmuxd until it was removed via +`/sys/bus/usb/devices//remove` and usbmuxd was restarted. + +## Preparing a phone + +- Developer options on, USB debugging on, and on Xiaomi/Oppo-type phones also + "USB debugging (Security settings)" so adb can tap and install. +- Screen lock **None**. Maestro can't get past a PIN. +- An existing Soundscape is fine if it's a sideloaded release. The farm + only tests a build it installed itself (it records the install time), but + it first tries installing the latest release over a foreign copy. A + release-signed copy accepts that, keeping its data like a user's update, + and the phone is adopted. A Play Store or debug copy is signed differently + and refuses, which leaves the phone **blocked** and untested unless + `allow_reinstall = true`, which uninstalls the copy and its data. +- Turn off Play Store auto-updates, so Play doesn't replace the farm's build. +- Charge limit (e.g. 80%) where the phone has one. A phone on charge 24/7 + without one will eventually swell its battery. +- Use a powered USB hub. + +The farm sets the screen to minimum brightness and media volume to zero when a +phone is attached and again after every monkey soak. The audio still plays at +zero volume, so the audio path is still tested. + +The farm deliberately leaves the OEM battery savers alone. If one kills the +service, that shows up as a failure, which is a real user problem. + +## Known limits + +- Set-up wipes app data (Onboarding uses `clearState`), once per release. The + data a release upgrades over is whatever the previous release's cycles left. +- The monkey can change other settings (language, voice, callouts). + Restore only puts back search mode, the map and the markers and route. +- The monkey can open the notification shade. After each soak the farm turns + off airplane mode, turns off Wi-Fi and collapses the shade. +- Each phone's Maestro gets its own `XDG_CACHE_HOME`/`XDG_STATE_HOME` + (`runs//maestro-home/`). Maestro prunes all but its newest few + runs' logs, so with a shared `~/.cache/maestro` a long run lost its log + folder to another phone's run, crashed at exit and hung until the flow + timeout. +- Maestro runs on several phones at once. One run lost its driver connection + (`gRPC UNAVAILABLE`) when another phone's session was starting, and that + error is now retried once. If it keeps happening, set + `serialize_maestro = true`. diff --git a/test-farm/farm.py b/test-farm/farm.py new file mode 100644 index 000000000..12d10fd13 --- /dev/null +++ b/test-farm/farm.py @@ -0,0 +1,1406 @@ +#!/usr/bin/env python3 +"""Soundscape Android test farm. + +Runs long-term tests of release builds on every Android phone plugged into this +PC. One worker thread per phone loops forever: + + upgrade (if a newer release exists) -> PostUpgrade check + monkey soak -> the Maestro suite (maestro/Master.yaml order) + collect crashes/ANRs, memory, battery, logcat + +New releases are picked up from the upstream "Tag and build release" workflow's +release-apk artifact (the same APK that goes to Play) and installed with +`adb install -r` over the phone's existing data, so the real upgrade path is +exercised. Location is driven over adb with a shell mock provider replaying a +GPX track, because release builds don't have the debug REPLAY_GPX intent. + +Results go into farm.db and report.html next to this script. See README.md. +""" + +import argparse +import contextlib +import datetime as dt +import glob +import gzip +import json +import html +import math +import os +import random +import re +import signal +import sqlite3 +import subprocess +import sys +import threading +import time +import tomllib +import xml.etree.ElementTree as ET + +import usb_proxy + +FARM_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_DIR = os.path.dirname(FARM_DIR) +MAESTRO_DIR = os.path.join(REPO_DIR, "maestro") +POST_UPGRADE_FLOW = os.path.join(FARM_DIR, "flows", "PostUpgrade.yaml") +APK_DIR = os.path.join(FARM_DIR, "apks") +EXTRACT_DIR = os.path.join(FARM_DIR, "extracts") +FARM_SETTINGS_FLOW = os.path.join(FARM_DIR, "flows", "FarmSettings.yaml") +DOWNLOAD_EXTRACT_FLOW = os.path.join(FARM_DIR, "flows", "DownloadExtract.yaml") +SHOW_MAP_FLOW = os.path.join(FARM_DIR, "flows", "ShowMap.yaml") +RESTORE_FLOW = os.path.join(FARM_DIR, "flows", "Restore.yaml") +ROAM_DIR = os.path.join(FARM_DIR, "flows", "roam") +RUNS_DIR = os.path.join(FARM_DIR, "runs") +DB_PATH = os.path.join(FARM_DIR, "farm.db") +REPORT_PATH = os.path.join(FARM_DIR, "report.html") + +PACKAGE = "org.scottishtecharmy.soundscape" +MOCK_PROVIDERS = ("gps", "network") +# GeoEngine reads extracts from /Download, which defaults to +# the app's external files dir (as CI's push in run-maestro-tests.yaml). +DEVICE_EXTRACT_DIR = f"/storage/emulated/0/Android/data/{PACKAGE}/files/Download" + +DEFAULTS = { + "release_repo": "Scottish-Tech-Army/Soundscape-Android", + "release_workflow": "build-app.yaml", + "release_artifact": "release-apk", + "poll_minutes": 15, + "monkey_events": 5000, + # Rounds of the roaming flows (flows/roam/, shuffled each round) per cycle. + "roam_rounds": 2, + "monkey_throttle_ms": 250, + "flow_timeout_minutes": 15, + "keep_days": 14, + "serialize_maestro": False, + "allow_reinstall": False, + "include_emulators": False, + # iPhones (via pymobiledevice3) are observed, not driven: see IosWorker. + "include_ios": True, + "ios_poll_minutes": 10, + "max_battery_temp_c": 42, + "replay_speed_mps": 1.4, + "default_location": [55.9474, -4.3058], + # Held while the Maestro suite runs: the flows assert distance-sorted list + # order, so they need the fixed point CI pins with `adb emu geo fix`. + "suite_location": [55.955360, -3.223538], + # The offline map pushed to every phone, by its manifest "name". It has to + # cover suite_location and the GPX track. The manifest URL is read from + # local.properties' extractProviderUrl unless set here. + "extract_name": "Glasgow", + "extract_provider_url": "", + "default_gpx": "", + "devices": {}, +} + +# Same transient launch failures CI retries once (see run-maestro-tests.yaml). +MAESTRO_RETRY_RE = re.compile( + r"Unable to launch app|device offline|Unknown error|Not able to reach the gRPC server") + +stop_event = threading.Event() +maestro_lock = threading.Lock() +state_lock = threading.Lock() +device_state = {} # serial -> {"state": str, "since": float, "model": str, ...} + + +def log(serial, msg): + stamp = dt.datetime.now().strftime("%H:%M:%S") + print(f"{stamp} [{serial or 'farm'}] {msg}", flush=True) + + +def set_state(serial, state, **extra): + with state_lock: + entry = device_state.setdefault(serial, {}) + entry.update(extra) + entry["state"] = state + entry["since"] = time.time() + log(serial, state) + + +# --------------------------------------------------------------------------- config + + +def load_config(path): + cfg = dict(DEFAULTS) + if path and os.path.exists(path): + with open(path, "rb") as f: + cfg.update(tomllib.load(f)) + base = os.path.dirname(os.path.abspath(path)) + else: + base = FARM_DIR + # GPX paths in the config are relative to the config file. + def resolve(p): + return os.path.normpath(os.path.join(base, p)) if p else "" + cfg["default_gpx"] = resolve(cfg["default_gpx"]) + for dev in cfg["devices"].values(): + if "gpx" in dev: + dev["gpx"] = resolve(dev["gpx"]) + return cfg + + +# --------------------------------------------------------------------------- database + + +class Db: + def __init__(self, path): + self.conn = sqlite3.connect(path, check_same_thread=False) + self.conn.row_factory = sqlite3.Row + self.lock = threading.Lock() + self.conn.executescript(""" + CREATE TABLE IF NOT EXISTS builds( + run_id INTEGER PRIMARY KEY, version_code INTEGER, version_name TEXT, + created_at TEXT, apk_path TEXT); + CREATE TABLE IF NOT EXISTS cycles( + id INTEGER PRIMARY KEY, serial TEXT, model TEXT, android TEXT, + version_code INTEGER, started REAL, ended REAL, status TEXT, + flows_passed INTEGER, flows_failed INTEGER, crashes INTEGER, anrs INTEGER, + pss_kb INTEGER, battery_level INTEGER, battery_temp REAL, dir TEXT); + CREATE TABLE IF NOT EXISTS results( + id INTEGER PRIMARY KEY, cycle_id INTEGER, serial TEXT, ts REAL, + kind TEXT, name TEXT, status TEXT, duration_s REAL, detail TEXT, dir TEXT); + CREATE TABLE IF NOT EXISTS installs( + serial TEXT PRIMARY KEY, last_update TEXT); + CREATE TABLE IF NOT EXISTS setups( + serial TEXT PRIMARY KEY, version_code INTEGER); + CREATE TABLE IF NOT EXISTS events( + id INTEGER PRIMARY KEY, serial TEXT, ts REAL, kind TEXT, + version_code INTEGER, detail TEXT); + """) + try: # added after the table: whether the set-up suite all passed + self.conn.execute("ALTER TABLE setups ADD COLUMN complete INTEGER DEFAULT 1") + except sqlite3.OperationalError: + pass + try: # added after the table: how long the app process had been running + self.conn.execute("ALTER TABLE cycles ADD COLUMN app_uptime_s INTEGER") + except sqlite3.OperationalError: + pass + + def execute(self, sql, args=()): + with self.lock: + cur = self.conn.execute(sql, args) + self.conn.commit() + return cur.lastrowid + + def query(self, sql, args=()): + with self.lock: + return self.conn.execute(sql, args).fetchall() + + def event(self, serial, kind, version_code, detail): + log(serial, f"EVENT {kind}: {detail.splitlines()[0][:200] if detail else ''}") + self.execute("INSERT INTO events(serial, ts, kind, version_code, detail) VALUES(?,?,?,?,?)", + (serial, time.time(), kind, version_code, detail)) + + def builds(self): + return self.query("SELECT * FROM builds ORDER BY version_code") + + +# --------------------------------------------------------------------------- adb + + +children = set() # running subprocesses, stopped when the farm stops +children_lock = threading.Lock() + + +def run(cmd, timeout=120, env=None): + """subprocess.run, except that the farm can stop the process when it stops. + A Maestro run left behind by a stopped farm keeps hold of the phone's + driver, and every flow after a restart then fails with gRPC UNAVAILABLE.""" + with children_lock: + # Once the farm is stopping, start nothing new: stop_children() may + # already have run, and anything started now would be orphaned. + if stop_event.is_set(): + return subprocess.CompletedProcess(cmd, -1, "", "farm stopping") + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, errors="replace", + env={**os.environ, **env} if env else None) + children.add(proc) + try: + out, err = proc.communicate(timeout=timeout) + return subprocess.CompletedProcess(cmd, proc.returncode, out, err) + except subprocess.TimeoutExpired: + proc.kill() + out, _ = proc.communicate() + return subprocess.CompletedProcess(cmd, -1, out or "", f"timeout after {timeout}s") + finally: + with children_lock: + children.discard(proc) + + +def stop_children(): + with children_lock: + procs = list(children) + for proc in procs: + proc.terminate() + for proc in procs: + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + + +def adb(serial, *args, timeout=120): + return run(["adb", "-s", serial, *args], timeout=timeout) + + +def shell(serial, command, timeout=120): + return adb(serial, "shell", command, timeout=timeout).stdout.strip() + + +def connected_devices(include_emulators): + out = run(["adb", "devices"]).stdout + serials = [] + for line in out.splitlines()[1:]: + parts = line.split() + if len(parts) == 2 and parts[1] == "device": + if include_emulators or not parts[0].startswith("emulator-"): + serials.append(parts[0]) + return serials + + +def installed_version(serial): + out = shell(serial, f"dumpsys package {PACKAGE} | grep -m1 versionCode") + m = re.search(r"versionCode=(\d+)", out) + return int(m.group(1)) if m else None + + +def find_aapt2(): + sdk = os.environ.get("ANDROID_HOME") or os.path.expanduser("~/Android/Sdk") + tools = sorted(glob.glob(os.path.join(sdk, "build-tools", "*", "aapt2"))) + if not tools: + sys.exit("aapt2 not found: set ANDROID_HOME to your Android SDK") + return tools[-1] + + +# --------------------------------------------------------------------------- releases + + +def release_watcher(cfg, db): + """Caches the APKs of the two newest successful release builds.""" + aapt2 = find_aapt2() + known = {b["run_id"] for b in db.builds()} + while not stop_event.is_set(): + res = run(["gh", "run", "list", "-R", cfg["release_repo"], "-w", cfg["release_workflow"], + "-s", "success", "-L", "2", "--json", "databaseId,createdAt", + "-q", '.[] | "\\(.databaseId) \\(.createdAt)"']) + if res.returncode != 0: + log(None, f"gh run list failed: {res.stderr.strip()}") + for line in res.stdout.split("\n"): + if not line.strip(): + continue + run_id, created = line.split() + if int(run_id) in known: + continue + dest = os.path.join(APK_DIR, f"run-{run_id}") + log(None, f"downloading release build from run {run_id}") + dl = run(["gh", "run", "download", run_id, "-R", cfg["release_repo"], + "-n", cfg["release_artifact"], "-D", dest], timeout=900) + apks = glob.glob(os.path.join(dest, "**", "*.apk"), recursive=True) + if dl.returncode != 0 or not apks: + log(None, f"download of run {run_id} failed: {dl.stderr.strip()}") + continue + badging = run([aapt2, "dump", "badging", apks[0]]).stdout + m = re.search(r"versionCode='(\d+)' versionName='([^']*)'", badging) + if not m: + log(None, f"could not read version from {apks[0]}") + continue + db.execute("INSERT INTO builds VALUES(?,?,?,?,?)", + (int(run_id), int(m.group(1)), m.group(2), created, apks[0])) + known.add(int(run_id)) + log(None, f"new release {m.group(2)} ({m.group(1)}) cached") + sync_extract(cfg) + stop_event.wait(cfg["poll_minutes"] * 60) + + +# --------------------------------------------------------------------------- offline map + +current_extract = None # local path of the cached .pmtiles; its sidecar is path + ".geojson" + + +def extract_provider_url(cfg): + if cfg["extract_provider_url"]: + return cfg["extract_provider_url"] + try: + with open(os.path.join(REPO_DIR, "local.properties")) as f: + m = re.search(r"^extractProviderUrl=(.+)$", f.read(), re.M) + return m.group(1).strip() if m else "" + except OSError: + return "" + + +def sync_extract(cfg): + """Keeps the manifest's current extract for cfg["extract_name"] cached locally. + + Downloads only when the manifest names a new file, so the monthly rebuild + is picked up and nothing else is fetched. + """ + global current_extract + base = extract_provider_url(cfg) + if not base: + log(None, "no extractProviderUrl in local.properties; phones will use the network") + return + res = subprocess.run(["curl", "-sfL", f"{base.rstrip('/')}/manifest.geojson.gz"], + capture_output=True, timeout=120) + try: + manifest = json.loads(gzip.decompress(res.stdout)) + feature = next(f for f in manifest["features"] + if f["properties"].get("name") == cfg["extract_name"]) + except Exception as e: # manifest unreachable or no such extract; keep what we have + log(None, f"extract manifest unusable ({e!r}); keeping {current_extract}") + if not current_extract: + cached = sorted(glob.glob(os.path.join(EXTRACT_DIR, "*.pmtiles"))) + current_extract = cached[-1] if cached else None + return + filename = feature["properties"]["filename"] + path = os.path.join(EXTRACT_DIR, filename) + if not os.path.exists(path): + log(None, f"downloading offline map {filename}") + part = path + ".part" + dl = run(["curl", "-sSfL", "-o", part, f"{base.rstrip('/')}/{filename}"], timeout=3600) + if dl.returncode != 0: + log(None, f"offline map download failed (exit {dl.returncode}): {dl.stderr.strip()}; " + "retrying at the next release poll") + return + os.replace(part, path) + with open(path + ".geojson", "w") as f: + json.dump(feature, f) + for old in glob.glob(os.path.join(EXTRACT_DIR, "*")): + if not old.startswith(path): + os.remove(old) + current_extract = path + + +# --------------------------------------------------------------------------- location + + +TRACK_MIN_STEP_M = 15 + + +def load_track(path): + """The track's points, keeping only those at least TRACK_MIN_STEP_M from the + last one kept. A phone recording that starts with its owner standing still + is minutes of GPS jitter (local-pixel.gpx: ~700 m of "path" within 15 m of + the start), which a constant-speed replay would otherwise walk on the spot.""" + root = ET.parse(path).getroot() + raw = [(float(p.get("lat")), float(p.get("lon"))) + for p in root.iter() if p.tag.split("}")[-1] in ("trkpt", "rtept")] + points = raw[:1] + for p in raw[1:]: + if haversine(points[-1], p) >= TRACK_MIN_STEP_M: + points.append(p) + if len(points) < 2: + raise ValueError(f"{path}: need at least 2 track points {TRACK_MIN_STEP_M} m apart") + return points + + +def haversine(a, b): + lat1, lon1, lat2, lon2 = map(math.radians, (*a, *b)) + h = (math.sin((lat2 - lat1) / 2) ** 2 + + math.cos(lat1) * math.cos(lat2) * math.sin((lon2 - lon1) / 2) ** 2) + return 2 * 6371000 * math.asin(math.sqrt(h)) + + +class Track: + """Walks a GPX track at a constant speed, turning round at each end rather + than jumping back to the start. With no track, holds one fixed point.""" + + def __init__(self, gpx, fixed, speed): + self.speed = speed + self.points = load_track(gpx) if gpx else [tuple(fixed), tuple(fixed)] + self.cumulative = [0.0] + for a, b in zip(self.points, self.points[1:]): + self.cumulative.append(self.cumulative[-1] + haversine(a, b)) + + def round_trip_seconds(self): + return max(60, int(2 * self.cumulative[-1] / self.speed)) + + def position(self, t): + total = self.cumulative[-1] + if total == 0: + return self.points[0] + d = (self.speed * t) % (2 * total) + if d > total: + d = 2 * total - d + i = max(j for j, c in enumerate(self.cumulative) if c <= d) + if i == len(self.points) - 1: + return self.points[-1] + seg = self.cumulative[i + 1] - self.cumulative[i] + f = (d - self.cumulative[i]) / seg if seg else 0 + (la, lo), (lb, lob) = self.points[i], self.points[i + 1] + return la + (lb - la) * f, lo + (lob - lo) * f + + +class LocationFeeder(threading.Thread): + """Feeds the Track's position to an Android phone once a second over adb. + Setting `hold` to a (lat, lon) pins the phone there until it's set to None. + """ + + def __init__(self, serial, gpx, fixed, speed, db, alive): + super().__init__(daemon=True) + self.serial, self.db, self.alive = serial, db, alive + self.hold = None + self.track = Track(gpx, fixed, speed) + + def fix(self, t): + """(lat, lon, speed, bearing) at t seconds. Speed and bearing matter: + Soundscape only takes its direction from travel when a fix says the + user is moving, and otherwise uses the compass of a phone on a desk.""" + if self.hold: + return (*self.hold, 0.0, None) + here, ahead = self.track.position(t), self.track.position(t + 1) + if here == ahead: + return (*here, 0.0, None) + return (*here, self.track.speed, initial_bearing(here, ahead)) + + def run(self): + start = time.time() + failures = 0 + while not stop_event.is_set() and self.alive.is_set(): + lat, lon, speed, bearing = self.fix(time.time() - start) + extras = f"--es lat {lat:.7f} --es lon {lon:.7f} --es speed {speed:.2f}" + if bearing is not None: + extras += f" --es bearing {bearing:.1f}" + out = shell(self.serial, f"am broadcast -f 0x10000000 -n {HELPER_PKG}/.FixReceiver " + f"-a {HELPER_PKG}.FIX {extras}", timeout=10) + if 'data="ok"' in out: + failures = 0 + else: + # Keep going: if the feed stops, the phone falls back to its real + # position mid-suite, which breaks flows in confusing ways. + failures += 1 + if failures in (1, 60) or failures % 600 == 0: + self.db.event(self.serial, "error", None, + f"mock location rejected ({failures} in a row):\n{out}") + allow_helper(self.serial) + time.sleep(1) + + +def initial_bearing(a, b): + lat1, lon1, lat2, lon2 = map(math.radians, (*a, *b)) + y = math.sin(lon2 - lon1) * math.cos(lat2) + x = math.cos(lat1) * math.sin(lat2) - math.sin(lat1) * math.cos(lat2) * math.cos(lon2 - lon1) + return (math.degrees(math.atan2(y, x)) + 360) % 360 + + +# Broadcasts to the helper go on the foreground queue (-f 0x10000000, +# FLAG_RECEIVER_FOREGROUND): on the Android 11 A50 the background queue backed +# up behind other apps' broadcasts and a fix took over 15 s to land. +# +# The mock location helper in mock-location/. `cmd location providers` can only +# set a position, so fixes from it have no speed or bearing; the helper sets +# both. It isn't Soundscape, so the farm still tests only release builds. +# Deliberately not under org.scottishtecharmy.soundscape: as +# org.scottishtecharmy.soundscape.farmlocation, Maestro's launchApp stopped +# granting Soundscape's permissions on the AOSP Samsung (it worked again as soon +# as the helper was uninstalled or renamed), and Onboarding failed every cycle. +HELPER_PKG = "org.scottishtecharmy.farmlocation" +HELPER_DIR = os.path.join(FARM_DIR, "mock-location") +HELPER_APK = os.path.join(HELPER_DIR, "build", "farm-location.apk") + + +def build_helper(): + if not os.path.exists(HELPER_APK): + res = run([os.path.join(HELPER_DIR, "build.sh")], timeout=600) + if res.returncode != 0: + sys.exit("mock location helper build failed:\n" + res.stdout + res.stderr) + + +def enable_mock_location(serial): + # Shell test providers from older farm runs would sit beside the helper's. + disable_mock_location(serial, shell_providers=True) + out = adb(serial, "install", "-r", HELPER_APK, timeout=300) + if "Success" not in out.stdout + out.stderr: + return "helper install failed: " + out.stdout + out.stderr + return allow_helper(serial) + + +def allow_helper(serial): + """Grants the helper the mock location app-op. Straight after a fresh + install the grant can be lost (seen on the AOSP Samsung), so it's read back + and repeated until it sticks.""" + for _ in range(5): + shell(serial, f"appops set {HELPER_PKG} android:mock_location allow") + if "allow" in shell(serial, f"appops get {HELPER_PKG} android:mock_location"): + return "" + time.sleep(2) + return "failed to allow mock location for " + HELPER_PKG + + +def disable_mock_location(serial, shell_providers=False): + if shell_providers: + shell(serial, "; ".join(f"cmd location providers remove-test-provider {p}" + for p in MOCK_PROVIDERS), timeout=20) + else: + shell(serial, f"am broadcast -f 0x10000000 -n {HELPER_PKG}/.FixReceiver -a {HELPER_PKG}.STOP", + timeout=20) + + +# --------------------------------------------------------------------------- crashes + + +DROPBOX_TAGS = ("data_app_crash", "data_app_anr", "data_app_native_crash", "SYSTEM_TOMBSTONE") + + +def collect_dropbox(serial, since): + """Returns (kind, text) for every crash/ANR of our app logged at or after `since`. + + `since` is a device-local "YYYY-MM-DD HH:MM:SS" string, which is also the + format dropbox prints entries with, so plain string comparison works. + """ + # One tag per call: dumpsys dropbox only prints entries that match every + # argument, so several tags in one call print nothing. + out = "\n".join(shell(serial, f"dumpsys dropbox --print {tag}", timeout=60) + for tag in DROPBOX_TAGS) + found = [] + for entry in re.split(r"^={10,}\s*$", out, flags=re.M): + m = re.search(r"^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d) (\S+)", entry.strip(), re.M) + if m and m.group(1) >= since and PACKAGE in entry: + kind = "anr" if "anr" in m.group(2) else "crash" + found.append((kind, entry.strip())) + return found + + +def pss_kb(serial): + out = shell(serial, f"dumpsys meminfo {PACKAGE}", timeout=60) + m = re.search(r"TOTAL PSS:\s+(\d+)", out) or re.search(r"^\s*TOTAL\s+(\d+)", out, re.M) + return int(m.group(1)) if m else None + + +def app_uptime_s(serial): + """Seconds the app's main process has been running, or None if it isn't.""" + pid = shell(serial, f"pidof {PACKAGE}").split() + if not pid: + return None + # ps prints ELAPSED as [[dd-]hh:]mm:ss. + out = shell(serial, f"ps -o ETIME= -p {pid[0]}").strip() + m = re.fullmatch(r"(?:(?:(\d+)-)?(\d+):)?(\d+):(\d+)", out) + if not m: + return None + d, h, mins, secs = (int(g or 0) for g in m.groups()) + return ((d * 24 + h) * 60 + mins) * 60 + secs + + +def format_uptime(seconds): + if seconds is None: + return "app not running" + h, rem = divmod(seconds, 3600) + return f"app up {h}h{rem // 60:02d}m" if h else f"app up {rem // 60}m" + + +def battery(serial): + out = shell(serial, "dumpsys battery") + level = re.search(r"level: (\d+)", out) + temp = re.search(r"temperature: (\d+)", out) + return (int(level.group(1)) if level else None, + int(temp.group(1)) / 10 if temp else None) + + +# --------------------------------------------------------------------------- worker + + +def suite_flows(): + """The suite in Master.yaml order, which is the order the flows depend on.""" + with open(os.path.join(MAESTRO_DIR, "Master.yaml")) as f: + return re.findall(r"^- runFlow: (\S+\.yaml)", f.read(), re.M) + + +def roam_flows(): + return sorted(f for f in os.listdir(ROAM_DIR) if f.endswith(".yaml")) + + +class Worker(threading.Thread): + def __init__(self, serial, cfg, db): + super().__init__(daemon=True, name=serial) + self.serial, self.cfg, self.db = serial, cfg, db + self.dev_cfg = cfg["devices"].get(serial, {}) + self.alive = threading.Event() + self.alive.set() + self.model = shell(serial, "getprop ro.product.model") + self.android = shell(serial, "getprop ro.build.version.release") + self.last_cycle_ok = False + self.feeder = None + self.adopt_tried = set() # versions of a foreign install already tried + self.reinstall_driver = False # set after a Maestro run had to be killed + + # ----- setup + + def prepare(self): + s = self.serial + shell(s, "svc power stayon usb") + # Everything the farm tests works offline, and the farm's Wi-Fi is + # unreliable. It also keeps Wi-Fi scans out of Google's fused location, + # which would otherwise mix the phone's real position into the mock one. + shell(s, "svc wifi disable") + self.clear_usb_proxy() # in case the farm stopped mid-download + # MainActivity uses dummy analytics when this is set, which keeps the + # farm's traffic out of the production Firebase numbers. + shell(s, "settings put system firebase.test.lab true") + adb(s, "logcat", "-G", "16M") + self.quieten() + out = enable_mock_location(s) + if "Exception" in out or "failed" in out: + self.db.event(s, "error", None, f"could not enable mock location:\n{out}") + else: + gpx = self.dev_cfg.get("gpx", self.cfg["default_gpx"]) + self.feeder = LocationFeeder(s, gpx, self.cfg["default_location"], + self.cfg["replay_speed_mps"], self.db, self.alive) + self.feeder.start() + + def quieten(self): + """Dims the screen right down and turns media volume to zero.""" + s = self.serial + shell(s, "settings put system screen_brightness_mode 0; " + "settings put system screen_brightness 1; " + "cmd media_session volume --stream 3 --set 0") + # Some ROMs (the AOSP Samsung) ignore that. Volume-down keys work there + # because Soundscape is playing, so the keys adjust the media stream. + if "volume is 0 " not in shell(s, "cmd media_session volume --stream 3 --get"): + shell(s, "for i in $(seq 1 16); do input keyevent KEYCODE_VOLUME_DOWN; done") + + def push_extract(self): + """Copies the cached offline map and its metadata sidecar into the app's + extracts directory, named as CI does (without the build-date prefix). + Returns False if the phone won't allow it.""" + if not current_extract: + self.db.event(self.serial, "error", None, "no offline map cached; using the network") + return True + set_state(self.serial, "pushing offline map") + name = re.sub(r"^\d{8}-\d{4}-", "", os.path.basename(current_extract)) + shell(self.serial, f"rm -f {DEVICE_EXTRACT_DIR}/*.pmtiles {DEVICE_EXTRACT_DIR}/*.geojson; " + f"mkdir -p {DEVICE_EXTRACT_DIR}") + for src, dest in ((current_extract, name), (current_extract + ".geojson", name + ".geojson")): + res = adb(self.serial, "push", src, f"{DEVICE_EXTRACT_DIR}/{dest}", timeout=900) + if res.returncode != 0 and not stop_event.is_set(): + if "Permission denied" in res.stdout + res.stderr: + return False + self.db.event(self.serial, "error", None, + f"offline map push failed:\n{res.stdout}{res.stderr}") + return True + return True + + def download_extract(self, cycle_id, cycle_dir): + """For phones whose shell can't write to the app's Android/data (the + Cubot J10): the app downloads the extract itself from its Offline maps + screen, through this PC's network. usb_proxy's port is reversed onto + the phone and set as its global HTTP proxy for the download only, so + Wi-Fi stays off (the Cubot's barely reaches the access point).""" + s = self.serial + set_state(s, "downloading the offline map over USB") + adb(s, "reverse", f"tcp:{usb_proxy.PORT}", f"tcp:{usb_proxy.PORT}") + shell(s, f"settings put global http_proxy 127.0.0.1:{usb_proxy.PORT}") + try: + return self.run_flow(cycle_id, DOWNLOAD_EXTRACT_FLOW, cycle_dir, kind="setup", + env={"EXTRACT": self.cfg["extract_name"]}, + timeout_minutes=70) + finally: + self.clear_usb_proxy() + + def clear_usb_proxy(self): + shell(self.serial, "settings put global http_proxy :0") + adb(self.serial, "reverse", "--remove", f"tcp:{usb_proxy.PORT}") + + def repair(self): + """Undoes things the monkey can do from the notification shade.""" + shell(self.serial, "cmd connectivity airplane-mode disable; svc wifi disable; " + "cmd statusbar collapse; input keyevent KEYCODE_WAKEUP") + self.quieten() + + # ----- install / upgrade + + def last_update(self): + out = shell(self.serial, f"dumpsys package {PACKAGE} | grep -m1 lastUpdateTime") + return out.split("=", 1)[-1].strip() if out else None + + def farm_installed(self): + """True if the app on the phone is the one the farm last installed. + + Anything else (a Play Store install, a debug build, a Play update over + the farm's build) must not be tested as if it were a farm release. + """ + rows = self.db.query("SELECT last_update FROM installs WHERE serial=?", (self.serial,)) + return bool(rows) and rows[0]["last_update"] == self.last_update() + + def install(self, build, upgrade): + s = self.serial + before = installed_version(s) + res = adb(s, "install", "-r", build["apk_path"], timeout=600) + out = res.stdout + res.stderr + ok = res.returncode == 0 and "Success" in out + kind = "upgrade" if upgrade and before else "install" + if ok: + self.db.execute("INSERT OR REPLACE INTO installs VALUES(?,?)", (s, self.last_update())) + self.db.event(s, kind if ok else "error", build["version_code"], + f"{kind} {before} -> {build['version_name']} ({build['version_code']})" + + ("" if ok else "\n" + out)) + return ok + + def ensure_version(self, cycle_dir): + """Installs or upgrades as needed. Returns the version to test, or None.""" + builds = self.db.builds() + if not builds: + return None + latest = builds[-1] + current = installed_version(self.serial) + if (current is not None and not self.farm_installed() + and current <= latest["version_code"] and current not in self.adopt_tried): + self.adopt_tried.add(current) + # A sideloaded release (same signing key) accepts the latest release + # over the top, keeping its data, just as a user's update would. Play + # Store and debug installs are signed differently and refuse it. + set_state(self.serial, f"adopting Soundscape {current}") + if self.install(latest, upgrade=True): + self.db.event(self.serial, "adopt", latest["version_code"], + f"adopted a sideloaded release {current} by installing " + f"{latest['version_name']} over it") + return installed_version(self.serial) + if current is not None and not self.farm_installed(): + if not self.cfg["allow_reinstall"]: + set_state(self.serial, f"blocked: Soundscape {current} on the phone wasn't " + "installed by the farm; uninstall it by hand") + return None + self.db.event(self.serial, "uninstall", current, + "uninstalling a Soundscape the farm didn't install (allow_reinstall)") + adb(self.serial, "uninstall", PACKAGE) + current = None + if current is None: + # Start one release behind when we can, so the first upgrade + # happens as soon as the first cycle is done. + set_state(self.serial, "installing") + self.install(builds[-2] if len(builds) > 1 else latest, upgrade=False) + elif current < latest["version_code"]: + set_state(self.serial, f"upgrading to {latest['version_name']}") + if self.install(latest, upgrade=True): + note = "" if self.last_cycle_ok else \ + "previous cycle failed, so the suite's markers/route may not have existed" + with self.suite_location(): + self.run_flow(None, POST_UPGRADE_FLOW, cycle_dir, kind="upgrade", note=note) + return installed_version(self.serial) if self.farm_installed() else None + + @contextlib.contextmanager + def suite_location(self): + if self.feeder: + self.feeder.hold = tuple(self.cfg["suite_location"]) + try: + yield + finally: + if self.feeder: + self.feeder.hold = None + + # ----- tests + + def run_flow(self, cycle_id, path, cycle_dir, kind="flow", note="", record=True, env=None, + timeout_minutes=None): + """Runs one Maestro flow and records its result. Returns True or False, + or None if the farm stopped it (which isn't recorded).""" + name = os.path.splitext(os.path.basename(path))[0] + out_dir = os.path.join(cycle_dir, name) + n = 1 + while os.path.exists(out_dir): # the roaming flows run more than once + n += 1 + out_dir = os.path.join(cycle_dir, f"{name}-{n}") + os.makedirs(out_dir, exist_ok=True) + report = os.path.join(out_dir, "report.xml") + # Whether the phone roams with the map on (show_map in its [devices] + # entry). ShowMap.yaml sets it, and the roaming flows skip the map + # without it. Suite flows ignore it. + env = {"SHOW_MAP": "true" if self.dev_cfg.get("show_map", True) else "false", + **(env or {})} + cmd = [os.path.expanduser("~/.maestro/bin/maestro"), "test", "--device", self.serial, + "--format", "junit", "--output", report, "--test-output-dir", out_dir, + "--debug-output", os.path.join(out_dir, "debug"), "--flatten-debug-output", + "--no-ansi", *[a for k, v in env.items() for a in ("-e", f"{k}={v}")], path] + if self.reinstall_driver: + # A Maestro run killed at the timeout leaves the phone's driver + # unreachable, and the next flows fail at launch until it's + # reinstalled. + cmd = cmd[:2] + ["--reinstall-driver"] + cmd[2:] + self.reinstall_driver = False + timeout = (timeout_minutes or self.cfg["flow_timeout_minutes"]) * 60 + # Each Maestro run keeps its debug log in $XDG_CACHE_HOME/maestro/logs + # (else ~/.cache) and prunes all but the newest few runs' logs there. + # Shared between phones, a long run had its log folder deleted by + # another phone's run; its main thread then died at exit while the + # driver's threads kept the JVM alive, until the flow timeout. + own = os.path.join(RUNS_DIR, self.serial, "maestro-home") + maestro_env = {"XDG_CACHE_HOME": os.path.join(own, "cache"), + "XDG_STATE_HOME": os.path.join(own, "state")} + start = time.time() + for attempt in range(2): + set_state(self.serial, f"running {name}") + if self.cfg["serialize_maestro"]: + with maestro_lock: + res = run(cmd, timeout=timeout, env=maestro_env) + else: + res = run(cmd, timeout=timeout, env=maestro_env) + output = res.stdout + res.stderr + # The driver errors that decide a retry only reach the debug log. + for extra in (report, os.path.join(out_dir, "debug", "maestro.log")): + try: + with open(extra, errors="replace") as f: + output += f.read() + except OSError: + pass + if res.returncode == 0 or not MAESTRO_RETRY_RE.search(output) or attempt: + break + log(self.serial, f"{name}: transient launch failure, retrying once") + if "gRPC" in output and "--reinstall-driver" not in cmd: + # Maestro's on-device driver is unreachable (seen after a phone + # sat on its lock screen); a fresh install brings it back. + cmd = cmd[:2] + ["--reinstall-driver"] + cmd[2:] + with open(os.path.join(out_dir, "maestro.log"), "w") as f: + f.write(res.stdout + res.stderr) + if "timeout after" in res.stderr: + self.reinstall_driver = True + if stop_event.is_set(): + return None + status = "pass" if res.returncode == 0 else "fail" + if status == "fail" and "timeout after" in res.stderr and "Flow Passed" in res.stdout: + # Maestro reported the pass and then hung on the way out until the + # flow timeout killed it (see maestro_env above). + status = "pass" + note = "\n".join(filter(None, [note, "Maestro hung after the flow passed; killed"])) + if not record: + return status == "pass" + detail = "\n".join(filter(None, [note, "" if status == "pass" else + (res.stderr or res.stdout)[-2000:]])) + self.db.execute( + "INSERT INTO results(cycle_id, serial, ts, kind, name, status, duration_s, detail, dir) " + "VALUES(?,?,?,?,?,?,?,?,?)", + (cycle_id, self.serial, start, kind, name, status, time.time() - start, detail, out_dir)) + return status == "pass" + + def run_monkey(self, cycle_id, cycle_dir): + set_state(self.serial, "monkey soak") + seed = int(time.time()) % 100000 + start = time.time() + res = adb(self.serial, "shell", "monkey", "-p", PACKAGE, "-s", str(seed), + "--throttle", str(self.cfg["monkey_throttle_ms"]), + "--pct-syskeys", "0", "-v", str(self.cfg["monkey_events"]), + timeout=self.cfg["monkey_events"] * self.cfg["monkey_throttle_ms"] / 1000 + 600) + with open(os.path.join(cycle_dir, "monkey.log"), "w") as f: + f.write(res.stdout + res.stderr) + ok = "Monkey finished" in res.stdout + self.db.execute( + "INSERT INTO results(cycle_id, serial, ts, kind, name, status, duration_s, detail, dir) " + "VALUES(?,?,?,?,?,?,?,?,?)", + (cycle_id, self.serial, start, "monkey", f"monkey seed {seed}", + "pass" if ok else "fail", time.time() - start, + "" if ok else res.stdout[-2000:], cycle_dir)) + self.repair() + return ok + + def set_up_version(self): + rows = self.db.query("SELECT version_code FROM setups WHERE serial=?", (self.serial,)) + return rows[0]["version_code"] if rows else None + + def set_up(self, cycle_id, cycle_dir, version, tally): + """Onboards from scratch and builds the state the roaming flows and + PostUpgrade use: runs the Maestro suite in Master.yaml order, held at + suite_location, with the offline map pushed and the farm's settings + applied after Onboarding. Done once per release on each phone.""" + # Onboarding wipes whatever an earlier set-up made, so until this one + # finishes the phone isn't set up (a farm stopped mid-way redoes it). + self.db.execute("DELETE FROM setups WHERE serial=?", (self.serial,)) + failed_before = tally[False] + with self.suite_location(): + for flow in suite_flows(): + if stop_event.is_set(): + return + result = self.run_flow(cycle_id, os.path.join(MAESTRO_DIR, flow), cycle_dir) + if result is not None: + tally[result] += 1 + if flow == "Onboarding.yaml": + if not result: + return # tried again next cycle + # Onboarding cleared the app's storage and settings, so put the + # offline map back and switch search to offline. + if not self.push_extract(): + result = self.download_extract(cycle_id, cycle_dir) + if result is not None: + tally[result] += 1 + if not result: + return + result = self.run_flow(cycle_id, FARM_SETTINGS_FLOW, cycle_dir, kind="setup") + if result is not None: + tally[result] += 1 + if not result: + return + if stop_event.is_set(): + return + # The suite runs with the map off, as in CI; roaming has it on. + result = self.run_flow(cycle_id, SHOW_MAP_FLOW, cycle_dir, kind="setup") + if result is None: + return + tally[result] += 1 + # Recorded even if a later suite flow failed: that's a result, and a + # missing marker or route is caught by the check after the monkey. + self.db.execute("INSERT OR REPLACE INTO setups VALUES(?,?,?)", + (self.serial, version, int(tally[False] == failed_before))) + + def cycle(self): + s = self.serial + level, temp = battery(s) + if temp and temp > self.cfg["max_battery_temp_c"]: + set_state(s, f"cooling down ({temp}°C)") + stop_event.wait(600) + return + stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") + cycle_dir = os.path.join(RUNS_DIR, s, stamp) + + if "isKeyguardShowing=true" in shell(s, "dumpsys window | grep isKeyguardShowing"): + shell(s, "input keyevent KEYCODE_WAKEUP; wm dismiss-keyguard") + if "isKeyguardShowing=true" in shell(s, "dumpsys window | grep isKeyguardShowing"): + # A PIN or pattern: Maestro can't get past it, and every flow + # fails in under a second with the driver unreachable. + set_state(s, "locked: set Screen lock to None on this phone") + stop_event.wait(300) + return + + version = self.ensure_version(cycle_dir) + if version is None: + if not self.db.builds(): + set_state(s, "waiting for a release download") + stop_event.wait(300) + return + os.makedirs(cycle_dir, exist_ok=True) + + device_since = shell(s, "date '+%Y-%m-%d %H:%M:%S'") + adb(s, "logcat", "-c") + cycle_id = self.db.execute( + "INSERT INTO cycles(serial, model, android, version_code, started, status, dir) " + "VALUES(?,?,?,?,?,?,?)", + (s, self.model, self.android, version, time.time(), "running", cycle_dir)) + + tally = {True: 0, False: 0} + if self.set_up_version() != version: + self.set_up(cycle_id, cycle_dir, version, tally) + # The roaming flows only look around and don't assert what's nearby, so + # they run on the moving walk rather than held at suite_location. + # Without a finished set-up there's no offline map, settings, markers + # or route to roam with; the monkey still runs and set-up is retried + # next cycle. + flows = roam_flows() if self.set_up_version() == version else [] + complete = self.db.query("SELECT complete FROM setups WHERE serial=?", (s,)) + if flows and not complete[0]["complete"] and not stop_event.is_set(): + # A suite flow failed during set-up; build what it missed first. + with self.suite_location(): + restored = self.run_flow(cycle_id, RESTORE_FLOW, cycle_dir, kind="restore") + if restored is not None: + tally[restored] += 1 + if restored: + self.db.execute("UPDATE setups SET complete=1 WHERE serial=?", (s,)) + for _ in range(self.cfg["roam_rounds"]): + random.shuffle(flows) + for flow in flows: + if stop_event.is_set(): + break + result = self.run_flow(cycle_id, os.path.join(ROAM_DIR, flow), cycle_dir, + kind="roam") + if result is not None: + tally[result] += 1 + if not stop_event.is_set(): + self.run_monkey(cycle_id, cycle_dir) + # The monkey can delete the markers or route, or change settings, and + # roaming and the next upgrade's PostUpgrade check need them. Restore + # puts them back without restarting the app; only if it can't is the + # phone set up again, now rather than at the start of the next cycle, + # which may begin with an upgrade. + # Restore also runs when a suite flow failed during set-up, and builds + # what that flow didn't; otherwise every roam round fails for want of + # a marker or the route until the next release. A reset only follows + # a set-up that completed: after one that didn't, it would most likely + # fail the same way and onboard again every cycle. + complete = self.db.query("SELECT complete FROM setups WHERE serial=?", (s,)) + if not stop_event.is_set() and self.set_up_version() == version: + with self.suite_location(): + restored = self.run_flow(cycle_id, RESTORE_FLOW, cycle_dir, kind="restore") + if restored is not None: + tally[restored] += 1 + if restored and not complete[0]["complete"]: + self.db.execute("UPDATE setups SET complete=1 WHERE serial=?", (s,)) + if restored is False and complete[0]["complete"]: + self.db.event(s, "reset", version, "couldn't restore the markers, route or " + "settings after the monkey soak; setting up again") + self.set_up(cycle_id, cycle_dir, version, tally) + passed, failed = tally[True], tally[False] + if stop_event.is_set(): + self.db.execute("UPDATE cycles SET ended=?, status=? WHERE id=?", + (time.time(), "stopped", cycle_id)) + return + + with open(os.path.join(cycle_dir, "logcat.txt"), "w") as f: + f.write(adb(s, "logcat", "-d", timeout=120).stdout) + problems = collect_dropbox(s, device_since) + for i, (kind, text) in enumerate(problems): + with open(os.path.join(cycle_dir, f"{kind}-{i}.txt"), "w") as f: + f.write(text) + self.db.event(s, kind, version, text) + crashes = sum(1 for k, _ in problems if k == "crash") + anrs = len(problems) - crashes + level, temp = battery(s) + status = "pass" if failed == 0 and not problems else "fail" + self.last_cycle_ok = failed == 0 + uptime = app_uptime_s(s) + self.db.execute( + "UPDATE cycles SET ended=?, status=?, flows_passed=?, flows_failed=?, crashes=?, anrs=?, " + "pss_kb=?, battery_level=?, battery_temp=?, app_uptime_s=? WHERE id=?", + (time.time(), status, passed, failed, crashes, anrs, pss_kb(s), level, temp, uptime, + cycle_id)) + set_state(s, f"cycle {status}: {passed} passed, {failed} failed, " + f"{crashes} crashes, {anrs} ANRs, {format_uptime(uptime)}") + + def run(self): + set_state(self.serial, "preparing", model=self.model, android=self.android) + try: + self.prepare() + while not stop_event.is_set(): + if adb(self.serial, "get-state", timeout=15).stdout.strip() != "device": + break + if self.dev_cfg.get("skip"): + set_state(self.serial, "skipped in config") + stop_event.wait(300) + continue + try: + self.cycle() + except Exception as e: # keep the farm going; record and retry + self.db.event(self.serial, "error", None, f"worker exception: {e!r}") + stop_event.wait(60) + finally: + self.alive.clear() + set_state(self.serial, "disconnected") + + +# --------------------------------------------------------------------------- iPhone + + +IOS_BUNDLE = "org.scottishtecharmy.soundscape" +PMD3 = os.path.expanduser("~/.local/bin/pymobiledevice3") + + +def pmd3(udid, *args, timeout=120, tunnel=False): + """Runs pymobiledevice3 against one iPhone. Developer services (location, + launch, process list, screenshots) go over the iOS 17+ tunnel, which + --userspace builds in-process, so nothing here needs root.""" + extra = ["--userspace"] if tunnel else [] + return run([PMD3, *args, *extra, "--udid", udid], timeout=timeout) + + +def ios_devices(): + """UDIDs of USB-connected iPhones. --simple skips lockdown, so a phone that + isn't answering can't hang discovery.""" + if not os.path.exists(PMD3): + return [] + res = run([PMD3, "usbmux", "list", "--usb", "--simple"], timeout=20) + try: + return json.loads(res.stdout or "[]") + except json.JSONDecodeError: + return [] + + +class IosWorker(threading.Thread): + """Observes Soundscape on an iPhone. Maestro can't drive a physical iPhone, + so there is no UI testing: the app (installed and upgraded by TestFlight) + is kept running on a replayed GPX walk while the farm records upgrades, + restarts, crash and hang reports, battery and a Soundscape syslog.""" + + def __init__(self, udid, cfg, db): + super().__init__(daemon=True, name=udid) + self.udid, self.cfg, self.db = udid, cfg, db + self.dev_cfg = cfg["devices"].get(udid, {}) + self.dir = os.path.join(RUNS_DIR, udid) + self.player = self.syslog = None + self.syslog_day = None + self.version = self.pid = None + info = {} + res = run([PMD3, "usbmux", "list", "--usb"], timeout=60) + try: + info = next(d for d in json.loads(res.stdout) if d.get("UniqueDeviceID") == udid) + except Exception: + pass + self.model = f"{info.get('DeviceName', 'iPhone')} ({info.get('ProductType', '?')})" + self.os = f"iOS {info.get('ProductVersion', '?')}" + + # ----- background processes + + def write_walk(self): + """One there-and-back of the GPX track as a timestamped GPX at 1 Hz, + which `simulate-location play` replays in real time. A moving track + matters: the app ignored a single fixed simulated point.""" + track = Track(self.dev_cfg.get("gpx", self.cfg["default_gpx"]), + self.cfg["default_location"], self.cfg["replay_speed_mps"]) + t0 = dt.datetime.now(dt.timezone.utc) + pts = [] + for i in range(track.round_trip_seconds()): + lat, lon = track.position(i) + stamp = (t0 + dt.timedelta(seconds=i)).strftime("%Y-%m-%dT%H:%M:%SZ") + pts.append(f'') + path = os.path.join(self.dir, "walk.gpx") + with open(path, "w") as f: + f.write('' + f'{"".join(pts)}') + return path + + def keep_processes_running(self): + if self.player is None or self.player.poll() is not None: + with open(os.path.join(self.dir, "player.log"), "w") as out: + self.player = subprocess.Popen( + [PMD3, "developer", "dvt", "simulate-location", "play", "--userspace", + "--udid", self.udid, self.write_walk()], + stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT) + day = dt.date.today().strftime("%Y%m%d") + if self.syslog_day != day or self.syslog is None or self.syslog.poll() is not None: + if self.syslog and self.syslog.poll() is None: + self.syslog.terminate() + self.syslog = subprocess.Popen( + # DEBUG is mostly CoreMotion sensor fusion: ~1.7 GB a day. + [PMD3, "syslog", "live", "--udid", self.udid, "-pn", "Soundscape", "--no-debug", + "-o", os.path.join(self.dir, f"syslog-{day}.txt")], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.syslog_day = day + + def stop_processes(self): + for p in (self.player, self.syslog): + if p and p.poll() is None: + p.terminate() + pmd3(self.udid, "developer", "dvt", "simulate-location", "clear", tunnel=True, timeout=60) + + # ----- one observation + + def installed_version(self): + res = pmd3(self.udid, "apps", "list", timeout=120) + try: + app = json.loads(res.stdout)[IOS_BUNDLE] + return f"{app['CFBundleShortVersionString']} ({app['CFBundleVersion']})" + except Exception: + return None + + def app_pid(self): + res = pmd3(self.udid, "developer", "dvt", "process-id-for-bundle-id", IOS_BUNDLE, + tunnel=True, timeout=60) + m = re.search(r"^\s*(\d+)\s*$", res.stdout, re.M) + return int(m.group(1)) if m and int(m.group(1)) > 0 else None + + def new_reports(self, obs_dir): + """Pulls Soundscape crash/hang reports not seen before. Returns + (kind, name, text); bug_type 309 is a crash, anything else (hangs, + resource reports) is counted as an ANR.""" + res = pmd3(self.udid, "crash", "ls", timeout=120) + names = [n.strip().strip('",') for n in res.stdout.splitlines()] + names = [n for n in names if re.search(r"(^|/)Soundscape[-_.]", n)] + seen = {r["detail"] for r in self.db.query( + "SELECT detail FROM events WHERE serial=? AND kind='ios-report'", (self.udid,))} + found = [] + for name in names: + if name in seen: + continue + base = os.path.basename(name) + os.makedirs(obs_dir, exist_ok=True) + pmd3(self.udid, "crash", "pull", obs_dir, "--match", re.escape(base), timeout=300) + text = "" + for path in glob.glob(os.path.join(obs_dir, "**", base), recursive=True): + with open(path, errors="replace") as f: + text = f.read() + m = re.search(r'"bug_type"\s*:\s*"?(\d+)', text[:2000]) + kind = "crash" if m and m.group(1) == "309" else "anr" + self.db.execute("INSERT INTO events(serial, ts, kind, version_code, detail) " + "VALUES(?,?,?,?,?)", (self.udid, time.time(), "ios-report", None, name)) + found.append((kind, base, f"{base} (bug_type {m.group(1) if m else '?'})\n{text[:4000]}")) + return found + + def battery(self): + res = pmd3(self.udid, "diagnostics", "battery", "single", timeout=60) + level = re.search(r'"CurrentCapacity":\s*(\d+)', res.stdout) + temp = re.search(r'"Temperature":\s*(\d+)', res.stdout) + return (int(level.group(1)) if level else None, + int(temp.group(1)) / 100 if temp else None) + + def observe(self): + u = self.udid + started = time.time() + obs_dir = os.path.join(self.dir, dt.datetime.now().strftime("%Y%m%d-%H%M%S")) + set_state(u, "checking") + # Needed after every reboot; succeeds immediately when already mounted. + mount = pmd3(u, "mounter", "auto-mount", tunnel=True, timeout=300) + if mount.returncode != 0: + self.db.event(u, "error", None, "developer disk image mount failed (is Developer " + "Mode on?)\n" + (mount.stdout + mount.stderr)[-1500:]) + self.keep_processes_running() + + version = self.installed_version() + if version and self.version and version != self.version: + self.db.event(u, "upgrade", None, f"TestFlight upgrade {self.version} -> {version}") + self.version = version or self.version + + problems = self.new_reports(obs_dir) + for kind, name, text in problems: + self.db.event(u, kind, None, text) + pid = self.app_pid() + if pid is None: + if version: + self.db.event(u, "relaunch", None, "Soundscape wasn't running; relaunching it") + pmd3(u, "developer", "dvt", "launch", "--no-kill-existing", IOS_BUNDLE, + tunnel=True, timeout=60) + pid = self.app_pid() + else: + self.db.event(u, "error", None, "Soundscape isn't installed (TestFlight)") + elif self.pid and pid != self.pid and not problems: + self.db.event(u, "restart", None, f"Soundscape restarted (pid {self.pid} -> {pid}) " + "with no crash report: killed by iOS or the user") + self.pid = pid + + level, temp = battery = self.battery() + crashes = sum(1 for k, _, _ in problems if k == "crash") + anrs = len(problems) - crashes + status = "pass" if pid and not problems else "fail" + self.db.execute( + "INSERT INTO cycles(serial, model, android, version_code, started, ended, status, " + "flows_passed, flows_failed, crashes, anrs, battery_level, battery_temp, dir) " + "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + (u, self.model, self.os, None, started, time.time(), status, 0, 0, crashes, anrs, + level, temp, obs_dir)) + set_state(u, f"observing Soundscape {version or '?'} (pid {pid or 'none'}); " + f"{crashes} crashes, {anrs} hang/other reports", + model=self.model, android=self.os, app=version) + + def run(self): + os.makedirs(self.dir, exist_ok=True) + set_state(self.udid, "preparing", model=self.model, android=self.os) + try: + while not stop_event.is_set(): + if self.udid not in ios_devices(): + break + try: + self.observe() + except Exception as e: # keep observing; record and retry + self.db.event(self.udid, "error", None, f"iPhone worker exception: {e!r}") + stop_event.wait(self.cfg["ios_poll_minutes"] * 60) + finally: + self.stop_processes() + set_state(self.udid, "disconnected") + + +# --------------------------------------------------------------------------- report + + +def fmt_time(ts): + return dt.datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M") if ts else "" + + +def write_report(db): + e = html.escape + day_ago = time.time() - 86400 + versions = {b["version_code"]: b["version_name"] for b in db.builds()} + rows = [] + with state_lock: + states = {k: dict(v) for k, v in device_state.items()} + for serial, st in sorted(states.items()): + last = db.query("SELECT * FROM cycles WHERE serial=? AND ended IS NOT NULL " + "ORDER BY id DESC LIMIT 1", (serial,)) + day = db.query("SELECT COUNT(*) n, SUM(status='pass') ok, SUM(crashes) c, SUM(anrs) a " + "FROM cycles WHERE serial=? AND started>? AND ended IS NOT NULL", + (serial, day_ago))[0] + last = last[0] if last else None + ver = last["version_code"] if last else None + rows.append(f"""{e(serial)}{e(st.get('model', ''))} + {e(st.get('android', ''))}{e(versions.get(ver, str(ver or st.get('app', ''))))} + {e(st['state'])}
since {fmt_time(st['since'])}
+ {e(last['status']) if last else ''} + {day['ok'] or 0}/{day['n']}{day['c'] or 0}{day['a'] or 0} + {last['pss_kb'] // 1024 if last and last['pss_kb'] else ''} MB + {last['battery_level'] if last else ''}% / {last['battery_temp'] if last else ''}°C""") + failures = "".join( + f"{fmt_time(r['ts'])}{e(r['serial'])}{e(r['kind'])}" + f"{e(r['name'])}files" + f"
{e((r['detail'] or '')[-600:])}
" + for r in db.query("SELECT * FROM results WHERE status!='pass' ORDER BY id DESC LIMIT 50")) + events = "".join( + f"{fmt_time(r['ts'])}{e(r['serial'] or '')}{e(r['kind'])}" + f"{e(versions.get(r['version_code'], str(r['version_code'] or '')))}" + f"
{e((r['detail'] or '')[:600])}
" + for r in db.query("SELECT * FROM events ORDER BY id DESC LIMIT 50")) + builds = ", ".join(f"{e(b['version_name'])} ({b['version_code']})" for b in db.builds()) + page = f""" + +Soundscape Test Farm +

Soundscape test farm

Updated {fmt_time(time.time())}. Cached releases: {builds or 'none yet'}

+

Devices

+ +{''.join(rows)}
SerialModelOSAppNowLast cyclePassed 24hCrashes 24hANRs 24hPSSBattery
+

Recent failures

+{failures}
WhenDeviceKindNameDetail
+

Events

+{events}
WhenDeviceKindAppDetail
""" + tmp = REPORT_PATH + ".tmp" + with open(tmp, "w") as f: + f.write(page) + os.replace(tmp, REPORT_PATH) + + +def prune_runs(keep_days): + cutoff = time.time() - keep_days * 86400 + # Per-cycle folders (named by timestamp) and iPhone syslogs; not the + # per-phone Maestro home or the iPhone's walk and player log. + for path in (glob.glob(os.path.join(RUNS_DIR, "*", "[0-9]*")) + + glob.glob(os.path.join(RUNS_DIR, "*", "syslog-*"))): + if os.path.getmtime(path) < cutoff: + subprocess.run(["rm", "-rf", path]) + + +# --------------------------------------------------------------------------- main + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--config", default=os.path.join(FARM_DIR, "farm.toml")) + args = parser.parse_args() + cfg = load_config(args.config) + os.makedirs(APK_DIR, exist_ok=True) + os.makedirs(RUNS_DIR, exist_ok=True) + db = Db(DB_PATH) + + def handle_signal(*_): + log(None, "stopping...") + stop_event.set() + signal.signal(signal.SIGINT, handle_signal) + signal.signal(signal.SIGTERM, handle_signal) + + build_helper() + os.makedirs(EXTRACT_DIR, exist_ok=True) + sync_extract(cfg) # before any phone starts, so the first cycle has it + threading.Thread(target=release_watcher, args=(cfg, db), daemon=True).start() + usb_proxy.start() # network over USB for in-app map downloads + workers = {} + last_prune = 0 + while not stop_event.is_set(): + for serial in connected_devices(cfg["include_emulators"]): + if serial not in workers or not workers[serial].is_alive(): + workers[serial] = Worker(serial, cfg, db) + workers[serial].start() + for udid in ios_devices() if cfg["include_ios"] else []: + if udid not in workers or not workers[udid].is_alive(): + workers[udid] = IosWorker(udid, cfg, db) + workers[udid].start() + write_report(db) + if time.time() - last_prune > 86400: + prune_runs(cfg["keep_days"]) + last_prune = time.time() + stop_event.wait(30) + + stop_children() + for serial, worker in workers.items(): + if isinstance(worker, IosWorker): + worker.stop_processes() + else: + disable_mock_location(serial) + write_report(db) + + +if __name__ == "__main__": + main() diff --git a/test-farm/farm.toml.example b/test-farm/farm.toml.example new file mode 100644 index 000000000..7bc7991f2 --- /dev/null +++ b/test-farm/farm.toml.example @@ -0,0 +1,47 @@ +# Copy to farm.toml and edit. Every setting is optional; these are the defaults +# except where noted. Paths are relative to this file. + +# Where release builds come from: the newest two successful runs of this +# workflow, using the APK that goes to Play. +release_repo = "Scottish-Tech-Army/Soundscape-Android" +release_workflow = "build-app.yaml" +release_artifact = "release-apk" +poll_minutes = 15 + +monkey_events = 5000 # several minutes; not every event waits the throttle +monkey_throttle_ms = 250 +roam_rounds = 2 # rounds of flows/roam/ (shuffled) per cycle +flow_timeout_minutes = 15 +keep_days = 14 # per-cycle logs/screenshots under runs/ + +# Run one Maestro flow at a time across all phones, if parallel runs collide. +serialize_maestro = false + +# If a phone has a Soundscape the farm didn't install (Play Store, debug build), +# uninstall it, wiping its data. Off by default: the phone is reported as blocked. +allow_reinstall = false + +include_emulators = false +include_ios = true # observe USB iPhones via pymobiledevice3 (see README) +ios_poll_minutes = 10 +max_battery_temp_c = 42 # pause the phone for 10 minutes above this + +# Mock location: walk this GPX track at a constant speed, turning round at each +# end. With no track, hold default_location (lat, lon). +replay_speed_mps = 1.4 +default_location = [55.9474, -4.3058] +default_gpx = "../local-pixel.gpx" # (the default is no track) +# Held while the Maestro suite runs; the flows assert distance order. +suite_location = [55.955360, -3.223538] + +# Offline map pushed to every phone, by manifest name; must cover +# suite_location and the GPX track. URL defaults to local.properties' +# extractProviderUrl. +extract_name = "Glasgow" +# extract_provider_url = "" + +# Per-phone overrides, keyed by `adb devices` serial. +# [devices."R5CW12345AB"] +# gpx = "../local-cubot.gpx" +# skip = true +# show_map = false # keep the map off (low-memory phones); default true diff --git a/test-farm/flows/DownloadExtract.yaml b/test-farm/flows/DownloadExtract.yaml new file mode 100644 index 000000000..abd12a5b1 --- /dev/null +++ b/test-farm/flows/DownloadExtract.yaml @@ -0,0 +1,51 @@ +appId: org.scottishtecharmy.soundscape +# Downloads the offline map ${EXTRACT} from Menu > Offline maps. The farm runs +# this on phones where it can't adb push the extract (the Cubot J10's shell +# can't write to other apps' Android/data), with the phone's HTTP proxy set to +# the farm's usb_proxy over adb reverse, as Wi-Fi is off. The extract's name +# is map data rather than UI text, so it's matched by name. +--- +- runFlow: LaunchHome.yaml +- tapOn: + id: topBarMenu +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: menuOfflineMaps +# The nearby extracts come from the manifest, which needs the network. +- extendedWaitUntil: + visible: + text: ${EXTRACT} + timeout: 60000 +# Onboarding's clearState doesn't always remove a downloaded extract (it +# survived on the Cubot), so it may already be in "Maps already downloaded", +# which is listed first. Its details then offer Delete rather than Download. +- tapOn: + text: ${EXTRACT} + index: 0 +- runFlow: + when: + visible: + id: offlineMapDownload + commands: + - tapOn: + id: offlineMapDownload + # The progress screen shows a percentage until the download finishes + # and the list comes back, now with the extract in the downloaded + # section. 177 MB for Glasgow; minutes to half an hour over USB. + - extendedWaitUntil: + visible: + text: ".*%" + timeout: 30000 + optional: true + - extendedWaitUntil: + notVisible: + text: ".*%" + timeout: 3600000 + - tapOn: + text: ${EXTRACT} + index: 0 +- assertVisible: + id: offlineMapDelete +- back +- back diff --git a/test-farm/flows/FarmSettings.yaml b/test-farm/flows/FarmSettings.yaml new file mode 100644 index 000000000..229b82f78 --- /dev/null +++ b/test-farm/flows/FarmSettings.yaml @@ -0,0 +1,75 @@ +appId: org.scottishtecharmy.soundscape +# Run by the test farm after Onboarding (which wipes the app's settings): +# - Settings > Search > Search mode: Offline, so searches use the offline map +# extract the farm pushes instead of the phones' unreliable Wi-Fi. +# +# The map is turned on separately (ShowMap.yaml) once the set-up suite is done, +# because the suite's flows expect the map off, as in CI. +# +# Releases before the Settings testTags (settingsSection*, settingsSearchMode, +# settingsOption-*) are driven by their English labels +# instead, so the farm phones must be set to English until every release in +# apks/ has the tags. Then the label fallbacks can go. +--- +- runFlow: LaunchHome.yaml +- tapOn: + id: topBarMenu +# Second in the drawer, so never off-screen; SwipeAndTap's nudge scrolled it +# away on the slow Cubot. +- tapOn: + id: menuSettings +- runFlow: + file: ../../maestro/Wait.yaml + env: + timeout: 2000 +# The first section header is always on screen, so it says which kind of +# release this is. +- evalScript: ${output.tagged = false} +- runFlow: + when: + visible: + id: settingsSectionCallouts + commands: + - evalScript: ${output.tagged = true} + +# Search mode: Offline. +- runFlow: + when: + true: ${output.tagged} + commands: + - scrollUntilVisible: + element: + id: settingsSectionSearch + direction: DOWN + - tapOn: + id: settingsSectionSearch + - scrollUntilVisible: + element: + id: settingsSearchMode + direction: DOWN + - tapOn: + id: settingsSearchMode + - tapOn: + id: settingsOption-Offline +- runFlow: + when: + true: ${!output.tagged} + commands: + - scrollUntilVisible: + element: + text: "Search" + direction: DOWN + - tapOn: + text: "Search" + - scrollUntilVisible: + element: + text: "Search mode" + direction: DOWN + - tapOn: + text: "Search mode" + - tapOn: + text: "Offline" + # The summary under "Search mode" now names the chosen option. + - assertVisible: + text: ".*Offline.*" +- back diff --git a/test-farm/flows/LaunchHome.yaml b/test-farm/flows/LaunchHome.yaml new file mode 100644 index 000000000..5cbf0564b --- /dev/null +++ b/test-farm/flows/LaunchHome.yaml @@ -0,0 +1,47 @@ +appId: org.scottishtecharmy.soundscape +# Brings the app to the front and gets it back to Home, dismissing the release +# notes dialog if it shows. It appears on the first launch of a new release +# and can turn up after Onboarding; while it's up, Home's anchor isn't visible +# to Maestro. +# +# The app isn't restarted (stopApp: false), so one process runs through all +# the roam rounds and soaks for hours. The previous flow may have left it on +# any screen, or with the search or a dialog open, so it backs out to Home. +# +# Releases before the dialog's testTags were exposed (a dialog is its own +# window, outside the theme's testTagsAsResourceId) need its English button +# text. +--- +- launchApp: + clearState: false + stopApp: false +- runFlow: + file: ../../maestro/Wait.yaml + env: + timeout: 3000 +- tapOn: + id: newReleaseDialogDismiss + optional: true +- tapOn: + text: "Close Release Notes" + optional: true +- repeat: + while: + notVisible: + id: mainSearchBarCollapsed + times: 6 + commands: + - back + - runFlow: + file: ../../maestro/Wait.yaml + env: + timeout: 1000 +# Backing out of Home closes the activity (the process and service keep +# running), so bring it back if that happened. +- launchApp: + clearState: false + stopApp: false +- runFlow: + file: ../../maestro/WaitForScreen.yaml + env: + screen: Home diff --git a/test-farm/flows/PostUpgrade.yaml b/test-farm/flows/PostUpgrade.yaml new file mode 100644 index 000000000..ab8d81278 --- /dev/null +++ b/test-farm/flows/PostUpgrade.yaml @@ -0,0 +1,26 @@ +appId: org.scottishtecharmy.soundscape +# Run by the test farm straight after `adb install -r` of a newer release over a +# phone that has just finished the suite (see test-farm/farm.py). Checks that +# the upgraded app starts on the old data, and that the markers and the route +# the suite created survived the upgrade. +--- +# The release dialog shows on the first launch of a new major.minor version; +# LaunchHome dismisses it. +- runFlow: LaunchHome.yaml +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homeMarkers +# Markers.yaml's C replaced A (same location), so B and C are what's left. +- assertVisible: + id: "LocationItem-B-.*" +- assertVisible: + id: "LocationItem-C-.*" + +- runFlow: LaunchHome.yam +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homeRoutes +- assertVisible: + text: "Test Route.*" diff --git a/test-farm/flows/Restore.yaml b/test-farm/flows/Restore.yaml new file mode 100644 index 000000000..ca04995ce --- /dev/null +++ b/test-farm/flows/Restore.yaml @@ -0,0 +1,125 @@ +appId: org.scottishtecharmy.soundscape +# Run by the test farm after the monkey soak, held at suite_location, without +# restarting the app. Puts back what the roaming flows and the next upgrade's +# PostUpgrade.yaml rely on and the monkey may have changed: +# - search mode Offline (FarmSettings.yaml); +# - markers B and C and "Test Route" (B -> C), recreated the way the suite +# made them if they've gone; +# - the map as the farm wants it for roaming (SHOW_MAP, the phone's show_map). +# Fails if they can't be restored, and the farm then sets the phone up again. +# +# Recreating is idempotent: B is saved from the nearest place and C at the +# held location, the same spots the suite used, and saving a marker where one +# already is (say one the monkey renamed) updates that marker. +--- +- runFlow: FarmSettings.yaml +# The markers are made with the map off, as in the suite: on a small screen +# the map pushes controls under the bottom bar. +- runFlow: + file: ShowMap.yaml + env: + SHOW: "false" + +# Marker B: the first place nearby. +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homeMarkers +- evalScript: ${output.haveB = false} +- runFlow: + when: + visible: + id: "LocationItem-B-.*" + commands: + - evalScript: ${output.haveB = true} +- runFlow: LaunchHome.yaml +- runFlow: + when: + true: ${!output.haveB} + commands: + - runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homePlacesNearby + direction: down + - tapOn: + id: placesNearby-0 + - tapOn: + id: placesNearby-0 + - tapOn: + id: locationDetailsSaveAsMarker + optional: true + - tapOn: + id: locationDetailsEditMarker + optional: true + - tapOn: + id: name-clearTextField + optional: true + - tapOn: + id: markerName + - inputText: "B" + - tapOn: + id: flexibleAppBarRight + - runFlow: LaunchHome.yaml + +# Marker C: the current (held) location, from the Markers screen's New button. +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homeMarkers +- runFlow: + when: + notVisible: + id: "LocationItem-C-.*" + commands: + - tapOn: + id: appBarRight + - tapOn: + id: addMarkerCurrentLocation + - tapOn: + id: name-clearTextField + optional: true + - tapOn: + id: markerName + - inputText: "C" + - tapOn: + id: flexibleAppBarRight +- assertVisible: + id: "LocationItem-B-.*" +- assertVisible: + id: "LocationItem-C-.*" +- runFlow: LaunchHome.yaml + +# The route, B then C. +- runFlow: + file: ../../maestro/SwipeAndTap.yaml + env: + id: homeRoutes +- runFlow: + when: + notVisible: + text: "Test Route.*" + commands: + - tapOn: + id: appBarRight + - tapOn: + id: routeName + - inputText: "Test Route" + - pressKey: Enter + - tapOn: + id: addWaypointsButton + - tapOn: + id: "LocationItem-B-.*" + - tapOn: + id: "LocationItem-C-.*" + - tapOn: + id: flexibleAppBarRight + - tapOn: + id: flexibleAppBarRight +- assertVisible: + text: "Test Route.*" + +- runFlow: + file: ShowMap.yaml + env: + SHOW: ${SHOW_MAP} diff --git a/test-farm/flows/ShowMap.yaml b/test-farm/flows/ShowMap.yaml new file mode 100644 index 000000000..344b76d48 --- /dev/null +++ b/test-farm/flows/ShowMap.yaml @@ -0,0 +1,104 @@ +appId: org.scottishtecharmy.soundscape +# Sets Settings > Accessibility > Show maps on screen to SHOW ("true" or +# "false"), or if that isn't given to SHOW_MAP, the phone's show_map, which +# the farm passes to every flow (default "true"). Run by the +# test farm after the set-up suite, to turn the map on for roaming, and by +# Restore.yaml. Onboarding.yaml turns the +# map off and the suite's flows expect it off (on a small screen the map +# pushes controls under the bottom bar), but the farm wants the map drawing +# on real phones while it roams. +# +# Releases before the settingsSectionAccessibility and settingsShowMap +# testTags are driven by their English labels. +--- +- runFlow: LaunchHome.yaml +# Whether the map is on is read from Home (its full screen button only shows +# with the map), because Maestro's checked: matches the switch's label as well +# as the switch. +- evalScript: ${output.mapOn = false} +- evalScript: | + ${ + const show = typeof SHOW !== 'undefined' ? SHOW + : typeof SHOW_MAP !== 'undefined' ? SHOW_MAP : 'true'; + output.want = show !== 'false'; + } +- runFlow: + when: + visible: + id: homeFullScreenMap + commands: + - evalScript: ${output.mapOn = true} +- tapOn: + id: topBarMenu +# Second in the drawer, so never off-screen; SwipeAndTap's nudge scrolled it +# away on the slow Cubot. +- tapOn: + id: menuSettings +- runFlow: + file: ../../maestro/Wait.yaml + env: + timeout: 2000 +- evalScript: ${output.tagged = false} +- runFlow: + when: + visible: + id: settingsSectionCallouts + commands: + - evalScript: ${output.tagged = true} +- runFlow: + when: + true: ${output.tagged} + commands: + - scrollUntilVisible: + element: + id: settingsSectionAccessibility + direction: DOWN + - tapOn: + id: settingsSectionAccessibility + - scrollUntilVisible: + element: + id: settingsShowMap + direction: DOWN + - runFlow: + when: + true: ${output.mapOn !== output.want} + commands: + - tapOn: + id: settingsShowMap +- runFlow: + when: + true: ${!output.tagged} + commands: + - scrollUntilVisible: + element: + text: "Accessibility" + direction: DOWN + - tapOn: + text: "Accessibility" + - scrollUntilVisible: + element: + text: "Show maps on screen" + direction: DOWN + - runFlow: + when: + true: ${output.mapOn !== output.want} + commands: + - tapOn: + text: "Show maps on screen" +- back +- runFlow: + file: ../../maestro/WaitForScreen.yaml + env: + screen: Home +- runFlow: + when: + true: ${output.want} + commands: + - assertVisible: + id: homeFullScreenMap +- runFlow: + when: + true: ${!output.want} + commands: + - assertNotVisible: + id: homeFullScreenMap diff --git a/test-farm/flows/StopBeacon.yaml b/test-farm/flows/StopBeacon.yaml new file mode 100644 index 000000000..34223f946 --- /dev/null +++ b/test-farm/flows/StopBeacon.yaml @@ -0,0 +1,17 @@ +appId: org.scottishtecharmy.soundscape +# Stops the audio beacon if one is still playing. A beacon on a nearby place +# ends by itself when the walk reaches it, so there may be nothing to stop. +# The beacon card can sit below the fold, so look further down the home +# screen before deciding it has gone. +--- +- runFlow: + when: + notVisible: + id: routeStop + commands: + - swipe: + start: 50%, 70% + end: 50%, 40% +- tapOn: + id: routeStop + optional: true diff --git a/test-farm/flows/roam/Callouts.yaml b/test-farm/flows/roam/Callouts.yaml new file mode 100644 index 000000000..72bc3d245 --- /dev/null +++ b/test-farm/flows/roam/Callouts.yaml @@ -0,0 +1,49 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow (see test-farm/README.md): the home screen's callout buttons and +# the full screen map, while the farm walks the phone along its GPX track. The +# waits give the callouts time to play and the walk time to move on, so the +# automatic callouts run in between. +--- +- runFlow: ../LaunchHome.yaml + +- tapOn: + id: homeMyLocation +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 15000 +- tapOn: + id: homeAroundMe +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 30000 +- tapOn: + id: homeAheadOfMe +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 30000 +- tapOn: + id: homeNearbyMarkers +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 10000 + +# Watch the map follow the walk for a while, then put the controls back. +# Phones with show_map = false (the Cubot) have no map. +- runFlow: + when: + true: ${SHOW_MAP !== 'false'} + commands: + - tapOn: + id: homeFullScreenMap + - runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 60000 + - tapOn: + id: homeFullScreenMap +- assertVisible: + id: homeMyLocation diff --git a/test-farm/flows/roam/MarkerBrowse.yaml b/test-farm/flows/roam/MarkerBrowse.yaml new file mode 100644 index 000000000..5bc677d05 --- /dev/null +++ b/test-farm/flows/roam/MarkerBrowse.yaml @@ -0,0 +1,42 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow: flips the Markers list's sort order and opens each of the +# suite's markers (B and C) as the walk moves away from them. Changes nothing. +--- +- runFlow: ../LaunchHome.yaml + +- runFlow: + file: ../../../maestro/SwipeAndTap.yaml + env: + id: homeMarkers +- tapOn: + id: SortOption +- tapOn: + id: SortOption + +- tapOn: + id: "LocationItem-B-.*" +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: LocationDetails +- back +- tapOn: + id: "LocationItem-C-.*" +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: LocationDetails +- runFlow: + when: + true: ${SHOW_MAP !== 'false'} + commands: + - tapOn: + id: locationDetailsFullScreenMapFab + - runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 5000 + - tapOn: + id: locationDetailsFullScreenMapFab +- back +- back diff --git a/test-farm/flows/roam/Menus.yaml b/test-farm/flows/roam/Menus.yaml new file mode 100644 index 000000000..7977be0df --- /dev/null +++ b/test-farm/flows/roam/Menus.yaml @@ -0,0 +1,83 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow: looks around the menu's screens without changing anything +# (help topics, offline maps, settings, about) and puts the app to sleep and +# wakes it again. Uses testTags only, so it doesn't depend on the language. +--- +- runFlow: ../LaunchHome.yaml + +# Help: open three topics. +- tapOn: + id: topBarMenu +- tapOn: + id: menuHelpAndTutorials +- tapOn: + id: "helpScreenTopic.*" + index: 0 +- back +- tapOn: + id: "helpScreenTopic.*" + index: 2 +- back +- tapOn: + id: "helpScreenTopic.*" + index: 4 +- back +- back + +# Offline maps: the list of extracts covering the current location. +- tapOn: + id: topBarMenu +- tapOn: + id: menuOfflineMaps +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 5000 +- back + +# Settings: scroll to the bottom and back out without touching anything. +- tapOn: + id: topBarMenu +- tapOn: + id: menuSettings +# Settings has no testTags (WaitForScreen's Settings anchor no longer exists). +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 3000 +- repeat: + times: 4 + commands: + - swipe: + start: 50%, 80% + end: 50%, 30% +- back + +- tapOn: + id: topBarMenu +- runFlow: + file: ../../../maestro/SwipeAndTap.yaml + env: + id: menuAboutSoundscape +- back +# Some screens come back to the open menu drawer rather than Home. +- tapOn: + id: menuDrawerBack + optional: true + +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: Home +- tapOn: + id: topBarSleep +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 30000 +- tapOn: + id: sleepWakeUpNow +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: Home diff --git a/test-farm/flows/roam/NearbyBeacon.yaml b/test-farm/flows/roam/NearbyBeacon.yaml new file mode 100644 index 000000000..f7df626cd --- /dev/null +++ b/test-farm/flows/roam/NearbyBeacon.yaml @@ -0,0 +1,47 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow: picks a place from Places Nearby, looks at its details and map, +# then walks with an audio beacon on it for a minute before stopping it. +# The walk reorders the list, so the flow doesn't care which place it gets. +--- +- runFlow: ../LaunchHome.yaml + +- tapOn: + id: homePlacesNearby +# The first folder is "All", then the places in distance order. +- tapOn: + id: placesNearby-0 +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: PlacesNearby +- tapOn: + id: placesNearby-0 +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: LocationDetails + +- runFlow: + when: + true: ${SHOW_MAP !== 'false'} + commands: + - tapOn: + id: locationDetailsFullScreenMapFab + - runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 5000 + - tapOn: + id: locationDetailsFullScreenMapFab + +- tapOn: + id: locationDetailsStartBeacon +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: Home +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 60000 +- runFlow: ../StopBeacon.yaml diff --git a/test-farm/flows/roam/RouteWalk.yaml b/test-farm/flows/roam/RouteWalk.yaml new file mode 100644 index 000000000..b56795b55 --- /dev/null +++ b/test-farm/flows/roam/RouteWalk.yaml @@ -0,0 +1,33 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow: plays the suite's "Test Route" for two minutes of the walk, +# then stops it. The route's markers are at the suite point, so the beacon is +# a long way off, which is fine for exercising route playback. +--- +- runFlow: ../LaunchHome.yaml + +- runFlow: + file: ../../../maestro/SwipeAndTap.yaml + env: + id: homeRoutes +- tapOn: + id: "LocationItem-Test Route-.*" +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: RouteDetails +- tapOn: + id: routeDetailsStartButton +- runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: Home +- assertVisible: + id: routeBeaconTitle +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 120000 +- runFlow: + file: ../../../maestro/SwipeAndTap.yaml + env: + id: routeStop diff --git a/test-farm/flows/roam/Search.yaml b/test-farm/flows/roam/Search.yaml new file mode 100644 index 000000000..97060cc52 --- /dev/null +++ b/test-farm/flows/roam/Search.yaml @@ -0,0 +1,82 @@ +appId: org.scottishtecharmy.soundscape +# Roaming flow: an offline search for a random common word, then a beacon on +# the first result for half a minute. Search mode was set to Offline when the +# farm set the phone up, so this searches the pushed extract. +--- +- runFlow: ../LaunchHome.yaml + +- evalScript: | + ${ + const words = ['Station', 'Park', 'Church', 'School', 'Road', 'Library', 'Bank', 'Cafe']; + output.searchWord = words[Math.floor(Math.random() * words.length)]; + } +- tapOn: + id: mainSearchBarCollapsed +# The search field takes focus as it opens. +- inputText: ${output.searchWord} +- pressKey: Enter +# Offline search takes a few seconds, longer on slow phones. +- runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 15000 +# Releases before searchResult-* (the results are in a Dialog, whose testTags +# weren't exposed) need the result found by its text: by a distance in its +# accessibility text, or failing that by the searched word. A road's text is +# just its address, with no distance. Index 1 skips the search field, which +# holds the word too. +- evalScript: ${output.tapped = false} +- runFlow: + when: + visible: + id: searchResult-0 + commands: + - evalScript: ${output.tapped = true} + - tapOn: + id: searchResult-0 +- runFlow: + when: + true: ${!output.tapped} + visible: + text: "(?s).*(metres|km|kilometres|miles|feet).*" + commands: + - evalScript: ${output.tapped = true} + - tapOn: + text: "(?s).*(metres|km|kilometres|miles|feet).*" + index: 0 +- runFlow: + when: + true: ${!output.tapped} + visible: + text: "(?s).*${output.searchWord}.*" + index: 1 + commands: + - evalScript: ${output.tapped = true} + - tapOn: + text: "(?s).*${output.searchWord}.*" + index: 1 +# Nothing called that nearby: the search still ran, so that's not a failure. +- runFlow: + when: + true: ${!output.tapped} + commands: + - back +- runFlow: + when: + true: ${output.tapped} + commands: + - runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: LocationDetails + - tapOn: + id: locationDetailsStartBeacon + - runFlow: + file: ../../../maestro/WaitForScreen.yaml + env: + screen: Home + - runFlow: + file: ../../../maestro/Wait.yaml + env: + timeout: 30000 + - runFlow: ../StopBeacon.yaml diff --git a/test-farm/mock-location/.gitignore b/test-farm/mock-location/.gitignore new file mode 100644 index 000000000..567609b12 --- /dev/null +++ b/test-farm/mock-location/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/test-farm/mock-location/AndroidManifest.xml b/test-farm/mock-location/AndroidManifest.xml new file mode 100644 index 000000000..89847586d --- /dev/null +++ b/test-farm/mock-location/AndroidManifest.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + diff --git a/test-farm/mock-location/build.sh b/test-farm/mock-location/build.sh new file mode 100755 index 000000000..de15713d5 --- /dev/null +++ b/test-farm/mock-location/build.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Builds the test farm's mock location helper (build/farm-location.apk) straight +# from the SDK tools: one Java class, no resources, no Gradle. Signed with the +# local debug key; farm.py installs it on each Android phone. +set -euo pipefail + +cd "$(dirname "$0")" +SDK="${ANDROID_HOME:-$HOME/Android/Sdk}" +BT="$(ls -d "$SDK"/build-tools/* | sort -V | tail -1)" +JAR="$(ls -d "$SDK"/platforms/android-[0-9]* | sort -V | tail -1)/android.jar" +OUT=build +rm -rf "$OUT" && mkdir -p "$OUT/classes" "$OUT/dex" + +javac -nowarn --release 11 -classpath "$JAR" -d "$OUT/classes" \ + $(find src -name '*.java') +"$BT/d8" --min-api 26 --lib "$JAR" --output "$OUT/dex" $(find "$OUT/classes" -name '*.class') +"$BT/aapt2" link -o "$OUT/unsigned.apk" -I "$JAR" --manifest AndroidManifest.xml +(cd "$OUT/dex" && zip -q ../unsigned.apk classes.dex) +"$BT/zipalign" -f 4 "$OUT/unsigned.apk" "$OUT/aligned.apk" +"$BT/apksigner" sign --ks "$HOME/.android/debug.keystore" --ks-pass pass:android \ + --out "$OUT/farm-location.apk" "$OUT/aligned.apk" +echo "built $OUT/farm-location.apk" diff --git a/test-farm/mock-location/src/org/scottishtecharmy/farmlocation/FixReceiver.java b/test-farm/mock-location/src/org/scottishtecharmy/farmlocation/FixReceiver.java new file mode 100644 index 000000000..209cc5ec1 --- /dev/null +++ b/test-farm/mock-location/src/org/scottishtecharmy/farmlocation/FixReceiver.java @@ -0,0 +1,96 @@ +package org.scottishtecharmy.farmlocation; + +import android.app.Activity; +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.location.Criteria; +import android.location.Location; +import android.location.LocationManager; +import android.os.SystemClock; + +/** + * Sets one mock fix on the gps and network test providers, sent by the test farm + * once a second: + * + * am broadcast -n org.scottishtecharmy.farmlocation/.FixReceiver \ + * -a org.scottishtecharmy.farmlocation.FIX \ + * --es lat 55.9474 --es lon -4.3058 --es speed 1.4 --es bearing 92.5 + * + * speed and bearing are optional. Soundscape only takes its direction from travel + * when a fix says the user is moving, so without them it falls back to the + * phone's compass. Action STOP removes the test providers again. + * + * Needs the mock location app-op: appops set android:mock_location allow + */ +public class FixReceiver extends BroadcastReceiver { + private static final String FIX = "org.scottishtecharmy.farmlocation.FIX"; + private static final String STOP = "org.scottishtecharmy.farmlocation.STOP"; + private static final String[] PROVIDERS = { + LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER + }; + + @Override + @SuppressWarnings("deprecation") + public void onReceive(Context context, Intent intent) { + LocationManager lm = context.getSystemService(LocationManager.class); + try { + if (STOP.equals(intent.getAction())) { + for (String p : PROVIDERS) { + try { + lm.removeTestProvider(p); + } catch (IllegalArgumentException notAdded) { + // nothing to remove + } + } + setResult(Activity.RESULT_OK, "stopped", null); + return; + } + if (!FIX.equals(intent.getAction())) { + setResult(Activity.RESULT_CANCELED, "unknown action", null); + return; + } + double lat = Double.parseDouble(intent.getStringExtra("lat")); + double lon = Double.parseDouble(intent.getStringExtra("lon")); + float accuracy = parse(intent.getStringExtra("accuracy"), 5f); + float speed = parse(intent.getStringExtra("speed"), -1f); + float bearing = parse(intent.getStringExtra("bearing"), -1f); + + for (String p : PROVIDERS) { + try { + lm.addTestProvider(p, false, false, false, false, true, true, true, + Criteria.POWER_LOW, Criteria.ACCURACY_FINE); + } catch (IllegalArgumentException alreadyAdded) { + // a provider from an earlier fix + } + lm.setTestProviderEnabled(p, true); + + Location fix = new Location(p); + fix.setLatitude(lat); + fix.setLongitude(lon); + fix.setAltitude(0); + fix.setAccuracy(accuracy); + fix.setVerticalAccuracyMeters(accuracy); + fix.setTime(System.currentTimeMillis()); + fix.setElapsedRealtimeNanos(SystemClock.elapsedRealtimeNanos()); + if (speed >= 0) { + fix.setSpeed(speed); + fix.setSpeedAccuracyMetersPerSecond(0.3f); + } + if (bearing >= 0) { + fix.setBearing(bearing); + fix.setBearingAccuracyDegrees(10f); + } + lm.setTestProviderLocation(p, fix); + } + setResult(Activity.RESULT_OK, "ok", null); + } catch (RuntimeException e) { + // Most likely a SecurityException: the mock location app-op isn't allowed. + setResult(Activity.RESULT_CANCELED, e.getClass().getSimpleName() + ": " + e.getMessage(), null); + } + } + + private static float parse(String value, float fallback) { + return value == null ? fallback : Float.parseFloat(value); + } +} diff --git a/test-farm/test_farm.py b/test-farm/test_farm.py new file mode 100644 index 000000000..1d8f53c42 --- /dev/null +++ b/test-farm/test_farm.py @@ -0,0 +1,216 @@ +"""Checks farm.py without any phones: run-tests runs these on every PR. + + python3 -m unittest discover -s test-farm -v + +They cover what can break without a device: the config, the wiring between +the farm's flows and maestro/, the settings the farm shares with CI's Maestro +run, and the parsers that read adb output. +""" + +import glob +import os +import re +import tempfile +import tomllib +import unittest + +import farm + +MAESTRO_WORKFLOW = os.path.join(farm.REPO_DIR, ".github", "workflows", "run-maestro-tests.yaml") +FLOW_REF_RE = re.compile(r"^\s*(?:-\s*runFlow:|file:)\s*(\S+\.yaml)\s*$", re.M) + + +class FakeShell: + """Stands in for farm.shell: answers each command from `replies`, keyed by a + substring of the command, and records the commands it was given.""" + + def __init__(self, replies): + self.replies = replies + self.commands = [] + + def __call__(self, serial, command, timeout=120): + self.commands.append(command) + for key, reply in self.replies.items(): + if key in command: + return reply + return "" + + +class ShellTestCase(unittest.TestCase): + def fake_shell(self, replies): + fake = FakeShell(replies) + original = farm.shell + farm.shell = fake + self.addCleanup(setattr, farm, "shell", original) + return fake + + +class ConfigTest(unittest.TestCase): + EXAMPLE = os.path.join(farm.FARM_DIR, "farm.toml.example") + + def test_example_loads(self): + cfg = farm.load_config(self.EXAMPLE) + self.assertEqual(cfg["extract_name"], "Glasgow") + + def test_example_only_uses_known_settings(self): + with open(self.EXAMPLE, "rb") as f: + keys = set(tomllib.load(f)) + self.assertEqual(keys - set(farm.DEFAULTS), set()) + + def test_gpx_paths_are_relative_to_the_config(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "farm.toml") + with open(path, "w") as f: + f.write('default_gpx = "a.gpx"\n[devices.X]\ngpx = "../b.gpx"\n') + cfg = farm.load_config(path) + self.assertEqual(cfg["default_gpx"], os.path.join(d, "a.gpx")) + self.assertEqual(cfg["devices"]["X"]["gpx"], os.path.join(os.path.dirname(d), "b.gpx")) + + +class FlowWiringTest(unittest.TestCase): + def test_suite_flows_exist(self): + flows = farm.suite_flows() + self.assertTrue(flows, "no runFlow lines found in maestro/Master.yaml") + for flow in flows: + self.assertTrue(os.path.isfile(os.path.join(farm.MAESTRO_DIR, flow)), flow) + + def test_roam_flows_exist(self): + self.assertTrue(farm.roam_flows()) + + def test_farm_flows_exist(self): + for path in (farm.POST_UPGRADE_FLOW, farm.FARM_SETTINGS_FLOW, farm.DOWNLOAD_EXTRACT_FLOW, + farm.SHOW_MAP_FLOW, farm.RESTORE_FLOW): + self.assertTrue(os.path.isfile(path), path) + + def test_flow_references_resolve(self): + # Roam flows sit a level deeper than the others, so their paths into + # maestro/ need one more "../". + flows = glob.glob(os.path.join(farm.FARM_DIR, "flows", "**", "*.yaml"), recursive=True) + self.assertTrue(flows) + for flow in flows: + with open(flow) as f: + refs = FLOW_REF_RE.findall(f.read()) + for ref in refs: + target = os.path.normpath(os.path.join(os.path.dirname(flow), ref)) + with self.subTest(flow=os.path.relpath(flow, farm.FARM_DIR), ref=ref): + self.assertTrue(os.path.isfile(target)) + + +class CiDriftTest(unittest.TestCase): + """The farm repeats two things from CI's Maestro run, which must agree.""" + + @classmethod + def setUpClass(cls): + with open(MAESTRO_WORKFLOW) as f: + cls.workflow = f.read() + + def test_suite_location_matches_ci_geo_fix(self): + # The suite's flows assert distance-sorted lists, so the farm has to + # hold the phone at the point CI pins the emulator to. + m = re.search(r"adb emu geo fix (-?[\d.]+) (-?[\d.]+)", self.workflow) + self.assertIsNotNone(m) + lon, lat = float(m.group(1)), float(m.group(2)) + self.assertEqual([lat, lon], farm.DEFAULTS["suite_location"]) + + def test_retry_regex_covers_ci_retries(self): + m = re.search(r'grep -Eq "([^"]+)" "maestro_outputs', self.workflow) + self.assertIsNotNone(m) + for message in m.group(1).split("|"): + with self.subTest(message=message): + self.assertRegex(message, farm.MAESTRO_RETRY_RE) + + +class DropboxTest(ShellTestCase): + @staticmethod + def entry(stamp, tag, body): + return f"========================================\n{stamp} {tag} (text, 100 bytes)\n{body}\n" + + def test_collects_our_crashes_and_anrs_since(self): + pkg = farm.PACKAGE + self.fake_shell({ + "data_app_crash": self.entry("2026-10-01 09:00:00", "data_app_crash", f"Process: {pkg}") + + self.entry("2026-10-02 10:00:00", "data_app_crash", f"Process: {pkg}") + + self.entry("2026-10-02 11:00:00", "data_app_crash", "Process: com.other"), + "data_app_anr": self.entry("2026-10-02 12:00:00", "data_app_anr", f"Process: {pkg}"), + }) + found = farm.collect_dropbox("S", "2026-10-02 00:00:00") + self.assertEqual([kind for kind, _ in found], ["crash", "anr"]) + self.assertIn("2026-10-02 10:00:00", found[0][1]) + + def test_asks_for_one_tag_per_call(self): + # dumpsys dropbox only prints entries matching every argument, so + # several tags in one call print nothing. + fake = self.fake_shell({}) + farm.collect_dropbox("S", "2026-10-02 00:00:00") + self.assertEqual(fake.commands, [f"dumpsys dropbox --print {tag}" for tag in farm.DROPBOX_TAGS]) + + +class DeviceReadingsTest(ShellTestCase): + def test_app_uptime(self): + for etime, seconds in (("03:04", 184), ("02:03:04", 7384), ("1-02:03:04", 93784)): + with self.subTest(etime=etime): + self.fake_shell({"pidof": "1234\n", "ps -o": f" {etime}\n"}) + self.assertEqual(farm.app_uptime_s("S"), seconds) + + def test_app_uptime_not_running(self): + self.fake_shell({"pidof": ""}) + self.assertIsNone(farm.app_uptime_s("S")) + + def test_pss(self): + self.fake_shell({"meminfo": " TOTAL PSS: 123456 TOTAL RSS: 200000"}) + self.assertEqual(farm.pss_kb("S"), 123456) + self.fake_shell({"meminfo": " Native Heap 1000\n TOTAL 98765 1234"}) + self.assertEqual(farm.pss_kb("S"), 98765) + + def test_battery(self): + self.fake_shell({"battery": " level: 80\n temperature: 315\n"}) + self.assertEqual(farm.battery("S"), (80, 31.5)) + self.fake_shell({"battery": ""}) + self.assertEqual(farm.battery("S"), (None, None)) + + +class TrackTest(unittest.TestCase): + START = (55.9474, -4.3058) + + def write_gpx(self, points): + f = tempfile.NamedTemporaryFile("w", suffix=".gpx", delete=False) + self.addCleanup(os.unlink, f.name) + pts = "".join(f'' for lat, lon in points) + f.write('' + f"{pts}") + f.close() + return f.name + + def north(self, metres): + return self.START[0] + metres / 111_195, self.START[1] + + def test_jitter_is_dropped(self): + jitter = [self.north(m) for m in (0, 3, 1, 5, 2)] + points = farm.load_track(self.write_gpx(jitter + [self.north(100), self.north(200)])) + self.assertEqual(len(points), 3) + + def test_too_short_track_is_rejected(self): + with self.assertRaises(ValueError): + farm.load_track(self.write_gpx([self.north(0), self.north(5)])) + + def test_walks_to_the_end_and_back(self): + track = farm.Track(self.write_gpx([self.north(0), self.north(100), self.north(200)]), + None, speed=2.0) + total = track.cumulative[-1] + self.assertAlmostEqual(total, 200, delta=0.5) + self.assertEqual(track.position(0), self.START) + end = track.position(total / 2.0) + self.assertAlmostEqual(end[0], self.north(200)[0], places=6) + # A quarter of the way back is three quarters of the way along. + back = track.position(total * 1.25 / 2.0) + self.assertAlmostEqual(back[0], self.north(150)[0], places=5) + self.assertAlmostEqual(track.position(total / 1.0)[0], self.START[0], places=6) + + def test_fixed_point(self): + track = farm.Track("", self.START, speed=1.4) + for t in (0, 10, 1000): + self.assertEqual(track.position(t), self.START) + + +if __name__ == "__main__": + unittest.main() diff --git a/test-farm/usb_proxy.py b/test-farm/usb_proxy.py new file mode 100644 index 000000000..a17d543e2 --- /dev/null +++ b/test-farm/usb_proxy.py @@ -0,0 +1,98 @@ +"""A minimal HTTP proxy for giving farm phones network access over USB. + +The farm runs it on this PC, `adb reverse`s its port onto a phone and sets the +phone's global HTTP proxy to 127.0.0.1:, so apps that honour the system +proxy (OkHttp does) reach the internet through the PC with Wi-Fi off. It +handles CONNECT (HTTPS tunnels, which is all the app uses) and plain HTTP. +""" + +import select +import socket +import socketserver +import threading +import time +from urllib.parse import urlsplit + +PORT = 8118 + + +class _Handler(socketserver.StreamRequestHandler): + timeout = 60 + + def handle(self): + line = self.rfile.readline(65537).decode("latin-1") + try: + method, target, version = line.split() + except ValueError: + return + headers = [] + while True: + h = self.rfile.readline(65537) + if h in (b"\r\n", b"\n", b""): + break + headers.append(h) + if method == "CONNECT": + host, _, port = target.rpartition(":") + try: + upstream = socket.create_connection((host, int(port)), timeout=30) + except OSError: + self.wfile.write(b"HTTP/1.1 502 Bad Gateway\r\n\r\n") + return + self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + self.wfile.flush() + leftover = self.rfile.read1(65536) if self.rfile.peek(1) else b"" + if leftover: + upstream.sendall(leftover) + else: + url = urlsplit(target) + try: + upstream = socket.create_connection((url.hostname, url.port or 80), timeout=30) + except OSError: + self.wfile.write(b"HTTP/1.1 502 Bad Gateway\r\n\r\n") + return + path = url.path or "/" + if url.query: + path += "?" + url.query + upstream.sendall(f"{method} {path} {version}\r\n".encode("latin-1")) + upstream.sendall(b"".join(h for h in headers + if not h.lower().startswith(b"proxy-")) + b"\r\n") + self._pipe(self.connection, upstream) + + @staticmethod + def _pipe(a, b): + socks = [a, b] + try: + while True: + ready, _, _ = select.select(socks, [], [], 120) + if not ready: + return + for s in ready: + data = s.recv(65536) + if not data: + return + (b if s is a else a).sendall(data) + except OSError: + pass + finally: + b.close() + + +class _Server(socketserver.ThreadingTCPServer): + daemon_threads = True + allow_reuse_address = True + + +def start(port=PORT): + """Starts the proxy on 127.0.0.1 in a background thread. Retries for a + while if the port is busy, as it is while a farm that's just been + stopped is still shutting down.""" + for attempt in range(30): + try: + server = _Server(("127.0.0.1", port), _Handler) + break + except OSError: + if attempt == 29: + raise + time.sleep(1) + threading.Thread(target=server.serve_forever, daemon=True, name="usb-proxy").start() + return server