ci(e2e): Replace maestro with e2e in the expo native integration tests

.changeset/expo-native-e2e-runner.md

@@ -0,0 +1,2 @@
+---
+---

.github/workflows/expo-native-build.yml

@@ -7,6 +7,7 @@ on:
       - main
     paths:
       - '.github/workflows/expo-native-build.yml'
+      - 'integration/e2e.expo-native.config.ts'
       - 'integration/templates/expo-native/**'
       - 'integration/tests/expo-native/**'
       - 'packages/expo/**'
@@ -28,17 +29,17 @@ env:
   SDK_PACK_DIR: /tmp/clerk-expo-pack
   E2E_INSTANCE_NAME: clerkstage-with-native-components
   BAPI_URL: https://api.clerkstage.dev
-  MAESTRO_VERSION: '2.10.0'
-  MAESTRO_CLI_NO_ANALYTICS: '1'
-  MAESTRO_SHARDS: '2'
-  MAESTRO_DRIVER_STARTUP_TIMEOUT: '180000'
+  IOS_SIMULATORS: '2'
+  E2E_TELEMETRY_DISABLED: '1'
 
 jobs:
   native-build:
     if: ${{ github.head_ref != 'changeset-release/main' }}
     name: Native ${{ matrix.run-e2e == true && 'E2E' || 'Build' }} (${{ matrix.platform }}, sdk ${{ matrix.expo-sdk }})
     runs-on: ${{ matrix.runner }}
     timeout-minutes: ${{ matrix.run-e2e == true && 60 || 45 }}
+    env:
+      CLERK_TEST_USERNAME_PREFIX: e2e_${{ github.run_id }}_${{ matrix.platform }}_
     strategy:
       fail-fast: false
       matrix:
@@ -108,16 +109,16 @@ jobs:
           # Bump the version when native build commands change.
           key: expo-native-build-v1-${{ runner.os }}-${{ matrix.expo-sdk }}-${{ matrix.platform }}-${{ matrix.run-e2e == true && 'e2e' || 'build' }}-${{ env.E2E_INSTANCE_NAME }}-${{ steps.native-build-key.outputs.hash }}
 
-      - if: steps.native-build-cache.outputs.cache-hit != 'true'
+      - if: steps.native-build-cache.outputs.cache-hit != 'true' || matrix.run-e2e == true
         uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
-      - if: steps.native-build-cache.outputs.cache-hit != 'true'
+      - if: steps.native-build-cache.outputs.cache-hit != 'true' || matrix.run-e2e == true
         uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
         with:
           node-version: 24.15.0
           cache: pnpm
 
       - name: Install monorepo dependencies
-        if: steps.native-build-cache.outputs.cache-hit != 'true'
+        if: steps.native-build-cache.outputs.cache-hit != 'true' || matrix.run-e2e == true
         run: pnpm install --frozen-lockfile
 
       - name: Build and pack Clerk packages
@@ -144,7 +145,7 @@ jobs:
           BIOMETRICS_TARBALL="$(ls "$SDK_PACK_DIR"/clerk-expo-biometrics-*.tgz)"
           pnpm add "$SDK_TARBALL" "$GOOGLE_SIGNIN_TARBALL" "$BIOMETRICS_TARBALL" -w
           # expo-dev-client makes even release builds boot into the dev
-          # launcher (unreachable Metro in CI), which stalls every Maestro
+          # launcher (unreachable Metro in CI), which stalls every e2e
           # flow on a blank screen. Skip it on e2e jobs only.
           DEV_CLIENT="expo-dev-client"
           if [ "$RUN_E2E" = "true" ]; then DEV_CLIENT=""; fi
@@ -241,91 +242,46 @@ jobs:
 
       - name: Boot iOS simulators
         if: matrix.platform == 'ios' && matrix.run-e2e == true && steps.keys.outputs.pk != ''
-        run: ${{ env.E2E_DIR }}/boot-ios-simulators.sh boot "$MAESTRO_SHARDS"
+        run: ${{ env.E2E_DIR }}/boot-ios-simulators.sh boot "$IOS_SIMULATORS"
 
-      - name: Cache maestro CLI
-        if: steps.keys.outputs.pk != ''
+      - name: Cache iOS automation runner
+        if: matrix.platform == 'ios' && steps.keys.outputs.pk != ''
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
-          path: ~/.maestro
-          key: maestro-${{ runner.os }}-${{ env.MAESTRO_VERSION }}
+          path: ~/.agent-device
+          key: agent-device-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}
+          restore-keys: agent-device-${{ runner.os }}-
 
-      - name: Install maestro CLI
-        if: steps.keys.outputs.pk != ''
-        run: |
-          if [ -x "$HOME/.maestro/bin/maestro" ]; then
-            echo "Using cached Maestro"
-          else
-            archive="$RUNNER_TEMP/maestro.zip"
-            curl -fLs --retry 3 --retry-delay 5 -o "$archive" \
-              "https://github.com/mobile-dev-inc/Maestro/releases/download/cli-$MAESTRO_VERSION/maestro.zip"
-            unzip -q "$archive" -d "$RUNNER_TEMP/maestro-dist"
-            mv "$RUNNER_TEMP/maestro-dist/maestro" "$HOME/.maestro"
-          fi
-          echo "$HOME/.maestro/bin" >> "$GITHUB_PATH"
-          test "$("$HOME/.maestro/bin/maestro" --version)" = "$MAESTRO_VERSION"
-
-      # Provisioned as late as possible so live credentials exist for the
-      # shortest window.
-      - name: Provision test user via BAPI
-        id: user
-        if: steps.keys.outputs.pk != ''
-        continue-on-error: true
-        env:
-          CLERK_SECRET_KEY: ${{ steps.keys.outputs.sk }}
-        run: |
-          email="ci-${GITHUB_RUN_ID}-${RANDOM}+clerk_test@clerkcookie.com"
-          username="ci_${GITHUB_RUN_ID}_${RANDOM}"
-          password="ClerkCI!$(openssl rand -hex 8)Aa1"
-          http_code=$(curl -sS -o /tmp/bapi_response.json -w "%{http_code}" -X POST "$BAPI_URL/v1/users" \
-            -H "Authorization: Bearer $CLERK_SECRET_KEY" \
-            -H "Content-Type: application/json" \
-            -d "{\"email_address\":[\"$email\"],\"username\":\"$username\",\"password\":\"$password\"}")
-          if [ "$http_code" -lt 200 ] || [ "$http_code" -ge 300 ]; then
-            echo "::error::BAPI user creation failed (HTTP $http_code)"
-            cat /tmp/bapi_response.json
-            exit 1
-          fi
-          user_id=$(jq -er '.id' /tmp/bapi_response.json)
-          echo "::add-mask::$password"
-          {
-            echo "email=$email"
-            echo "password=$password"
-            echo "user_id=$user_id"
-          } >> "$GITHUB_OUTPUT"
-
-      - name: Run iOS e2e (Maestro)
+      - name: Run iOS e2e
         id: run_e2e_ios
-        if: matrix.platform == 'ios' && steps.user.outputs.user_id != ''
-        working-directory: ${{ env.FIXTURE_DIR }}
+        if: matrix.platform == 'ios' && steps.keys.outputs.sk != ''
         env:
-          CLERK_TEST_EMAIL: ${{ steps.user.outputs.email }}
-          CLERK_TEST_PASSWORD: ${{ steps.user.outputs.password }}
+          CLERK_SECRET_KEY: ${{ steps.keys.outputs.sk }}
+          CLERK_API_URL: ${{ env.BAPI_URL }}
         run: |
-          "$GITHUB_WORKSPACE/$E2E_DIR/boot-ios-simulators.sh" wait
-          mkdir -p "$RUNNER_TEMP/maestro-debug"
+          "$E2E_DIR/boot-ios-simulators.sh" wait
+          mkdir -p "$RUNNER_TEMP/e2e-device-logs"
           log_pids=()
           trap 'kill "${log_pids[@]}" 2>/dev/null || true; wait "${log_pids[@]}" 2>/dev/null || true' EXIT
-          for udid in ${MAESTRO_UDID//,/ }; do
-            xcrun simctl install "$udid" ios/build/Build/Products/Release-iphonesimulator/ClerkExpoNativeBuildFixture.app
+          for udid in ${CLERK_TEST_DEVICES//,/ }; do
+            xcrun simctl install "$udid" "$FIXTURE_DIR/ios/build/Build/Products/Release-iphonesimulator/ClerkExpoNativeBuildFixture.app"
             xcrun simctl spawn "$udid" log stream --style compact \
               --predicate 'processImagePath CONTAINS "ClerkExpoNativeBuildFixture"' \
-              > "$RUNNER_TEMP/maestro-debug/$udid-console.log" 2>&1 &
+              > "$RUNNER_TEMP/e2e-device-logs/$udid-console.log" 2>&1 &
             log_pids+=($!)
           done
-          PLATFORM=ios MAESTRO_DEBUG_OUTPUT="$RUNNER_TEMP/maestro-debug" \
-            "$GITHUB_WORKSPACE/$E2E_DIR/run-flows.sh"
+          pnpm test:integration:expo-native:ios --reporter list,junit,markdown
 
       - name: Enable KVM
-        if: matrix.platform == 'android' && steps.user.outputs.user_id != ''
+        if: matrix.platform == 'android' && steps.keys.outputs.sk != ''
         run: |
           echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
           sudo udevadm control --reload-rules
           sudo udevadm trigger --name-match=kvm
 
       - name: AVD cache
         id: avd-cache
-        if: matrix.platform == 'android' && steps.user.outputs.user_id != ''
+        if: matrix.platform == 'android' && steps.keys.outputs.sk != ''
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: |
@@ -334,7 +290,7 @@ jobs:
           key: avd-34-google_apis-x86_64-v1
 
       - name: Create AVD snapshot
-        if: matrix.platform == 'android' && steps.user.outputs.user_id != '' && steps.avd-cache.outputs.cache-hit != 'true'
+        if: matrix.platform == 'android' && steps.keys.outputs.sk != '' && steps.avd-cache.outputs.cache-hit != 'true'
         uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2
         with:
           api-level: 34
@@ -345,73 +301,69 @@ jobs:
           disable-animations: false
           script: echo "Generated AVD snapshot for caching."
 
-      - name: Run Android e2e (Maestro)
+      - name: Run Android e2e
         id: run_e2e_android
-        if: matrix.platform == 'android' && steps.user.outputs.user_id != ''
+        if: matrix.platform == 'android' && steps.keys.outputs.sk != ''
         uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2
         env:
-          CLERK_TEST_EMAIL: ${{ steps.user.outputs.email }}
-          CLERK_TEST_PASSWORD: ${{ steps.user.outputs.password }}
+          CLERK_SECRET_KEY: ${{ steps.keys.outputs.sk }}
+          CLERK_API_URL: ${{ env.BAPI_URL }}
         with:
           api-level: 34
           target: google_apis
           arch: x86_64
           force-avd-creation: false
-          # read-only: a writable instance holds a lock the shard emulators wait on forever
-          emulator-options: -read-only -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim
+          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim
           disable-animations: true
           # The action runs each script line in a separate sh -c; the folded
           # scalar (>-) plus && keeps everything in one shell invocation.
           script: >-
-            cd integration/tests/expo-native &&
-            MAESTRO_DEBUG_OUTPUT="$RUNNER_TEMP/maestro-debug" ./run-android-flows.sh
-            "$GITHUB_WORKSPACE/$FIXTURE_DIR/android/app/build/outputs/apk/release/app-release.apk"
-
-      # Runs before the artifact upload so the credentials are already dead
-      # by the time the debug files become downloadable.
-      - name: Cleanup test user
-        if: always() && steps.user.outputs.user_id != ''
+            mkdir -p "$RUNNER_TEMP/e2e-device-logs" &&
+            adb install -r "$FIXTURE_DIR/android/app/build/outputs/apk/release/app-release.apk" &&
+            (adb logcat -v threadtime > "$RUNNER_TEMP/e2e-device-logs/logcat.log" 2>&1 &) &&
+            pnpm test:integration:expo-native:android --reporter list,junit,markdown
+
+      - name: Delete leftover test users
+        if: always() && steps.keys.outputs.sk != ''
         env:
           CLERK_SECRET_KEY: ${{ steps.keys.outputs.sk }}
-          USER_ID: ${{ steps.user.outputs.user_id }}
-        run: |
-          curl -fsS -X DELETE "$BAPI_URL/v1/users/$USER_ID" \
-            -H "Authorization: Bearer $CLERK_SECRET_KEY" || true
-
-      # Test reports record flow env (and typed input) in plaintext;
-      # add-mask only covers step logs, not artifact contents. Must run whenever
-      # the upload does, or a passing run uploads the password.
-      - name: Scrub test credentials from Maestro debug output
-        id: scrub
-        if: always() && steps.user.outputs.user_id != ''
-        env:
-          CLERK_TEST_PASSWORD: ${{ steps.user.outputs.password }}
         run: |
-          [ -d "$RUNNER_TEMP/maestro-debug" ] || exit 0
-          [ -n "$CLERK_TEST_PASSWORD" ] || exit 0
-          # shellcheck disable=SC2016
-          find "$RUNNER_TEMP/maestro-debug" -type f \
-            \( -name '*.html' -o -name '*.json' -o -name '*.log' -o -name '*.txt' -o -name '*.xml' -o -name '*.yaml' \) \
-            -exec perl -pi -e 's/\Q$ENV{CLERK_TEST_PASSWORD}\E/[REDACTED]/g' {} +
-
-      - name: Upload Maestro artifacts
-        if: always() && steps.scrub.outcome == 'success'
+          curl -fsS -G "$BAPI_URL/v1/users" --data-urlencode "limit=100" --data-urlencode "query=$CLERK_TEST_USERNAME_PREFIX" -H "Authorization: Bearer $CLERK_SECRET_KEY" |
+            jq -r --arg p "$CLERK_TEST_USERNAME_PREFIX" '.[] | select((.username // "") | startswith($p)) | .id' |
+            while read -r user_id; do
+              curl -fsS --retry 3 --retry-delay 1 -o /dev/null -X DELETE "$BAPI_URL/v1/users/$user_id" -H "Authorization: Bearer $CLERK_SECRET_KEY" ||
+                echo "::warning::Could not delete leftover test user $user_id"
+            done
+
+      - name: Upload e2e artifacts
+        if: always() && steps.keys.outputs.sk != ''
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
-          name: maestro-${{ matrix.platform }}
-          path: ${{ runner.temp }}/maestro-debug
+          name: e2e-${{ matrix.platform }}
+          include-hidden-files: true
+          path: |
+            integration/.e2e
+            ${{ runner.temp }}/e2e-device-logs
           if-no-files-found: warn
           retention-days: 7
 
       - name: Report e2e outcome
-        if: always() && steps.user.outputs.user_id != ''
+        if: always() && steps.keys.outputs.sk != ''
         env:
           IOS_OUTCOME: ${{ steps.run_e2e_ios.outcome }}
           ANDROID_OUTCOME: ${{ steps.run_e2e_android.outcome }}
         run: |
           outcome="$IOS_OUTCOME"
           [ "$outcome" = "skipped" ] && outcome="$ANDROID_OUTCOME"
-          echo "## Maestro e2e (${{ matrix.platform }}): $outcome" >> "$GITHUB_STEP_SUMMARY"
+          echo "## Native e2e (${{ matrix.platform }}): $outcome" >> "$GITHUB_STEP_SUMMARY"
+          if [ -f integration/.e2e/summary.md ]; then
+            cat integration/.e2e/summary.md >> "$GITHUB_STEP_SUMMARY"
+          fi
           if [ "$outcome" = "failure" ]; then
-            echo "::error::Maestro e2e failed. See the maestro-${{ matrix.platform }} artifact."
+            echo "::error::Native e2e failed. See the e2e-${{ matrix.platform }} artifact."
+          fi
+          flaky=$(jq -r '[.run.results[] | select(.status == "flaky") | .file] | join(", ")' integration/.e2e/report.json 2>/dev/null || true)
+          if [ -n "$flaky" ]; then
+            echo "::error::Flaky flow(s), failed then passed on the retry: $flaky"
+            exit 1
           fi

.gitignore

@@ -67,6 +67,7 @@ packages/*/examples/*/package-lock.json
 packages/*/examples/*/yarn.lock
 /test-results/
 /playwright-report/
+/integration/.e2e/
 /playwright/.cache/
 
 # integration testing

integration/e2e.expo-native.config.ts

@@ -0,0 +1,15 @@
+import { mobile } from '@e2e-dev/mobile';
+import type { E2EConfig } from 'e2e';
+
+const app = { bundleId: 'com.clerk.exponativebuildfixture' };
+const devices = process.env.CLERK_TEST_DEVICES?.split(',');
+
+export default {
+  tests: 'tests/expo-native/*.e2e.ts',
+  targets: [
+    { name: 'ios', engine: mobile({ platform: 'ios', device: devices }), app },
+    { name: 'android', engine: mobile({ platform: 'android', device: devices }), app },
+  ],
+  workers: devices?.length ?? 1,
+  timeout: 300_000,
+} satisfies E2EConfig;

integration/tests/expo-native/auth-view.e2e.ts

@@ -0,0 +1,22 @@
+import { expect } from 'e2e';
+
+import { testWithUser as test } from './fixtures.ts';
+
+test.describe('@clerk/expo <AuthView />', () => {
+  test.beforeEach(({ po }) => po.app.reset());
+
+  test('rehosts the React logo and still signs in after the host view is detached and reattached', async ({
+    po,
+    screen,
+    user,
+  }) => {
+    await po.authView.open();
+    await expect(screen.getByText('E2E Custom Logo')).toBeVisible({ timeout: 10_000 });
+    await po.authView.dismiss();
+    await po.authView.open();
+    await expect(screen.getByText('E2E Custom Logo')).toBeVisible({ timeout: 15_000 });
+    await po.authView.signIn(user);
+    await po.app.expectSignedIn();
+    await po.app.signOut();
+  });
+});

integration/tests/expo-native/boot-ios-simulators.sh

@@ -14,14 +14,14 @@ boot() {
     udids+=("$(xcrun simctl create "$model $i" "$device_type" "$runtime")")
     xcrun simctl boot "${udids[$((i - 1))]}"
   done
-  MAESTRO_UDID=$(IFS=,; echo "${udids[*]}")
-  echo "Booting $count x $model on $runtime: $MAESTRO_UDID"
-  if [ -n "${GITHUB_ENV:-}" ]; then echo "MAESTRO_UDID=$MAESTRO_UDID" >> "$GITHUB_ENV"; fi
+  CLERK_TEST_DEVICES=$(IFS=,; echo "${udids[*]}")
+  echo "Booting $count x $model on $runtime: $CLERK_TEST_DEVICES"
+  if [ -n "${GITHUB_ENV:-}" ]; then echo "CLERK_TEST_DEVICES=$CLERK_TEST_DEVICES" >> "$GITHUB_ENV"; fi
 }
 
 wait_ready() {
   local udid key
-  IFS=, read -r -a udids <<< "${MAESTRO_UDID:?MAESTRO_UDID is required}"
+  IFS=, read -r -a udids <<< "${CLERK_TEST_DEVICES:?CLERK_TEST_DEVICES is required}"
   for udid in "${udids[@]}"; do
     xcrun simctl bootstatus "$udid" -b &
     local pid=$! elapsed=0

integration/tests/expo-native/fixtures.ts

@@ -0,0 +1,20 @@
+import { test as base } from '@e2e-dev/mobile';
+
+import type { PageObjects } from './page-objects/index.ts';
+import { createPageObjects } from './page-objects/index.ts';
+import type { TestUser } from './users.ts';
+import { createTestUser, deleteTestUser } from './users.ts';
+
+export const test = base.extend<{ po: PageObjects }>({
+  po: async (fixtures, provide) => {
+    await provide(createPageObjects(fixtures));
+  },
+});
+
+export const testWithUser = test.extend<{ user: TestUser }>({
+  user: async (_fixtures, provide) => {
+    const user = await createTestUser();
+    await provide(user);
+    await deleteTestUser(user);
+  },
+});