Add GitHub CI with unit, crypto compatibility and end-to-end tests

.github/workflows/pull-request-tests.yml

@@ -0,0 +1,152 @@
+# Unit tests, release-like (R8 minified) build and instrumented tests for every pull request against master.
+# The instrumented tests run against a real Passman server in docker (see ci/start-passman-server.sh).
+name: Pull request tests
+
+on:
+  pull_request:
+    branches: [master]
+
+concurrency:
+  group: pull-request-tests-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  # restores ci/test-vault, needs the passman:restore command (not part of a Passman release yet)
+  PASSMAN_SERVER_IMAGE: docker.io/binsky/passman-dev:cmd-backups
+
+jobs:
+  build:
+    name: Build and unit tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          submodules: true
+
+      - uses: actions/setup-java@v6
+        with:
+          distribution: temurin
+          java-version: 21
+
+      - uses: gradle/actions/setup-gradle@v6
+
+      - name: Prepare build configuration
+        run: |
+          # gradle.properties is not versioned, the example has no usable signing config (ciRelease uses the debug key)
+          cp gradle.properties.example gradle.properties
+          cp openssl.conf.example openssl.conf
+          NDK_VERSION=$(sed -n "s/.*ndkVersion = '\(.*\)'.*/\1/p" app/build.gradle)
+          echo "NDK_VERSION=$NDK_VERSION" >> "$GITHUB_ENV"
+          echo "OPENSSL_REVISION=$(git -C app/src/main/cpplibs/openssl rev-parse HEAD)" >> "$GITHUB_ENV"
+          yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" --install "ndk;$NDK_VERSION" "cmake;3.22.1" > /dev/null
+
+      - name: Cache OpenSSL build
+        id: openssl-cache
+        uses: actions/cache@v6
+        with:
+          path: app/src/main/cpplibs/openssl/build
+          key: openssl-${{ env.OPENSSL_REVISION }}-ndk-${{ env.NDK_VERSION }}-${{ hashFiles('build-openssl.sh', 'openssl.conf.example') }}
+
+      - name: Build OpenSSL
+        if: steps.openssl-cache.outputs.cache-hit != 'true'
+        run: |
+          sed -i "s/^export THREADS=.*/export THREADS=$(nproc)/" openssl.conf
+          ./build-openssl.sh > build-openssl.log 2>&1 || (tail -n 100 build-openssl.log; exit 1)
+
+      - name: Unit tests
+        run: ./gradlew --stacktrace -Ppassman.testBuildType=ciRelease :app:testCiReleaseUnitTest
+
+      - name: Build release-like apks
+        run: ./gradlew --stacktrace -Ppassman.testBuildType=ciRelease :app:assembleCiRelease :app:assembleCiReleaseAndroidTest
+
+      - name: Upload unit test reports
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: unit-test-reports
+          path: app/build/reports/tests/
+          retention-days: 7
+
+      - name: Upload apks
+        uses: actions/upload-artifact@v7
+        with:
+          name: ci-apks
+          path: |
+            app/build/outputs/apk/ciRelease/*.apk
+            app/build/outputs/apk/androidTest/ciRelease/*.apk
+          if-no-files-found: error
+          retention-days: 7
+
+  instrumented:
+    name: API ${{ matrix.api-level }} / ${{ matrix.abi }} apk / ${{ matrix.suite }}
+    needs: build
+    # the ubuntu-latest runners of the organization (GARM) provide no KVM for the emulator,
+    # like nextcloud/android, use runners with hardware virtualization for the emulator jobs
+    runs-on: ubuntu-26.04
+    timeout-minutes: 45
+    # armeabi-v7a only runs through the ARM translation of the emulator, not required until it proved to be stable
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # minSdk
+          - { api-level: 23, abi: x86_64, suite: all, experimental: false }
+          # oldest Android version with security updates, x86_64 images run ARM apps through binary translation
+          - { api-level: 34, abi: arm64-v8a, suite: all, experimental: false }
+          # targetSdk
+          - { api-level: '37.0', abi: x86_64, suite: all, experimental: false }
+          # 32-bit ABIs: newer x86_64 images are 64-bit only (no x86, no armeabi-v7a translation)
+          - { api-level: 30, abi: x86, suite: offline, experimental: false }
+          - { api-level: 30, abi: armeabi-v7a, suite: offline, experimental: true }
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: actions/download-artifact@v8
+        with:
+          name: ci-apks
+          path: apks
+
+      - name: Start Passman server
+        if: matrix.suite == 'all'
+        run: ci/start-passman-server.sh "$PASSMAN_SERVER_IMAGE"
+
+      - name: Enable KVM
+        run: |
+          # hardware acceleration for the emulator, without it the emulator is far too slow
+          if [ ! -e /dev/kvm ]; then
+            echo "::error::This runner provides no KVM (/dev/kvm)"
+            exit 1
+          fi
+          sudo chmod 666 /dev/kvm
+          ls -l /dev/kvm
+
+      - name: Instrumented tests
+        uses: reactivecircus/android-emulator-runner@v2
+        env:
+          TEST_PACKAGES: ${{ matrix.suite == 'all' && 'es.wolfi.app.passman.offline,es.wolfi.app.passman.e2e' || 'es.wolfi.app.passman.offline' }}
+        with:
+          api-level: ${{ matrix.api-level }}
+          target: google_apis
+          arch: x86_64
+          disable-animations: true
+          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -no-snapshot -camera-back none
+          script: ci/run-instrumented-tests.sh apks/ciRelease/app-${{ matrix.abi }}-ciRelease.apk apks/androidTest/ciRelease/app-ciRelease-androidTest.apk "$TEST_PACKAGES" -e serverUrl http://10.0.2.2:8080
+
+      - name: Collect server log
+        if: failure() && matrix.suite == 'all'
+        run: |
+          mkdir -p ci-output
+          docker exec passman-ci cat /var/www/html/data/nextcloud.log > ci-output/nextcloud.log 2>&1 || true
+
+      - name: Upload test output
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: instrumented-api${{ matrix.api-level }}-${{ matrix.abi }}
+          path: ci-output/
+          retention-days: 7

.gitignore

@@ -67,3 +67,6 @@ openssl.conf
 .idea/*
 
 gradle.properties
+
+# output of ci/run-instrumented-tests.sh
+ci-output/

app/build.gradle

@@ -137,6 +137,27 @@ android {
                 signingConfig = signingConfigs.alpha
             }
         }
+        // Release-like build for the instrumented tests in CI (same R8 and resource shrinking as release).
+        // Signed with the debug key and allows cleartext HTTP to the test server (see src/ciRelease).
+        ciRelease {
+            initWith release
+            applicationIdSuffix ".ci"
+            versionNameSuffix '-CI'
+            signingConfig = signingConfigs.debug
+            proguardFile 'proguard-ci-rules.pro'
+            testProguardFiles 'proguard-test-rules.pro'
+            matchingFallbacks = ['release']
+        }
+    }
+
+    // Instrumented tests run against debug by default, CI passes -Ppassman.testBuildType=ciRelease
+    testBuildType = project.findProperty('passman.testBuildType') ?: 'debug'
+
+    sourceSets {
+        androidTest {
+            // exchangeable test data (see ci/test-vault/README.md)
+            assets.srcDirs += "$rootDir/ci/test-vault"
+        }
     }
 
     externalNativeBuild {
@@ -178,6 +199,14 @@ androidComponents {
     }
 }
 
+// The androidTest apk runs in the app's process and brings its own copy of the desugared Java library (j$.*),
+// which shadows the app's copy. AGP shrinks and obfuscates both copies independently, so the app would run
+// into the test's incompatible copy. Keeping the complete library in both makes the copies identical.
+// Only applies to the CI build type, uses an internal AGP task property (see proguard-ci-rules.pro for the rest).
+tasks.matching { it.name in ['l8DexDesugarLibCiRelease', 'l8DexDesugarLibCiReleaseAndroidTest'] }.configureEach {
+    keepRulesConfigurations.add('-keep class j$.** { *; }')
+}
+
 // Pin the JDK that compiles sources. This is independent of compileOptions (bytecode is still Java 17).
 // See https://developer.android.com/build/jdks#toolchain
 java {
@@ -189,9 +218,16 @@ java {
 dependencies {
     implementation fileTree(include: ['*.jar'], dir: 'libs')
     androidTestImplementation 'androidx.test.ext:junit:1.3.0'
+    androidTestImplementation 'androidx.test:runner:1.7.0'
+    androidTestImplementation 'androidx.test:rules:1.7.0'
     androidTestImplementation('androidx.test.espresso:espresso-core:3.7.0', {
         exclude group: 'com.android.support', module: 'support-annotations'
     })
+    androidTestImplementation 'androidx.test.espresso:espresso-contrib:3.7.0'
+    androidTestImplementation('org.skyscreamer:jsonassert:1.5.3') {
+        // a copy of the org.json classes Android already provides
+        exclude group: 'com.vaadin.external.google', module: 'android-json'
+    }
 
     coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:2.1.5'
 
@@ -215,4 +251,6 @@ dependencies {
     implementation('androidx.camera:camera-lifecycle:1.6.2')
     implementation('androidx.camera:camera-view:1.6.2')
     testImplementation 'junit:junit:4.13.2'
+    // the org.json classes of android.jar are only stubs in local unit tests
+    testImplementation 'org.json:json:20260814'
 }

app/proguard-ci-rules.pro

@@ -0,0 +1,14 @@
+# Additional R8 rules for the ciRelease build type only, on top of the release rules.
+#
+# The androidTest apk doesn't bundle libraries the app already contains (like the Kotlin standard library),
+# it uses the app's copy at runtime. R8 removes everything of them the app itself doesn't use,
+# so keep the libraries the test runner and Espresso need. The app code is shrunk the same way as for release.
+
+-keep class kotlin.** { *; }
+-keep class kotlinx.coroutines.** { *; }
+-keep class androidx.tracing.** { *; }
+-keep class androidx.concurrent.futures.** { *; }
+-keep class com.google.common.util.concurrent.ListenableFuture { *; }
+
+# resource ids the tests look up but the app code doesn't reference (only used in layouts)
+-keep class es.wolfi.app.passman.R$* { public static <fields>; }

app/proguard-test-rules.pro

@@ -0,0 +1,13 @@
+# R8 rules for the androidTest apk when testing the minified ciRelease build.
+# The test apk is shrunk against the mapping of the app, the test code itself must stay untouched.
+
+-dontoptimize
+-dontobfuscate
+-dontwarn **
+
+-keep class es.wolfi.app.passman.offline.** { *; }
+-keep class es.wolfi.app.passman.e2e.** { *; }
+-keep class androidx.test.** { *; }
+-keep class org.junit.** { *; }
+-keep class junit.** { *; }
+-keep class org.hamcrest.** { *; }