Memory regression tests

.github/actions/test-capture-variant/action.yml

@@ -47,7 +47,6 @@ runs:
         import os
         import pathlib
         import subprocess
-        import time
         sources = sorted(set(os.environ["VARIANT_SOURCES"].split(",")))
         if not sources or any(source not in {"clock", "gstreamer", "pattern"} for source in sources):
             raise SystemExit("Invalid selective capture source set")
@@ -62,7 +61,6 @@ runs:
                         os.environ["VARIANT_CONFIG_HASH"], rust_sha])
         with open(os.environ["GITHUB_OUTPUT"], "a") as output:
             output.write(f"cargo-dir={cargo_dir.as_posix()}\nkey={key}\n")
-            output.write(f"started-at={time.monotonic()}\n")
             output.write("bundle-dir=sdk-out/capture-" + "-".join(sources) + "\n")
         PYCODE
     - name: Restore variant Cargo cache
@@ -99,9 +97,6 @@ runs:
       shell: bash
       env:
         VARIANT_CARGO_DIR: ${{ steps.variant.outputs.cargo-dir }}
-        VARIANT_SOURCES: ${{ inputs.sources }}
-        VARIANT_STARTED_AT: ${{ steps.variant.outputs.started-at }}
-        VARIANT_CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
       run: |
         set -euo pipefail
         variant_python=python
@@ -110,15 +105,10 @@ runs:
         import os
         import pathlib
         import shutil
-        import time
         path = pathlib.Path(os.environ["VARIANT_CARGO_DIR"]).resolve()
         root = pathlib.Path(os.environ["CARGO_TARGET_DIR"]).resolve()
         if path.parent != root or not path.name.startswith("cpp-"):
             raise SystemExit("Refusing to remove a path outside the feature-specific Cargo root")
         if path.exists():
             shutil.rmtree(path)
-        elapsed = time.monotonic() - float(os.environ["VARIANT_STARTED_AT"])
-        cache = "hit" if os.environ["VARIANT_CACHE_HIT"] == "true" else "miss"
-        with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary:
-            summary.write(f"Capture `{os.environ['VARIANT_SOURCES']}`: {elapsed:.0f}s; Cargo cache {cache}.\n\n")
         PYCODE

.github/workflows/ci.yml

@@ -67,6 +67,7 @@ jobs:
               - client-sdk-rust
               - client-sdk-rust/**
               - cmake/**
+              - scripts/memory-tester.sh
               - scripts/set-test-tokens.sh
               - scripts/run-with-backtrace.sh
               - scripts/setup-linux-toolchain.sh

.github/workflows/tests.yml

@@ -380,7 +380,11 @@ jobs:
             --gtest_output="xml:${{ env.BUILD_DIR }}\unit-test-results.xml"
 
       - name: Start livekit-server
-        if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        if: >-
+          matrix.e2e-testing &&
+          (inputs.integration_repeat > 0 ||
+           inputs.run_stress_tests ||
+           (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release'))
         id: livekit_server
         uses: livekit/dev-server-action@5d4d5337a875e2d1afd37bed03c601d159dab002 # v1.1.1
         with:
@@ -391,7 +395,11 @@ jobs:
 
       # Needed by token helper script
       - name: Install livekit-cli
-        if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        if: >-
+          matrix.e2e-testing &&
+          (inputs.integration_repeat > 0 ||
+           inputs.run_stress_tests ||
+           (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release'))
         shell: bash
         env:
           # Windows installs lk via `gh api` / `gh release download`, which need this env var
@@ -448,6 +456,23 @@ jobs:
             --gtest_recreate_environments_when_repeating=1 \
             --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml
 
+      - name: Run memory tester
+        if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release'
+        timeout-minutes: 10
+        shell: bash
+        run: |
+          set -euo pipefail
+          source scripts/set-test-tokens.sh
+          scripts/memory-tester.sh \
+            --build-dir "${{ env.BUILD_DIR }}" \
+            --scenario all \
+            --iterations 100 \
+            --warmup 20 \
+            --max-rss-growth-mib 24 \
+            --max-thread-growth 2 \
+            --advisory-max-rss-growth-mib 8 \
+            --advisory-max-thread-growth 0
+
       - name: Run stress tests
         if: matrix.e2e-testing && inputs.run_stress_tests
         timeout-minutes: ${{ inputs.stress_timeout_minutes }}

docs/testing.md

@@ -1,7 +1,21 @@
 # Testing
 
-The SDK includes integration and stress tests using
-[Google Test](https://github.com/google/googletest).
+The SDK includes unit, integration and stress tests using
+[Google Test](https://github.com/google/googletest). It also
+includes a memory-leak/resource checking tester.
+
+Unit, integration, and memory tests are run on every PR in CI.
+Stress tests are reserved for nightly tests in CI due to their
+length.
+
+## Test binaries
+
+| Executable | Description |
+|------------|-------------|
+| `livekit_unit_tests` | Unit tests (no server required) |
+| `livekit_integration_tests` | Integration tests (server required) |
+| `livekit_stress_tests` | Long-running stress tests (server required) |
+| `livekit_memory_tester` | Resource leak tester |
 
 ## Building the test binaries
 
@@ -19,31 +33,23 @@ The SDK includes integration and stress tests using
 
 ## Running tests
 
-After building, run tests using `ctest` or invoke the binaries directly:
-
 ```bash
-# Run all tests via ctest
-cd build-debug
+# Run all tests via ctest (warning, runs all GTest-based tests including stress)
+cd build-debug # or build-release
 ctest --output-on-failure
 
 # Or run test executables directly
+./build-debug/bin/livekit_unit_tests
 ./build-debug/bin/livekit_integration_tests
 ./build-debug/bin/livekit_stress_tests
 
 # Run specific test suites
+./build-debug/bin/livekit_unit_tests --gtest_filter="*Room*"
 ./build-debug/bin/livekit_integration_tests --gtest_filter="*Rpc*"
 ./build-debug/bin/livekit_stress_tests --gtest_filter="*MaxPayloadStress*"
 ```
 
-__Note:__ The tests require tokens and a running LiveKit server. See the section below for details.
-
-## Test binaries
-
-| Executable | Description |
-|------------|-------------|
-| `livekit_unit_tests` | Pure unit tests (no server required) |
-| `livekit_integration_tests` | Quick tests (~1-2 minutes) for SDK functionality |
-| `livekit_stress_tests` | Long-running tests (configurable, default 1 hour) |
+__Note:__ The integration and stress tests require tokens and a running LiveKit server. See the section below for details.
 
 ## Running a local LiveKit server for tests
 
@@ -52,9 +58,10 @@ path is `livekit-server --dev`, which uses the well-known dev API
 key/secret (`devkey` / `secret`).
 
 Install [`livekit-server`](https://docs.livekit.io/home/self-hosting/local/)
-and start it with data tracks enabled:
+and start it with data blobs enabled:
 
 ```bash
+LIVEKIT_CONFIG="enable_participant_data_blob: true"
 livekit-server --dev
 ```
 
@@ -177,7 +184,88 @@ cold runs even though C++ dependencies are reused.
 - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls.
 - **Stress**: high throughput, bidirectional RPC, memory pressure.
 
-## Memory checks (valgrind)
+## Memory tester
+
+> Note: Linux only
+
+The memory tester binary catches resource leak regressions in an
+automated fashion, and is invoked in CI. This single binary runs individual
+or groups of scenarios:
+
+- `audio-track` repeatedly initializes the SDK, connects, publishes an audio
+  track, captures synthetic PCM frames, unpublishes, disconnects, and shuts down.
+- `video-track` repeatedly initializes the SDK, connects, publishes a video
+  track, captures synthetic I420 frames, unpublishes, disconnects, and shuts down.
+- `data-track` repeatedly connects, publishes a data track, pushes synthetic
+  payloads, unpublishes it, and disconnects.
+- `room-reuse` repeatedly connects and disconnects the same room object.
+- `room-client-leave` repeatedly connects, disconnects, and destroys a room.
+- `room-server-delete` repeatedly connects, publishes one synthetic video
+  track, captures I420 frames without unpublishing, and uses `lk` to delete
+  the local test room. It then verifies the disconnect and end-of-stream
+  callbacks.
+- `capture-clock` repeatedly initializes the SDK, creates and publishes the
+  GPU-rendered clock capture source, starts and stops its frame pump, joins its
+  worker, unpublishes, disconnects, and shuts down. Each cycle must produce at
+  least three frames, so the default 100-cycle run captures at least 300 frames
+  (10 seconds of video at 30 fps).
+- `capture-pattern` exercises the same lifecycle with the GPU-rendered gradient
+  pattern capture source.
+- `capture-gstreamer` exercises the same lifecycle through an entirely
+  software-based GStreamer pipeline (`videotestsrc` and `vp8enc`), including
+  encoded-frame ingest and WebRTC passthrough.
+
+Each scenario reports process RSS and thread count at cycle 0, after warmup
+cycle 20, and after final cycle 100. Checkpoint sampling waits for the thread
+count to remain stable for 500 ms so short-lived teardown work does not cause a
+false failure. The gate applies to warmup-to-final growth. CI fails above its
+blocking limits of 24 MiB RSS or two threads, which protects against large
+customer-impacting regressions while allowing for allocator and dependency
+variation. It also emits a non-blocking advisory above 8 MiB RSS or zero
+threads. If threads grow, the tester reports when the final sustained growth
+began and groups the warmup and final threads by their Linux thread names.
+
+The binary can be run directly, or via a helper script at `scripts/memory-tester.sh`,
+which invokes the binary but also has GitHub actions support for a results summary table.
+The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process.
+This limits `glibc`'s allocator arenas, reducing Linux thread-churn RSS noise, but it is
+not evidence that the default allocator returns all freed pages to the OS.
+
+Start `livekit-server --dev`, install the `lk` CLI, and source the local test
+tokens before running any scenario:
+
+```bash
+./build.sh release-tests --capture=all
+source scripts/set-test-tokens.sh
+scripts/memory-tester.sh --scenario audio-track \
+  --iterations 100 --warmup 20 \
+  --max-rss-growth-mib 24 --max-thread-growth 2 \
+  --advisory-max-rss-growth-mib 8 --advisory-max-thread-growth 0
+scripts/memory-tester.sh --scenario video-track \
+  --iterations 100 --warmup 20 \
+  --max-rss-growth-mib 24 --max-thread-growth 2 \
+  --advisory-max-rss-growth-mib 8 --advisory-max-thread-growth 0
+```
+
+Use `--scenario all` to run all nine scenarios, each in a fresh tester process:
+
+```bash
+scripts/memory-tester.sh --scenario data-track
+scripts/memory-tester.sh --scenario room-reuse
+scripts/memory-tester.sh --scenario room-client-leave
+scripts/memory-tester.sh --scenario room-server-delete
+scripts/memory-tester.sh --scenario capture-clock
+scripts/memory-tester.sh --scenario capture-pattern
+scripts/memory-tester.sh --scenario capture-gstreamer
+scripts/memory-tester.sh --scenario all
+```
+
+The blocking and advisory limits are command-line options so a longer local run
+can use different budgets. When advisory limits are omitted, they default to the
+blocking limits. RSS growth thresholds are expressed in MiB and may be
+fractional, such as `--advisory-max-rss-growth-mib 0.5`.
+
+## Additional memory checks (valgrind)
 
 Run `valgrind` against the test binaries to check for memory leaks and other
 issues. See [tools.md](tools.md) for the recipe.