build: fix clang sanitizer and fuzzing builds, add build.sh fuzz

Documentation/TESTING.md

@@ -225,6 +225,15 @@ meson test -C .build-asanubsan --setup asanubsan
 No wrapper binary is needed — sanitizer reports are emitted directly to stderr
 when a violation is detected.
 
+With clang, set the C++ compiler too, and let shared libraries have the
+symbols of the sanitizer runtime undefined: clang links the runtime only into
+executables.
+
+```bash
+CC=clang CXX=clang++ meson setup .build-asanubsan \
+    -Db_sanitize=address,undefined -Db_lundef=false
+```
+
 The same known limitations apply as for Valgrind: Python tests and shell script
 tests may produce false positives. Focus triage on the `nvme-cli - *` unit tests.
 
@@ -255,25 +264,55 @@ FUZZ_SECONDS=60 FUZZ_SEED=7 meson test -C .build --suite fuzz
 Run in a sanitizer build (see above) they also catch memory errors which
 don't crash.
 
-For longer, coverage guided runs, build the targets for libFuzzer with clang:
+For longer, coverage guided runs, build the targets for libFuzzer with clang,
+in a sanitizer build as described above:
 
 ```bash
-CC=clang meson setup .build-fuzz -Dfuzzing=true -Db_sanitize=address,undefined
+CC=clang CXX=clang++ meson setup .build-fuzz -Dfuzzing=true \
+    -Db_sanitize=address,undefined -Db_lundef=false
 meson compile -C .build-fuzz
-mkdir corpus && cp tests/fuzz/corpus/argconfig/* corpus/
+```
+
+Run a target on a copy of its seeds in the build directory, so the seeds in
+the tree stay as they are. The new inputs libFuzzer finds go to the copy:
+
+```bash
+mkdir -p .build-fuzz/corpus/argconfig
+cp tests/fuzz/corpus/argconfig/* .build-fuzz/corpus/argconfig/
 .build-fuzz/tests/fuzz/fuzz-argconfig -close_fd_mask=3 \
-    -dict=tests/fuzz/argconfig.dict corpus
+    -dict=tests/fuzz/argconfig.dict .build-fuzz/corpus/argconfig
+```
+
+The config targets (`ini`, `global-config`, `fabrics-conf`) use
+`tests/fuzz/config.dict`. The `device` target reads log pages, so give it
+longer inputs. A device reports sizes of up to gigabytes, which nvme-cli
+allocates: let ASan fail such an allocation instead of libFuzzer aborting the
+run as out of memory:
+
+```bash
+mkdir -p .build-fuzz/corpus/device
+cp tests/fuzz/corpus/device/* .build-fuzz/corpus/device/
+ASAN_OPTIONS=allocator_may_return_null=1:max_allocation_size_mb=1024 \
+    .build-fuzz/tests/fuzz/fuzz-device -close_fd_mask=3 -max_len=16384 \
+    -timeout=10 .build-fuzz/corpus/device
 ```
 
-The config targets use `tests/fuzz/config.dict`. The `device` target
-reads log pages, so give it longer inputs, and let allocations of the sizes
-a device reports fail instead of aborting:
+`scripts/build.sh fuzz` does all of this for every target in turn, each with
+one libFuzzer job per CPU on a shared corpus, for 600 seconds by default:
 
 ```bash
-ASAN_OPTIONS=allocator_may_return_null=1 .build-fuzz/tests/fuzz/fuzz-device \
-    -close_fd_mask=3 -max_len=16384 -timeout=10 corpus
+scripts/build.sh --fuzz-seconds 3600 fuzz
+scripts/build.sh --fuzz-targets device --fuzz-jobs 8 fuzz
 ```
- The targets use the
-libFuzzer interface, so AFL++ can run them too (`afl-clang-fast` with
-`-fsanitize=fuzzer`). Add an input which found a bug to the seeds of its
-target.
+
+It writes the results to `.build-ci/fuzz`: `summary.txt`, and per target the
+logs of the jobs in `logs/<target>/`, the inputs which found a bug in
+`artifacts/<target>/` and the corpus in `corpus/<target>/`. A finding stops
+only its job, a UBSan finding like a crash. The next run of `build.sh` removes
+the results; pass a `--fuzz-dir` outside of `.build-ci` to keep them, and to
+grow the corpus across runs. `scripts/build.sh -h` lists all options.
+
+When run by hand, a crashing input is written to `crash-*` in the current
+directory. Add an input which found a bug to the seeds of its target. The
+targets use the libFuzzer interface, so AFL++ can run them too
+(`afl-clang-fast` with `-fsanitize=fuzzer`).

meson.build

@@ -35,6 +35,27 @@ cc = meson.get_compiler('c')
 cxx_available = add_languages('cpp', required: false, native: false)
 host_system = host_machine.system()
 
+# Sanitizer and fuzzing builds fail late, with misleading linker errors,
+# in some setups. Catch them here.
+if get_option('b_sanitize') != 'none' or get_option('fuzzing')
+    # The C++ tests link with the C++ compiler. With a different toolchain
+    # than the C compiler, the sanitizer runtimes do not match.
+    if cxx_available
+        cxx_id = meson.get_compiler('cpp').get_id()
+        if cxx_id != cc.get_id()
+            error('The C compiler is @0@, but the C++ compiler is @1@. '.format(
+                      cc.get_id(), cxx_id) +
+                  'Sanitizer and fuzzing builds need both from the same ' +
+                  'toolchain, e.g. CC=clang CXX=clang++')
+        endif
+    endif
+    # clang links the sanitizer runtime only into executables, so a shared
+    # library has undefined symbols, which -Wl,--no-undefined rejects.
+    if cc.get_id() == 'clang' and get_option('b_lundef')
+        error('Sanitizer and fuzzing builds with clang need -Db_lundef=false')
+    endif
+endif
+
 ################################################################################
 # Determine which features we need to build: The nvme executable, the libnvme
 # library, and/or the python bindings. These are controlled by the following
@@ -661,6 +682,12 @@ if get_option('fuzzing')
     if cc.get_id() != 'clang'
         error('-Dfuzzing=true needs clang')
     endif
+    # The coverage hooks of all code but the fuzz targets come from the
+    # sanitizer runtime.
+    if get_option('b_sanitize') == 'none'
+        error('-Dfuzzing=true needs a sanitizer, ' +
+              'e.g. -Db_sanitize=address,undefined')
+    endif
     add_project_arguments('-fsanitize=fuzzer-no-link', language : 'c')
 endif
 

scripts/build.sh

@@ -8,6 +8,8 @@
 
 set -e
 
+ALL_FUZZ_TARGETS="argconfig ini global-config fabrics-conf device"
+
 usage() {
     echo "Usage: build.sh [-b [release|debug]] "
     echo "                [-c [gcc|clang]]"
@@ -18,6 +20,8 @@ usage() {
     echo "                [--e2e-log-level level]"
     echo "                [--e2e-nvme-bin path]"
     echo "                [--plugin-tests plugin1,plugin2]"
+    echo "                [--fuzz-seconds n] [--fuzz-jobs n]"
+    echo "                [--fuzz-targets target1,target2] [--fuzz-dir dir]"
     echo "                [config]"
     echo ""
     echo "CI build script."
@@ -42,6 +46,14 @@ usage() {
     echo " --plugin-tests list     comma-separated list of plugin test suites to"
     echo "                         run against real hardware, e.g. micron,ocp"
     echo ""
+    echo "options for the 'fuzz' config:"
+    echo " --fuzz-seconds n        run each target for n seconds (600)"
+    echo " --fuzz-jobs n           run n libFuzzer jobs per target (all CPUs)"
+    echo " --fuzz-targets list     comma-separated list of targets (all):"
+    echo "                         ${ALL_FUZZ_TARGETS// /, }"
+    echo " --fuzz-dir dir          directory for the results (.build-ci/fuzz):"
+    echo "                         summary.txt, logs/, artifacts/ and corpus/"
+    echo ""
     echo "configs with meson:"
     echo "  [default]           default settings"
     echo "  libdbus             build with libdbus"
@@ -61,6 +73,8 @@ usage() {
     echo "  libnvme             build only libnvme"
     echo "  libnvme_abi         build only libnvme3.so, no tests/examples"
     echo "  tests               build for nightly build"
+    echo "  fuzz                build the fuzz targets for libFuzzer with clang"
+    echo "                      and run them, see the options above"
     echo ""
     echo "configs with muon:"
     echo "  [default]           minimal static build"
@@ -85,6 +99,17 @@ E2E_LOG_LEVEL=""
 E2E_NVME_BIN=""
 PLUGIN_TESTS=""
 
+fuzz_seconds=600
+fuzz_jobs=""
+fuzz_targets="${ALL_FUZZ_TARGETS}"
+fuzz_dir=""
+
+# The options which only one config takes, as "config option"
+CONFIG_OPTIONS=()
+config_option() {
+    CONFIG_OPTIONS+=("$1 $2")
+}
+
 ARGS=()
 while [[ $# -gt 0 ]]; do
     case "$1" in
@@ -134,26 +159,66 @@ while [[ $# -gt 0 ]]; do
             ;;
         --e2e-controller)
             E2E_CONTROLLER="$2"
+            config_option tests "$1"
             shift 2
             ;;
         --e2e-ns1)
             E2E_NS1="$2"
+            config_option tests "$1"
             shift 2
             ;;
         --e2e-log-dir)
             E2E_LOG_DIR="$2"
+            config_option tests "$1"
             shift 2
             ;;
         --e2e-log-level)
             E2E_LOG_LEVEL="$2"
+            config_option tests "$1"
             shift 2
             ;;
         --e2e-nvme-bin)
             E2E_NVME_BIN="$2"
+            config_option tests "$1"
             shift 2
             ;;
         --plugin-tests)
             PLUGIN_TESTS="$2"
+            config_option tests "$1"
+            shift 2
+            ;;
+        --fuzz-seconds|--fuzz-jobs)
+            if ! [[ "$2" =~ ^[1-9][0-9]*$ ]]; then
+                echo "error: $1 takes a positive number" >&2
+                exit 1
+            fi
+            if [[ "$1" == --fuzz-seconds ]]; then
+                fuzz_seconds="$2"
+            else
+                fuzz_jobs="$2"
+            fi
+            config_option fuzz "$1"
+            shift 2
+            ;;
+        --fuzz-targets)
+            fuzz_targets="${2//,/ }"
+            for t in ${fuzz_targets}; do
+                if [[ " ${ALL_FUZZ_TARGETS} " != *" ${t} "* ]]; then
+                    echo "error: unknown fuzz target '${t}'" >&2
+                    exit 1
+                fi
+            done
+            config_option fuzz "$1"
+            shift 2
+            ;;
+        --fuzz-dir)
+            # relative to where build.sh runs, not to the top directory
+            if [[ "$2" == /* ]]; then
+                fuzz_dir="$2"
+            else
+                fuzz_dir="$(pwd)/$2"
+            fi
+            config_option fuzz "$1"
             shift 2
             ;;
         -h|--help)
@@ -174,13 +239,12 @@ set -- "${ARGS[@]}"
 
 CONFIG=${1:-"default"}
 
-if [[ "${CONFIG}" != "tests" ]]; then
-    if [[ -n "${E2E_CONTROLLER}" || -n "${E2E_NS1}" || -n "${E2E_LOG_DIR}" || \
-          -n "${E2E_LOG_LEVEL}" || -n "${E2E_NVME_BIN}" || -n "${PLUGIN_TESTS}" ]]; then
-        echo "error: --e2e-* and --plugin-tests are only valid with the 'tests' config" >&2
+for opt in "${CONFIG_OPTIONS[@]}"; do
+    if [[ "${opt%% *}" != "${CONFIG}" ]]; then
+        echo "error: ${opt#* } is only valid with the '${opt%% *}' config" >&2
         exit 1
     fi
-fi
+done
 
 cd "$(git rev-parse --show-toplevel)" || exit 1
 
@@ -193,6 +257,12 @@ config_meson_default() {
     local extra_args=()
     if [ "${use_asan:-0}" -eq 1 ]; then
         extra_args+=(-Db_sanitize=address,undefined)
+        # The C++ tests need clang++ to match, and clang links the
+        # sanitizer runtime only into executables.
+        if [[ "${CC}" == *clang* ]]; then
+            export CXX="${CXX:-${CC/clang/clang++}}"
+            extra_args+=(-Db_lundef=false)
+        fi
     fi
     # Build nvme-keysd in CI. "enabled" fails without libsystemd >= 257.
     if pkg-config --atleast-version=257 libsystemd 2>/dev/null; then
@@ -462,6 +532,105 @@ test_meson_libnvme_abi() {
     :
 }
 
+# Coverage guided fuzzing with libFuzzer, see Documentation/TESTING.md.
+# The results are in the --fuzz-dir, by default in the build directory:
+# per target the corpus, the logs and the inputs which found a bug, and a
+# summary. A --fuzz-dir outside of it grows the corpus across runs.
+config_meson_fuzz() {
+    if [[ "${CC}" != *clang* ]]; then
+        CC=clang
+    fi
+    CXX="${CXX:-${CC/clang/clang++}}"
+
+    CC="${CC}" CXX="${CXX}" "${MESON}" setup    \
+        --werror                                \
+        --buildtype="${BUILDTYPE}"              \
+        -Dfuzzing=true                          \
+        -Db_sanitize=address,undefined          \
+        -Db_lundef=false                        \
+        "${BUILDDIR}"
+}
+
+test_meson_fuzz() {
+    local root
+    root="$(pwd)"
+    local dir="${fuzz_dir:-${BUILDDIR}/fuzz}"
+    local seconds="${fuzz_seconds}"
+    local jobs="${fuzz_jobs:-$(nproc 2>/dev/null || echo 1)}"
+    local failed=()
+    local t f exe corpus artifacts logs args status summary
+    # A device reports sizes of up to gigabytes: let ASan fail such an
+    # allocation instead of libFuzzer aborting as out of memory.
+    local asan="allocator_may_return_null=1:max_allocation_size_mb=1024"
+
+    mkdir -p "${dir}"
+    summary="${dir}/summary.txt"
+    : > "${summary}"
+
+    # A UBSan finding is a crash too
+    export UBSAN_OPTIONS="${UBSAN_OPTIONS:-halt_on_error=1:print_stacktrace=1}"
+
+    for t in ${fuzz_targets}; do
+        exe="${BUILDDIR}/tests/fuzz/fuzz-${t}"
+        if [ ! -x "${exe}" ]; then
+            echo "fuzz-${t}: not built, skipped"
+            continue
+        fi
+
+        corpus="${dir}/corpus/${t}"
+        artifacts="${dir}/artifacts/${t}"
+        logs="${dir}/logs/${t}"
+        rm -rf "${logs}"
+        mkdir -p "${corpus}" "${artifacts}" "${logs}"
+        # Add the seeds, but keep what earlier runs found
+        for f in "${root}"/tests/fuzz/corpus/"${t}"/*; do
+            [ -e "${corpus}/${f##*/}" ] || cp "${f}" "${corpus}/"
+        done
+
+        # Run the jobs as independent processes on the shared corpus. The
+        # -fork mode is newer, but the libFuzzer of clang 22 crashes in it.
+        args=(-close_fd_mask=3 -max_total_time="${seconds}"
+              -jobs="${jobs}" -workers="${jobs}"
+              -artifact_prefix="${artifacts}/")
+        case "${t}" in
+            argconfig)
+                args+=(-dict="${root}/tests/fuzz/argconfig.dict")
+                ;;
+            device)
+                args+=(-max_len=16384 -timeout=10)
+                ;;
+            *)
+                args+=(-dict="${root}/tests/fuzz/config.dict")
+                ;;
+        esac
+
+        echo "fuzz-${t}: ${seconds}s, ${jobs} jobs, logs in ${logs}"
+        # Each job writes fuzz-<job>.log to the current directory
+        status=0
+        (cd "${logs}" &&
+         ASAN_OPTIONS="${ASAN_OPTIONS:+${ASAN_OPTIONS}:}${asan}" \
+            "${exe}" "${args[@]}" "${corpus}" > main.log 2>&1) ||
+            status=$?
+        if [ "${status}" -ne 0 ]; then
+            failed+=("${t}")
+            echo "fuzz-${t}: finding, exit status ${status}" |
+                tee -a "${summary}"
+            {
+                grep -hE '^(SUMMARY|==[0-9]+==ERROR)' "${logs}"/fuzz-*.log ||
+                    true
+                ls "${artifacts}"/* 2>/dev/null || true
+            } >> "${summary}"
+        else
+            echo "fuzz-${t}: no finding in ${seconds}s" | tee -a "${summary}"
+        fi
+    done
+
+    if [ "${#failed[@]}" -gt 0 ]; then
+        echo "fuzz targets with findings: ${failed[*]}, see ${summary}"
+        return 1
+    fi
+}
+
 build_meson() {
     if [ "${use_analyzer:-0}" -eq 1 ]; then
         "${SCAN_BUILD}" -o "${BUILDDIR}/scan-results" \