diff --git a/Documentation/TESTING.md b/Documentation/TESTING.md index 417415ef0a..54484ceca1 100644 --- a/Documentation/TESTING.md +++ b/Documentation/TESTING.md @@ -225,6 +225,15 @@ meson test -C .build-asanubsan --setup asanubsan No wrapper binary is needed — sanitizer reports are emitted directly to stderr when a violation is detected. +With clang, set the C++ compiler too, and let shared libraries have the +symbols of the sanitizer runtime undefined: clang links the runtime only into +executables. + +```bash +CC=clang CXX=clang++ meson setup .build-asanubsan \ + -Db_sanitize=address,undefined -Db_lundef=false +``` + The same known limitations apply as for Valgrind: Python tests and shell script tests may produce false positives. Focus triage on the `nvme-cli - *` unit tests. @@ -255,25 +264,55 @@ FUZZ_SECONDS=60 FUZZ_SEED=7 meson test -C .build --suite fuzz Run in a sanitizer build (see above) they also catch memory errors which don't crash. -For longer, coverage guided runs, build the targets for libFuzzer with clang: +For longer, coverage guided runs, build the targets for libFuzzer with clang, +in a sanitizer build as described above: ```bash -CC=clang meson setup .build-fuzz -Dfuzzing=true -Db_sanitize=address,undefined +CC=clang CXX=clang++ meson setup .build-fuzz -Dfuzzing=true \ + -Db_sanitize=address,undefined -Db_lundef=false meson compile -C .build-fuzz -mkdir corpus && cp tests/fuzz/corpus/argconfig/* corpus/ +``` + +Run a target on a copy of its seeds in the build directory, so the seeds in +the tree stay as they are. The new inputs libFuzzer finds go to the copy: + +```bash +mkdir -p .build-fuzz/corpus/argconfig +cp tests/fuzz/corpus/argconfig/* .build-fuzz/corpus/argconfig/ .build-fuzz/tests/fuzz/fuzz-argconfig -close_fd_mask=3 \ - -dict=tests/fuzz/argconfig.dict corpus + -dict=tests/fuzz/argconfig.dict .build-fuzz/corpus/argconfig +``` + +The config targets (`ini`, `global-config`, `fabrics-conf`) use +`tests/fuzz/config.dict`. The `device` target reads log pages, so give it +longer inputs. A device reports sizes of up to gigabytes, which nvme-cli +allocates: let ASan fail such an allocation instead of libFuzzer aborting the +run as out of memory: + +```bash +mkdir -p .build-fuzz/corpus/device +cp tests/fuzz/corpus/device/* .build-fuzz/corpus/device/ +ASAN_OPTIONS=allocator_may_return_null=1:max_allocation_size_mb=1024 \ + .build-fuzz/tests/fuzz/fuzz-device -close_fd_mask=3 -max_len=16384 \ + -timeout=10 .build-fuzz/corpus/device ``` -The config targets use `tests/fuzz/config.dict`. The `device` target -reads log pages, so give it longer inputs, and let allocations of the sizes -a device reports fail instead of aborting: +`scripts/build.sh fuzz` does all of this for every target in turn, each with +one libFuzzer job per CPU on a shared corpus, for 600 seconds by default: ```bash -ASAN_OPTIONS=allocator_may_return_null=1 .build-fuzz/tests/fuzz/fuzz-device \ - -close_fd_mask=3 -max_len=16384 -timeout=10 corpus +scripts/build.sh --fuzz-seconds 3600 fuzz +scripts/build.sh --fuzz-targets device --fuzz-jobs 8 fuzz ``` - The targets use the -libFuzzer interface, so AFL++ can run them too (`afl-clang-fast` with -`-fsanitize=fuzzer`). Add an input which found a bug to the seeds of its -target. + +It writes the results to `.build-ci/fuzz`: `summary.txt`, and per target the +logs of the jobs in `logs//`, the inputs which found a bug in +`artifacts//` and the corpus in `corpus//`. A finding stops +only its job, a UBSan finding like a crash. The next run of `build.sh` removes +the results; pass a `--fuzz-dir` outside of `.build-ci` to keep them, and to +grow the corpus across runs. `scripts/build.sh -h` lists all options. + +When run by hand, a crashing input is written to `crash-*` in the current +directory. Add an input which found a bug to the seeds of its target. The +targets use the libFuzzer interface, so AFL++ can run them too +(`afl-clang-fast` with `-fsanitize=fuzzer`). diff --git a/meson.build b/meson.build index ea30d51ab5..08660288f5 100644 --- a/meson.build +++ b/meson.build @@ -35,6 +35,27 @@ cc = meson.get_compiler('c') cxx_available = add_languages('cpp', required: false, native: false) host_system = host_machine.system() +# Sanitizer and fuzzing builds fail late, with misleading linker errors, +# in some setups. Catch them here. +if get_option('b_sanitize') != 'none' or get_option('fuzzing') + # The C++ tests link with the C++ compiler. With a different toolchain + # than the C compiler, the sanitizer runtimes do not match. + if cxx_available + cxx_id = meson.get_compiler('cpp').get_id() + if cxx_id != cc.get_id() + error('The C compiler is @0@, but the C++ compiler is @1@. '.format( + cc.get_id(), cxx_id) + + 'Sanitizer and fuzzing builds need both from the same ' + + 'toolchain, e.g. CC=clang CXX=clang++') + endif + endif + # clang links the sanitizer runtime only into executables, so a shared + # library has undefined symbols, which -Wl,--no-undefined rejects. + if cc.get_id() == 'clang' and get_option('b_lundef') + error('Sanitizer and fuzzing builds with clang need -Db_lundef=false') + endif +endif + ################################################################################ # Determine which features we need to build: The nvme executable, the libnvme # library, and/or the python bindings. These are controlled by the following @@ -661,6 +682,12 @@ if get_option('fuzzing') if cc.get_id() != 'clang' error('-Dfuzzing=true needs clang') endif + # The coverage hooks of all code but the fuzz targets come from the + # sanitizer runtime. + if get_option('b_sanitize') == 'none' + error('-Dfuzzing=true needs a sanitizer, ' + + 'e.g. -Db_sanitize=address,undefined') + endif add_project_arguments('-fsanitize=fuzzer-no-link', language : 'c') endif diff --git a/scripts/build.sh b/scripts/build.sh index ad7589100a..03f1baa3ac 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -8,6 +8,8 @@ set -e +ALL_FUZZ_TARGETS="argconfig ini global-config fabrics-conf device" + usage() { echo "Usage: build.sh [-b [release|debug]] " echo " [-c [gcc|clang]]" @@ -18,6 +20,8 @@ usage() { echo " [--e2e-log-level level]" echo " [--e2e-nvme-bin path]" echo " [--plugin-tests plugin1,plugin2]" + echo " [--fuzz-seconds n] [--fuzz-jobs n]" + echo " [--fuzz-targets target1,target2] [--fuzz-dir dir]" echo " [config]" echo "" echo "CI build script." @@ -42,6 +46,14 @@ usage() { echo " --plugin-tests list comma-separated list of plugin test suites to" echo " run against real hardware, e.g. micron,ocp" echo "" + echo "options for the 'fuzz' config:" + echo " --fuzz-seconds n run each target for n seconds (600)" + echo " --fuzz-jobs n run n libFuzzer jobs per target (all CPUs)" + echo " --fuzz-targets list comma-separated list of targets (all):" + echo " ${ALL_FUZZ_TARGETS// /, }" + echo " --fuzz-dir dir directory for the results (.build-ci/fuzz):" + echo " summary.txt, logs/, artifacts/ and corpus/" + echo "" echo "configs with meson:" echo " [default] default settings" echo " libdbus build with libdbus" @@ -61,6 +73,8 @@ usage() { echo " libnvme build only libnvme" echo " libnvme_abi build only libnvme3.so, no tests/examples" echo " tests build for nightly build" + echo " fuzz build the fuzz targets for libFuzzer with clang" + echo " and run them, see the options above" echo "" echo "configs with muon:" echo " [default] minimal static build" @@ -85,6 +99,17 @@ E2E_LOG_LEVEL="" E2E_NVME_BIN="" PLUGIN_TESTS="" +fuzz_seconds=600 +fuzz_jobs="" +fuzz_targets="${ALL_FUZZ_TARGETS}" +fuzz_dir="" + +# The options which only one config takes, as "config option" +CONFIG_OPTIONS=() +config_option() { + CONFIG_OPTIONS+=("$1 $2") +} + ARGS=() while [[ $# -gt 0 ]]; do case "$1" in @@ -134,26 +159,66 @@ while [[ $# -gt 0 ]]; do ;; --e2e-controller) E2E_CONTROLLER="$2" + config_option tests "$1" shift 2 ;; --e2e-ns1) E2E_NS1="$2" + config_option tests "$1" shift 2 ;; --e2e-log-dir) E2E_LOG_DIR="$2" + config_option tests "$1" shift 2 ;; --e2e-log-level) E2E_LOG_LEVEL="$2" + config_option tests "$1" shift 2 ;; --e2e-nvme-bin) E2E_NVME_BIN="$2" + config_option tests "$1" shift 2 ;; --plugin-tests) PLUGIN_TESTS="$2" + config_option tests "$1" + shift 2 + ;; + --fuzz-seconds|--fuzz-jobs) + if ! [[ "$2" =~ ^[1-9][0-9]*$ ]]; then + echo "error: $1 takes a positive number" >&2 + exit 1 + fi + if [[ "$1" == --fuzz-seconds ]]; then + fuzz_seconds="$2" + else + fuzz_jobs="$2" + fi + config_option fuzz "$1" + shift 2 + ;; + --fuzz-targets) + fuzz_targets="${2//,/ }" + for t in ${fuzz_targets}; do + if [[ " ${ALL_FUZZ_TARGETS} " != *" ${t} "* ]]; then + echo "error: unknown fuzz target '${t}'" >&2 + exit 1 + fi + done + config_option fuzz "$1" + shift 2 + ;; + --fuzz-dir) + # relative to where build.sh runs, not to the top directory + if [[ "$2" == /* ]]; then + fuzz_dir="$2" + else + fuzz_dir="$(pwd)/$2" + fi + config_option fuzz "$1" shift 2 ;; -h|--help) @@ -174,13 +239,12 @@ set -- "${ARGS[@]}" CONFIG=${1:-"default"} -if [[ "${CONFIG}" != "tests" ]]; then - if [[ -n "${E2E_CONTROLLER}" || -n "${E2E_NS1}" || -n "${E2E_LOG_DIR}" || \ - -n "${E2E_LOG_LEVEL}" || -n "${E2E_NVME_BIN}" || -n "${PLUGIN_TESTS}" ]]; then - echo "error: --e2e-* and --plugin-tests are only valid with the 'tests' config" >&2 +for opt in "${CONFIG_OPTIONS[@]}"; do + if [[ "${opt%% *}" != "${CONFIG}" ]]; then + echo "error: ${opt#* } is only valid with the '${opt%% *}' config" >&2 exit 1 fi -fi +done cd "$(git rev-parse --show-toplevel)" || exit 1 @@ -193,6 +257,12 @@ config_meson_default() { local extra_args=() if [ "${use_asan:-0}" -eq 1 ]; then extra_args+=(-Db_sanitize=address,undefined) + # The C++ tests need clang++ to match, and clang links the + # sanitizer runtime only into executables. + if [[ "${CC}" == *clang* ]]; then + export CXX="${CXX:-${CC/clang/clang++}}" + extra_args+=(-Db_lundef=false) + fi fi # Build nvme-keysd in CI. "enabled" fails without libsystemd >= 257. if pkg-config --atleast-version=257 libsystemd 2>/dev/null; then @@ -462,6 +532,105 @@ test_meson_libnvme_abi() { : } +# Coverage guided fuzzing with libFuzzer, see Documentation/TESTING.md. +# The results are in the --fuzz-dir, by default in the build directory: +# per target the corpus, the logs and the inputs which found a bug, and a +# summary. A --fuzz-dir outside of it grows the corpus across runs. +config_meson_fuzz() { + if [[ "${CC}" != *clang* ]]; then + CC=clang + fi + CXX="${CXX:-${CC/clang/clang++}}" + + CC="${CC}" CXX="${CXX}" "${MESON}" setup \ + --werror \ + --buildtype="${BUILDTYPE}" \ + -Dfuzzing=true \ + -Db_sanitize=address,undefined \ + -Db_lundef=false \ + "${BUILDDIR}" +} + +test_meson_fuzz() { + local root + root="$(pwd)" + local dir="${fuzz_dir:-${BUILDDIR}/fuzz}" + local seconds="${fuzz_seconds}" + local jobs="${fuzz_jobs:-$(nproc 2>/dev/null || echo 1)}" + local failed=() + local t f exe corpus artifacts logs args status summary + # A device reports sizes of up to gigabytes: let ASan fail such an + # allocation instead of libFuzzer aborting as out of memory. + local asan="allocator_may_return_null=1:max_allocation_size_mb=1024" + + mkdir -p "${dir}" + summary="${dir}/summary.txt" + : > "${summary}" + + # A UBSan finding is a crash too + export UBSAN_OPTIONS="${UBSAN_OPTIONS:-halt_on_error=1:print_stacktrace=1}" + + for t in ${fuzz_targets}; do + exe="${BUILDDIR}/tests/fuzz/fuzz-${t}" + if [ ! -x "${exe}" ]; then + echo "fuzz-${t}: not built, skipped" + continue + fi + + corpus="${dir}/corpus/${t}" + artifacts="${dir}/artifacts/${t}" + logs="${dir}/logs/${t}" + rm -rf "${logs}" + mkdir -p "${corpus}" "${artifacts}" "${logs}" + # Add the seeds, but keep what earlier runs found + for f in "${root}"/tests/fuzz/corpus/"${t}"/*; do + [ -e "${corpus}/${f##*/}" ] || cp "${f}" "${corpus}/" + done + + # Run the jobs as independent processes on the shared corpus. The + # -fork mode is newer, but the libFuzzer of clang 22 crashes in it. + args=(-close_fd_mask=3 -max_total_time="${seconds}" + -jobs="${jobs}" -workers="${jobs}" + -artifact_prefix="${artifacts}/") + case "${t}" in + argconfig) + args+=(-dict="${root}/tests/fuzz/argconfig.dict") + ;; + device) + args+=(-max_len=16384 -timeout=10) + ;; + *) + args+=(-dict="${root}/tests/fuzz/config.dict") + ;; + esac + + echo "fuzz-${t}: ${seconds}s, ${jobs} jobs, logs in ${logs}" + # Each job writes fuzz-.log to the current directory + status=0 + (cd "${logs}" && + ASAN_OPTIONS="${ASAN_OPTIONS:+${ASAN_OPTIONS}:}${asan}" \ + "${exe}" "${args[@]}" "${corpus}" > main.log 2>&1) || + status=$? + if [ "${status}" -ne 0 ]; then + failed+=("${t}") + echo "fuzz-${t}: finding, exit status ${status}" | + tee -a "${summary}" + { + grep -hE '^(SUMMARY|==[0-9]+==ERROR)' "${logs}"/fuzz-*.log || + true + ls "${artifacts}"/* 2>/dev/null || true + } >> "${summary}" + else + echo "fuzz-${t}: no finding in ${seconds}s" | tee -a "${summary}" + fi + done + + if [ "${#failed[@]}" -gt 0 ]; then + echo "fuzz targets with findings: ${failed[*]}, see ${summary}" + return 1 + fi +} + build_meson() { if [ "${use_analyzer:-0}" -eq 1 ]; then "${SCAN_BUILD}" -o "${BUILDDIR}/scan-results" \ diff --git a/tests/fuzz/meson.build b/tests/fuzz/meson.build index 4d387be36e..c182fa5056 100644 --- a/tests/fuzz/meson.build +++ b/tests/fuzz/meson.build @@ -4,12 +4,11 @@ # config files and the data from the device. Without -Dfuzzing=true each # target runs as a test, with the seeds in corpus/ and a few # seconds of mutations of them (FUZZ_SECONDS, FUZZ_SEED). With -# -Dfuzzing=true and clang, the targets are built for libFuzzer: +# -Dfuzzing=true and clang, the targets are built for libFuzzer, see +# Documentation/TESTING.md: # -# CC=clang meson setup fuzz -Dfuzzing=true -Db_sanitize=address,undefined -# ninja -C fuzz -# fuzz/tests/fuzz/fuzz-argconfig -dict=tests/fuzz/argconfig.dict \ -# tests/fuzz/corpus/argconfig +# CC=clang CXX=clang++ meson setup fuzz -Dfuzzing=true \ +# -Db_sanitize=address,undefined -Db_lundef=false # # The targets use the libFuzzer interface, which AFL++ (afl-clang-fast # -fsanitize=fuzzer) runs as well.