From e04b890817855ce32a2a9c9089a94a6c76be3d49 Mon Sep 17 00:00:00 2001 From: sudden6 Date: Sun, 5 Dec 2021 13:11:20 +0100 Subject: [PATCH] add scripts to run the fuzzing process This adds scripts and Dockerfiles to run the fuzzing process standalone or with OSS-Fuzz/ClusterFuzzLite integrations. --- .clusterfuzzlite/Dockerfile | 22 +++++++ .clusterfuzzlite/build.sh | 21 ++++++ .hadolint.yaml | 1 + CMakeLists.txt | 20 +++--- other/analysis/gen-file.sh | 4 ++ testing/BUILD.bazel | 8 --- testing/Dockerfile | 67 +++++++++++++++++++ testing/coverage_live.sh | 7 ++ testing/distill_corpus.sh | 36 +++++++++++ testing/run_afl.sh | 126 ++++++++++++++++++++++++++++++++++-- 10 files changed, 287 insertions(+), 25 deletions(-) create mode 100644 .clusterfuzzlite/Dockerfile create mode 100644 .clusterfuzzlite/build.sh create mode 100644 testing/Dockerfile create mode 100755 testing/coverage_live.sh create mode 100755 testing/distill_corpus.sh diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile new file mode 100644 index 00000000..4421e6c5 --- /dev/null +++ b/.clusterfuzzlite/Dockerfile @@ -0,0 +1,22 @@ +# c-toxcore Clusterfuzzlite build environment + +# We want to use the latest tools always +FROM gcr.io/oss-fuzz-base/base-builder:latest + +RUN apt-get update && \ + apt-get -y install --no-install-suggests --no-install-recommends \ + cmake libtool autoconf automake pkg-config \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + + +RUN git clone --depth 1 --branch 1.0.18 https://github.com/jedisct1/libsodium libsodium +WORKDIR $SRC/libsodium +RUN ./autogen.sh && ./configure --enable-shared=no && make install + +# Copy your project's source code. +COPY . $SRC/c-toxcore +# Working directory for build.sh. +WORKDIR $SRC/c-toxcore +# Copy build.sh into $SRC dir. +COPY ./.clusterfuzzlite/build.sh $SRC/ diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh new file mode 100644 index 00000000..ae4abf6c --- /dev/null +++ b/.clusterfuzzlite/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash -eu + +# out of tree build +cd "$WORK" + +ls /usr/local/lib/ + +# Debug build for asserts +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER="$CC" \ + -DCMAKE_CXX_COMPILER="$CXX" \ + -DCMAKE_C_FLAGS="$CFLAGS" \ + -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ + -DCMAKE_EXE_LINKER_FLAGS="$LIB_FUZZING_ENGINE" \ + -DBUILD_TOXAV=OFF -DENABLE_SHARED=NO -DBUILD_FUZZ_TESTS=ON \ + -DDHT_BOOTSTRAP=OFF -DBOOTSTRAP_DAEMON=OFF "$SRC"/c-toxcore + +# build fuzzer target +cmake --build ./ --target bootstrap_fuzzer + +# copy to output files +cp "$WORK"/bootstrap_fuzzer "$OUT"/ diff --git a/.hadolint.yaml b/.hadolint.yaml index bc95a173..980f970e 100644 --- a/.hadolint.yaml +++ b/.hadolint.yaml @@ -4,3 +4,4 @@ ignored: - DL3008 - DL3013 - DL3018 + - DL3059 diff --git a/CMakeLists.txt b/CMakeLists.txt index 9249a745..edfd3b2a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -540,18 +540,18 @@ endif() # Enabling this breaks all other tests and no network connections will be possible option(BUILD_FUZZ_TESTS "Build fuzzing harnesses" OFF) if (BUILD_FUZZ_TESTS) - # For coverage tests - target_compile_definitions(toxcore_static PUBLIC "FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION") + # For coverage tests + target_compile_definitions(toxcore_static PUBLIC "FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION") - # Override network and random functions - add_library(fuzz_adapter testing/fuzzing/fuzz_adapter.c) + # Override network and random functions + add_library(fuzz_adapter testing/fuzzing/fuzz_adapter.c) - # Fuzzes the toxsave API - add_executable(toxsave_fuzzer testing/fuzzing/toxsave_harness.cc) - target_link_libraries(toxsave_fuzzer toxcore_static fuzz_adapter -fsanitize=fuzzer) + # Fuzzes the toxsave API + add_executable(toxsave_fuzzer testing/fuzzing/toxsave_harness.cc) + target_link_libraries(toxsave_fuzzer toxcore_static fuzz_adapter -fsanitize=fuzzer) - # Fuzzes the bootstrap process - add_executable(bootstrap_fuzzer testing/fuzzing/bootstrap_harness.cc) - target_link_libraries(bootstrap_fuzzer toxcore_static fuzz_adapter -fsanitize=fuzzer) + # Fuzzes the bootstrap process + add_executable(bootstrap_fuzzer testing/fuzzing/bootstrap_harness.cc) + target_link_libraries(bootstrap_fuzzer toxcore_static fuzz_adapter -fsanitize=fuzzer) endif() diff --git a/other/analysis/gen-file.sh b/other/analysis/gen-file.sh index f802d9a3..bf32c18b 100644 --- a/other/analysis/gen-file.sh +++ b/other/analysis/gen-file.sh @@ -7,6 +7,7 @@ CPPFLAGS+=("-Iother") CPPFLAGS+=("-Iother/bootstrap_daemon/src") CPPFLAGS+=("-Iother/fun") CPPFLAGS+=("-Itesting") +CPPFLAGS+=("-Itesting/fuzzing") CPPFLAGS+=("-Itesting/groupchats") CPPFLAGS+=("-Itoxcore") CPPFLAGS+=("-Itoxav") @@ -44,7 +45,9 @@ callmain() { put auto_tests/check_compat.h +# Include all C and C++ code FIND_QUERY="find . '-(' -name '*.c' -or -name '*.cc' '-)'" +# Excludes FIND_QUERY="$FIND_QUERY -and -not -wholename './_build/*'" FIND_QUERY="$FIND_QUERY -and -not -wholename './super_donators/*'" FIND_QUERY="$FIND_QUERY -and -not -name amalgamation.cc" @@ -52,6 +55,7 @@ FIND_QUERY="$FIND_QUERY -and -not -name av_test.c" FIND_QUERY="$FIND_QUERY -and -not -name dht_test.c" FIND_QUERY="$FIND_QUERY -and -not -name trace.cc" FIND_QUERY="$FIND_QUERY -and -not -name version_test.c" +FIND_QUERY="$FIND_QUERY -and -not -wholename './testing/fuzzing/*'" readarray -t FILES <<<"$(eval "$FIND_QUERY")" diff --git a/testing/BUILD.bazel b/testing/BUILD.bazel index e0faaf9c..aa546400 100644 --- a/testing/BUILD.bazel +++ b/testing/BUILD.bazel @@ -56,11 +56,3 @@ cc_binary( "//c-toxcore/toxcore", ], ) - -cc_binary( - name = "afl_toxsave", - srcs = ["afl_toxsave.c"], - deps = [ - "//c-toxcore/toxcore", - ], -) diff --git a/testing/Dockerfile b/testing/Dockerfile new file mode 100644 index 00000000..e71dfbbe --- /dev/null +++ b/testing/Dockerfile @@ -0,0 +1,67 @@ +# based on https://github.com/AFLplusplus/AFLplusplus/blob/stable/Dockerfile + +FROM ubuntu:20.04 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG DEBIAN_FRONTEND=noninteractive + +ENV NO_ARCH_OPT 1 + +RUN apt-get update && \ + apt-get -y install --no-install-suggests --no-install-recommends \ + automake \ + ninja-build \ + bison flex \ + build-essential \ + git \ + python3 python3-dev python3-setuptools python-is-python3 \ + libtool libtool-bin \ + libglib2.0-dev \ + wget vim jupp nano bash-completion less \ + apt-utils apt-transport-https ca-certificates gnupg dialog \ + libpixman-1-dev \ + gnuplot-nox \ + screen \ + cmake \ + parallel \ + libsodium-dev \ + ninja-build\ + && rm -rf /var/lib/apt/lists/* + +RUN echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-12 main" >> /etc/apt/sources.list && \ + wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - + +RUN echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu focal main" >> /etc/apt/sources.list && \ + apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 1E9377A2BA9EF27F + +RUN apt-get update && apt-get full-upgrade -y && \ + apt-get -y install --no-install-suggests --no-install-recommends \ + gcc-10 g++-10 gcc-10-plugin-dev gcc-10-multilib gcc-multilib gdb lcov \ + clang-12 clang-tools-12 libc++1-12 libc++-12-dev \ + libc++abi1-12 libc++abi-12-dev libclang1-12 libclang-12-dev \ + libclang-common-12-dev libclang-cpp12 libclang-cpp12-dev liblld-12 \ + liblld-12-dev liblldb-12 liblldb-12-dev libllvm12 libomp-12-dev \ + libomp5-12 lld-12 lldb-12 llvm-12 llvm-12-dev llvm-12-runtime llvm-12-tools \ + && rm -rf /var/lib/apt/lists/* + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 0 +RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 0 + +ENV LLVM_CONFIG=llvm-config-12 +ENV AFL_SKIP_CPUFREQ=1 +ENV AFL_TRY_AFFINITY=1 +ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 + +RUN git clone --depth=1 https://github.com/vanhauser-thc/afl-cov /afl-cov +WORKDIR /afl-cov +RUN make install + +RUN git clone --depth=1 https://github.com/AFLplusplus/AFLplusplus /AFLplusplus +WORKDIR /AFLplusplus +RUN export CC=gcc-10 && export CXX=g++-10 && make install + +RUN echo '. /etc/bash_completion' >> ~/.bashrc +RUN echo 'alias joe="joe --wordwrap --joe_state -nobackup"' >> ~/.bashrc +ENV IS_DOCKER="1" +ENV CMAKE_GENERATOR=Ninja diff --git a/testing/coverage_live.sh b/testing/coverage_live.sh new file mode 100755 index 00000000..aceadf41 --- /dev/null +++ b/testing/coverage_live.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Move to repo root +cd ../ + +# Run code coverage only on minized corpus to save time +afl-cov --cover-corpus -d ./_afl_out --overwrite --live --coverage-cmd "_cov_build/bootstrap_fuzzer @@" --code-dir ../ diff --git a/testing/distill_corpus.sh b/testing/distill_corpus.sh new file mode 100755 index 00000000..644a6ac8 --- /dev/null +++ b/testing/distill_corpus.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +HARNESS_BIN="../_afl_build/bootstrap_fuzzer" +COV_BIN="../_cov_build/bootstrap_fuzzer" +# move to repo root +cd ../ + +cd _afl_out/ + +# Perform corpus minimization +mkdir -p corpus-cmin +rm corpus-cmin/* + +afl-cmin -i fuzz0/queue/ -o corpus-cmin/ -- "$HARNESS_BIN" + +# Minimize each testcase +mkdir -p corpus-tmin +rm corpus-tmin/* + +# afl-tmin is VERY slow +# massive parallel bash piping for the rescue +find corpus-cmin/ -maxdepth 1 -type f | + parallel --bar --joblog ./parallel.log afl-tmin -i ./corpus-cmin/{/} -o ./corpus-tmin/{/} -- "$HARNESS_BIN" + +# in case the tmin-process was aborted, just copy non-minimized files +cp -n ./corpus-cmin/* ./corpus-tmin + +# hack to let afl-cov run code coverage on our minimal corpus + +rm -R corpus-cov +mkdir -p corpus-cov/queue + +cp corpus-tmin/* corpus-cov/queue + +# Run code coverage only on minized corpus to save time +afl-cov --cover-corpus -d ./corpus-cov --overwrite --coverage-cmd "$COV_BIN @@" --code-dir ../ diff --git a/testing/run_afl.sh b/testing/run_afl.sh index c7a3bbc6..232904dc 100755 --- a/testing/run_afl.sh +++ b/testing/run_afl.sh @@ -1,14 +1,126 @@ -#! /bin/sh +#!/bin/sh + +COMMON_CMAKE_OPTIONS="-DCMAKE_C_COMPILER=afl-clang-lto -DCMAKE_CXX_COMPILER=afl-clang-lto++ -DBUILD_TOXAV=OFF -DENABLE_SHARED=NO -DBUILD_FUZZ_TESTS=ON -DDHT_BOOTSTRAP=OFF -DBOOTSTRAP_DAEMON=OFF" # move to repo root cd ../ -rm -R _afl_build -mkdir _afl_build + +# build fuzzer target UBSAN +mkdir -p _afl_build_ubsan +cd _afl_build_ubsan + +export AFL_USE_UBSAN=1 + +# build c-toxcore using afl instrumentation +cmake -DCMAKE_BUILD_TYPE=Debug "$COMMON_CMAKE_OPTIONS" .. + +# build fuzzer target +cmake --build ./ --target bootstrap_fuzzer + +unset AFL_USE_UBSAN + +cd .. + +# build fuzzer target MSAN +mkdir -p _afl_build_msan +cd _afl_build_msan + +export AFL_USE_MSAN=1 + +# build c-toxcore using afl instrumentation +cmake -DCMAKE_BUILD_TYPE=Debug "$COMMON_CMAKE_OPTIONS" .. + +# build fuzzer target +cmake --build ./ --target bootstrap_fuzzer + +unset AFL_USE_MSAN + +cd .. + +# build fuzzer target ASAN +mkdir -p _afl_build_asan +cd _afl_build_asan + +export AFL_USE_ASAN=1 + +# build c-toxcore using afl instrumentation +cmake -DCMAKE_BUILD_TYPE=Debug "$COMMON_CMAKE_OPTIONS" .. + +# build fuzzer target +cmake --build ./ --target bootstrap_fuzzer + +unset AFL_USE_ASAN + +cd .. + +# build fuzzer target without sanitizers for afl-tmin +mkdir -p _afl_build cd _afl_build # build c-toxcore using afl instrumentation -cmake -DCMAKE_C_COMPILER=afl-clang -DBUILD_MISC_TESTS=ON .. -make +cmake -DCMAKE_BUILD_TYPE=Debug "$COMMON_CMAKE_OPTIONS" .. -# start fuzzing -afl-fuzz -i ../testing/afl_testdata/tox_saves/ -o afl_out/ ./afl_toxsave @@ +# build fuzzer target +cmake --build ./ --target bootstrap_fuzzer + +cd .. + +# build fuzzer target with CmpLog +mkdir -p _afl_build_cmplog +cd _afl_build_cmplog + +export AFL_LLVM_CMPLOG=1 + +# build c-toxcore using afl instrumentation +cmake -DCMAKE_BUILD_TYPE=Debug "$COMMON_CMAKE_OPTIONS" .. + +# build fuzzer target +cmake --build ./ --target bootstrap_fuzzer + +unset AFL_LLVM_CMPLOG + +cd .. + +# build fuzzer target for code coverage +mkdir -p _cov_build +cd _cov_build + +# build c-toxcore using afl instrumentation +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-fprofile-arcs -ftest-coverage" -DCMAKE_C_FLAGS="-fprofile-arcs -ftest-coverage" -DCMAKE_VERBOSE_MAKEFILE=ON "$COMMON_CMAKE_OPTIONS" .. + +# build fuzzer target +cmake --build ./ --target bootstrap_fuzzer + +# back to repo root +cd ../ + +# Create fuzzer working directory + +mkdir -p _afl_out + +AFL_ARGS='-i testing/afl_testdata/tox_bootstraps/ -o _afl_out' + +export AFL_IMPORT_FIRST=1 +export AFL_AUTORESUME=1 + +# faster startup +export AFL_FAST_CAL=1 + +echo "connect to the fuzzers using: screen -x fuzz" +echo "if fuzzing doesn't start execute the following as root:" +echo "" +echo "echo core >/proc/sys/kernel/core_pattern" +echo "echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor" + +# Main fuzzer, keeps complete corpus +screen -dmS fuzz afl-fuzz -M fuzz0 "$AFL_ARGS" -c ./_afl_build_cmplog/bootstrap_fuzzer ./_afl_build/bootstrap_fuzzer +sleep 10s + +# Secondary fuzzers +screen -S fuzz -X screen afl-fuzz -S fuzz1 "$AFL_ARGS" -- ./_afl_build_msan/bootstrap_fuzzer +sleep 1s + +screen -S fuzz -X screen afl-fuzz -S fuzz2 "$AFL_ARGS" ./_afl_build_ubsan/bootstrap_fuzzer +sleep 1s + +screen -S fuzz -X screen afl-fuzz -S fuzz3 "$AFL_ARGS" ./_afl_build_asan/bootstrap_fuzzer