From df76f5cf47a1303865f428a840357f178bc27c73 Mon Sep 17 00:00:00 2001 From: iphydf Date: Fri, 12 Jan 2024 10:27:48 +0000 Subject: [PATCH] chore: Move from gcov to llvm source-based coverage. --- .github/scripts/coverage-linux | 62 ---------- .github/scripts/flags-coverage.sh | 4 +- auto_tests/check_compat.h | 6 +- heroku.yml | 4 - other/docker/coverage/Dockerfile | 64 +++++------ other/docker/coverage/Dockerfile.nginx | 3 +- other/docker/coverage/run | 2 +- other/docker/coverage/run_mallocfail | 34 ++++-- other/docker/coverage/syscall_funcs.c | 149 ------------------------- 9 files changed, 60 insertions(+), 268 deletions(-) delete mode 100755 .github/scripts/coverage-linux delete mode 100644 heroku.yml delete mode 100644 other/docker/coverage/syscall_funcs.c diff --git a/.github/scripts/coverage-linux b/.github/scripts/coverage-linux deleted file mode 100755 index 4b17fdf5..00000000 --- a/.github/scripts/coverage-linux +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -set -eu - -NPROC=$(nproc) - -sudo apt-get install -y --no-install-recommends \ - libgmock-dev \ - libgtest-dev \ - libopus-dev \ - libsodium-dev \ - libvpx-dev \ - llvm-14 \ - ninja-build -git clone --depth=1 https://github.com/ralight/mallocfail /tmp/mallocfail -cd /tmp/mallocfail # pushd -make -sudo make install -cd - # popd - -export CC=clang -export CXX=clang++ - -sudo install other/docker/coverage/run_mallocfail /usr/local/bin/run_mallocfail -(cd other/proxy && go get && go build) -other/proxy/proxy & - -. ".github/scripts/flags-coverage.sh" - -cmake -B_build -H. -GNinja \ - -DCMAKE_C_FLAGS="$C_FLAGS" \ - -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \ - -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \ - -DCMAKE_SHARED_LINKER_FLAGS="$LD_FLAGS" \ - -DCMAKE_INSTALL_PREFIX:PATH="$PWD/_install" \ - -DENABLE_SHARED=OFF \ - -DMIN_LOGGER_LEVEL=TRACE \ - -DMUST_BUILD_TOXAV=ON \ - -DNON_HERMETIC_TESTS=OFF \ - -DSTRICT_ABI=ON \ - -DTEST_TIMEOUT_SECONDS=120 \ - -DUSE_IPV6=OFF \ - -DAUTOTEST=ON \ - -DPROXY_TEST=ON - -cmake --build _build --parallel "$NPROC" --target install -- -k 0 - -cd _build # pushd -ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6 || - ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6 - -export PYTHONUNBUFFERED=1 -run_mallocfail --ctest=2 --jobs=8 -cd - # popd - -#coveralls \ -# --exclude auto_tests \ -# --exclude other \ -# --exclude testing \ -# --gcov-options '\-lp' - -bash <(curl -s https://codecov.io/bash) -x "llvm-cov-14 gcov" diff --git a/.github/scripts/flags-coverage.sh b/.github/scripts/flags-coverage.sh index ebfa2b1c..6e848842 100644 --- a/.github/scripts/flags-coverage.sh +++ b/.github/scripts/flags-coverage.sh @@ -5,10 +5,10 @@ add_ld_flag -Wl,-z,defs # Make compilation error on a warning -add_flag -Werror +add_flag -Werror -Wno-unsafe-buffer-usage # Coverage flags. -add_flag --coverage +add_flag -fprofile-instr-generate -fcoverage-mapping # Optimisation, but keep stack traces useful. add_c_flag -fno-inline -fno-omit-frame-pointer diff --git a/auto_tests/check_compat.h b/auto_tests/check_compat.h index 72fa55bc..f60bc7ae 100644 --- a/auto_tests/check_compat.h +++ b/auto_tests/check_compat.h @@ -9,7 +9,7 @@ #define ck_assert(ok) do { \ if (!(ok)) { \ fprintf(stderr, "%s:%d: failed `%s'\n", __FILE__, __LINE__, #ok); \ - abort(); \ + exit(7); \ } \ } while (0) @@ -18,7 +18,7 @@ fprintf(stderr, "%s:%d: failed `%s': ", __FILE__, __LINE__, #ok); \ fprintf(stderr, __VA_ARGS__); \ fprintf(stderr, "\n"); \ - abort(); \ + exit(7); \ } \ } while (0) @@ -26,7 +26,7 @@ fprintf(stderr, "%s:%d: ", __FILE__, __LINE__); \ fprintf(stderr, __VA_ARGS__); \ fprintf(stderr, "\n"); \ - abort(); \ + exit(7); \ } while (0) #endif // C_TOXCORE_AUTO_TESTS_CHECK_COMPAT_H diff --git a/heroku.yml b/heroku.yml deleted file mode 100644 index 96fee696..00000000 --- a/heroku.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -build: - docker: - web: other/bootstrap_daemon/websocket/Dockerfile diff --git a/other/docker/coverage/Dockerfile b/other/docker/coverage/Dockerfile index 7dcd635b..ba292ba2 100644 --- a/other/docker/coverage/Dockerfile +++ b/other/docker/coverage/Dockerfile @@ -1,56 +1,57 @@ FROM toxchat/c-toxcore:sources AS src FROM ubuntu:20.04 AS build +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + RUN apt-get update && \ DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ ca-certificates \ - clang \ - cmake \ curl \ - gcc \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main" >> /etc/apt/sources.list \ + && curl -L https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc \ + && apt-get update && \ + DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ + clang-17 \ + cmake \ git \ golang-1.18 \ + libclang-rt-17-dev \ libconfig-dev \ libgmock-dev \ libgtest-dev \ libopus-dev \ libsodium-dev \ + libunwind-17-dev \ libvpx-dev \ - llvm-dev \ + lld-17 \ + llvm-17-dev \ make \ ninja-build \ pkg-config \ - python3-lxml \ - python3-pip \ - python3-pygments \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && pip3 install --no-cache-dir gcovr -# strip gtest/gmock so it doesn't end up in stack traces. This speeds up the -# mallocfail run below by a lot. -RUN ["strip", "-g",\ - "/usr/lib/x86_64-linux-gnu/libgmock.a",\ - "/usr/lib/x86_64-linux-gnu/libgmock_main.a",\ - "/usr/lib/x86_64-linux-gnu/libgtest.a",\ - "/usr/lib/x86_64-linux-gnu/libgtest_main.a"] + && rm -rf /var/lib/apt/lists/* RUN ["curl", "-s", "https://codecov.io/bash", "-o", "/usr/local/bin/codecov"] RUN ["chmod", "+x", "/usr/local/bin/codecov"] -ENV CC=clang \ - CXX=clang++ \ +ENV CC=clang-17 \ + CXX=clang++-17 \ PYTHONUNBUFFERED=1 \ PATH=$PATH:/usr/lib/go-1.18/bin -SHELL ["/bin/bash", "-c"] + +COPY --from=src /src/ /work/ WORKDIR /work -COPY --from=src /src/ /work/ +RUN git clone --depth=1 https://github.com/TokTok/toktok-fuzzer /work/testing/fuzzing/toktok-fuzzer RUN source .github/scripts/flags-coverage.sh \ && go version \ && (cd other/proxy && go get github.com/things-go/go-socks5 && go build proxy_server.go) \ && cmake -B_build -H. -GNinja \ -DCMAKE_C_FLAGS="$C_FLAGS" \ -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \ - -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \ + -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS -fuse-ld=lld" \ -DCMAKE_UNITY_BUILD=ON \ -DENABLE_SHARED=OFF \ -DMIN_LOGGER_LEVEL=TRACE \ @@ -69,24 +70,15 @@ RUN /work/other/proxy/proxy_server \ ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6) WORKDIR /work/mallocfail -RUN ["git", "clone", "--depth=1", "https://github.com/ralight/mallocfail", "/work/mallocfail"] -COPY other/docker/coverage/syscall_funcs.c src/ -RUN gcc -fPIC -shared -O2 -g3 -Wall -Ideps/uthash -Ideps/sha3 deps/*/*.c src/*.c -o mallocfail.so -ldl -lbacktrace \ +RUN ["git", "clone", "--depth=1", "https://github.com/TokTok/mallocfail", "/work/mallocfail"] +RUN clang-17 -fuse-ld=lld -fPIC -shared -O2 -g3 -Wall -I/usr/lib/llvm-17/include -L/usr/lib/llvm-17/lib -Ideps/uthash -Ideps/sha3 deps/*/*.c src/*.c -o mallocfail.so -ldl -lunwind \ && install mallocfail.so /usr/local/lib/mallocfail.so WORKDIR /work/_build COPY other/docker/coverage/run_mallocfail /usr/local/bin/ -RUN ["run_mallocfail", "--ctest=2", "--jobs=8"] -RUN ["gcovr", \ - "--sort-percentage", \ - "--gcov-executable=llvm-cov gcov", \ - "--html-details=html/", \ - "--root=..", \ - "--exclude=CMakeFiles/", \ - "--exclude=_deps/", \ - "--exclude=(.+/)?auto_tests/", \ - "--exclude=.+_test.cc?$", \ - "--exclude=(.+/)?other/", \ - "--exclude=(.+/)?testing/"] +RUN ["run_mallocfail", "--ctest=1", "--jobs=8"] +RUN llvm-profdata-17 merge -sparse $(find . -name "*.profraw") -o toxcore.profdata +RUN llvm-cov-17 show -format=text -instr-profile=toxcore.profdata -sources $(cmake --build . --target help | grep -o '[^:]*_test:' | grep -o '[^:]*' | xargs -n1 find . -type f -name | awk '{print "-object "$1}') > coverage.txt +RUN llvm-cov-17 show -format=html -instr-profile=toxcore.profdata -sources $(cmake --build . --target help | grep -o '[^:]*_test:' | grep -o '[^:]*' | xargs -n1 find . -type f -name | awk '{print "-object "$1}') -output-dir=html WORKDIR /work diff --git a/other/docker/coverage/Dockerfile.nginx b/other/docker/coverage/Dockerfile.nginx index 5e085978..b858dd26 100644 --- a/other/docker/coverage/Dockerfile.nginx +++ b/other/docker/coverage/Dockerfile.nginx @@ -1,5 +1,4 @@ # vim:ft=dockerfile FROM toxchat/c-toxcore:coverage AS build FROM nginx:alpine -COPY --from=build /work/_build/html/coverage_details.html /usr/share/nginx/html/index.html -COPY --from=build /work/_build/html/ /usr/share/nginx/html/ +COPY --from=build --chown=nginx:nginx /work/_build/html/ /usr/share/nginx/html/ diff --git a/other/docker/coverage/run b/other/docker/coverage/run index 368d8429..aea7603c 100755 --- a/other/docker/coverage/run +++ b/other/docker/coverage/run @@ -6,4 +6,4 @@ read -a ci_env <<<"$(bash <(curl -s https://codecov.io/env))" docker build -t toxchat/c-toxcore:sources -f other/docker/sources/Dockerfile . docker build -t toxchat/c-toxcore:coverage -f other/docker/coverage/Dockerfile . -docker run "${ci_env[@]}" -e CI=true --name toxcore-coverage --rm -t toxchat/c-toxcore:coverage /usr/local/bin/codecov -x "llvm-cov gcov" +docker run "${ci_env[@]}" -e CI=true --name toxcore-coverage --rm -t toxchat/c-toxcore:coverage /usr/local/bin/codecov diff --git a/other/docker/coverage/run_mallocfail b/other/docker/coverage/run_mallocfail index d75e5f73..2604e961 100755 --- a/other/docker/coverage/run_mallocfail +++ b/other/docker/coverage/run_mallocfail @@ -27,11 +27,12 @@ from typing import NoReturn from typing import Optional from typing import Tuple -_PRIMER = "./unit_util_test" +_PRIMER = "auto_tests/auto_version_test" _MALLOCFAIL_SO = "/usr/local/lib/mallocfail.so" _HASHES = "mallocfail_hashes" _HASHES_PREV = "mallocfail_hashes.prev" _TIMEOUT = 3.0 +_BUILD_DIR = os.getcwd() _ENV = { "LD_PRELOAD": _MALLOCFAIL_SO, @@ -45,10 +46,19 @@ def run_mallocfail(tmpdir: str, timeout: float, exe: str, iteration: int, print(f"\x1b[1;33mmallocfail '{exe}' run #{iteration}\x1b[0m") hashes = os.path.join(tmpdir, _HASHES) hashes_prev = os.path.join(tmpdir, _HASHES_PREV) + profraw = os.path.join(_BUILD_DIR, "mallocfail.out", exe, "%p.profraw") if os.path.exists(hashes): shutil.copy(hashes, hashes_prev) try: - proc = subprocess.run([exe], timeout=timeout, env=_ENV, cwd=tmpdir) + proc = subprocess.run( + [exe], + timeout=timeout, + env={ + "LLVM_PROFILE_FILE": profraw, + **_ENV, + }, + cwd=tmpdir, + ) except subprocess.TimeoutExpired: print(f"\x1b[1;34mProgram {exe} timed out\x1b[0m") return True @@ -65,13 +75,16 @@ def run_mallocfail(tmpdir: str, timeout: float, exe: str, iteration: int, # Process exited cleanly (success or failure). pass elif proc.returncode == -6: - # Assertion failed. + # abort(), we allow it. + pass + elif proc.returncode == 7: + # ck_assert failed, also fine for us. pass elif proc.returncode == -14: print(f"\x1b[0;34mProgram '{exe}' timed out\x1b[0m") else: print( - f"\x1b[1;32mProgram '{exe}' failed to handle OOM situation cleanly\x1b[0m" + f"\x1b[1;32mProgram '{exe}' failed to handle OOM situation cleanly (code {proc.returncode})\x1b[0m" ) if not keep_going: raise Exception("Aborting test") @@ -96,8 +109,8 @@ def find_prog(name: str) -> Tuple[Optional[str], ...]: return path return None - return (attempt(f"./unit_{name}_test"), - attempt(f"auto_tests/auto_{name}_test")) + return (attempt(f"auto_tests/auto_{name}_test"), + ) # attempt(f"./unit_{name}_test"), def parse_flags(args: List[str]) -> Tuple[Dict[str, str], List[str]]: @@ -128,6 +141,9 @@ def isolated_mallocfail(timeout: int, exe: str) -> None: shutil.copy(exe, os.path.join(tmpdir, exe)) shutil.copy(_HASHES, os.path.join(tmpdir, _HASHES)) loop_mallocfail(tmpdir, timeout, exe) + profraw = os.path.join(tmpdir, "default.profraw") + if os.path.exists(profraw): + shutil.copy(profraw, exe + ".mallocfail.profraw") def main(args: List[str]) -> None: @@ -150,12 +166,12 @@ def main(args: List[str]) -> None: else: jobs = 1 - # Start by running util_test, which allocates no memory of its own, just + # Start by running version_test, which allocates no memory of its own, just # to prime the mallocfail hashes so it doesn't make global initialisers # such as llvm_gcov_init fail. if os.path.exists(_PRIMER): - print(f"\x1b[1;33mPriming hashes with unit_util_test\x1b[0m") - loop_mallocfail(".", timeout, _PRIMER, keep_going=True) + print(f"\x1b[1;33mPriming hashes with {_PRIMER}\x1b[0m") + loop_mallocfail(os.getcwd(), timeout, _PRIMER, keep_going=True) print(f"\x1b[1;33m--------------------------------\x1b[0m") print(f"\x1b[1;33mStarting mallocfail for {len(exes)} programs:\x1b[0m") diff --git a/other/docker/coverage/syscall_funcs.c b/other/docker/coverage/syscall_funcs.c deleted file mode 100644 index 3d768571..00000000 --- a/other/docker/coverage/syscall_funcs.c +++ /dev/null @@ -1,149 +0,0 @@ -#define _GNU_SOURCE - -#include "mallocfail.h" - -#include -#include -#include -#include -#include -#include -#include - -static int (*libc_ioctl)(int fd, unsigned long request, ...); -static int (*libc_bind)(int sockfd, const struct sockaddr *addr, - socklen_t addrlen); -static int (*libc_getsockopt)(int sockfd, int level, int optname, - void *optval, socklen_t *optlen); -static int (*libc_setsockopt)(int sockfd, int level, int optname, - const void *optval, socklen_t optlen); -static ssize_t (*libc_recv)(int sockfd, void *buf, size_t len, int flags); -static ssize_t (*libc_recvfrom)(int sockfd, void *buf, size_t len, int flags, - struct sockaddr *src_addr, socklen_t *addrlen); -static ssize_t (*libc_send)(int sockfd, const void *buf, size_t len, int flags); -static ssize_t(*libc_sendto)(int sockfd, const void *buf, size_t len, int flags, - const struct sockaddr *dest_addr, socklen_t addrlen); -static int (*libc_socket)(int domain, int type, int protocol); -static int (*libc_listen)(int sockfd, int backlog); - -__attribute__((__constructor__)) -static void init(void) -{ - libc_ioctl = dlsym(RTLD_NEXT, "ioctl"); - libc_bind = dlsym(RTLD_NEXT, "bind"); - libc_getsockopt = dlsym(RTLD_NEXT, "getsockopt"); - libc_setsockopt = dlsym(RTLD_NEXT, "setsockopt"); - libc_recv = dlsym(RTLD_NEXT, "recv"); - libc_recvfrom = dlsym(RTLD_NEXT, "recvfrom"); - libc_send = dlsym(RTLD_NEXT, "send"); - libc_sendto = dlsym(RTLD_NEXT, "sendto"); - libc_socket = dlsym(RTLD_NEXT, "socket"); - libc_listen = dlsym(RTLD_NEXT, "listen"); -} - -int ioctl(int fd, unsigned long request, ...) -{ - if (should_malloc_fail()) { - errno = ENOMEM; - return -1; - } - - va_list ap; - va_start(ap, request); - const int ret = libc_ioctl(fd, SIOCGIFCONF, va_arg(ap, void *)); - va_end(ap); - return ret; -} - -int bind(int sockfd, const struct sockaddr *addr, - socklen_t addrlen) -{ - // Unlike all others, if bind should fail once, it should fail always, because in toxcore we try - // many ports before giving up. If this only fails once, we'll never reach the code path where - // we give up. - static int should_fail = -1; - if (should_fail == -1) { - should_fail = should_malloc_fail(); - } - if (should_fail) { - errno = ENOMEM; - return -1; - } - return libc_bind(sockfd, addr, addrlen); -} - -int getsockopt(int sockfd, int level, int optname, - void *optval, socklen_t *optlen) -{ - if (should_malloc_fail()) { - errno = ENOMEM; - return -1; - } - return libc_getsockopt(sockfd, level, optname, optval, optlen); -} - -int setsockopt(int sockfd, int level, int optname, - const void *optval, socklen_t optlen) -{ - if (should_malloc_fail()) { - errno = ENOMEM; - return -1; - } - return libc_setsockopt(sockfd, level, optname, optval, optlen); -} - -ssize_t recv(int sockfd, void *buf, size_t len, int flags) -{ - if (should_malloc_fail()) { - errno = ENOMEM; - return -1; - } - return libc_recv(sockfd, buf, len, flags); -} - -ssize_t recvfrom(int sockfd, void *buf, size_t len, int flags, - struct sockaddr *src_addr, socklen_t *addrlen) -{ - if (should_malloc_fail()) { - errno = ENOMEM; - return -1; - } - return libc_recvfrom(sockfd, buf, len, flags, src_addr, addrlen); -} - -ssize_t send(int sockfd, const void *buf, size_t len, int flags) -{ - if (should_malloc_fail()) { - errno = ENOMEM; - return -1; - } - return libc_send(sockfd, buf, len, flags); -} - -ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, - const struct sockaddr *dest_addr, socklen_t addrlen) -{ - if (should_malloc_fail()) { - errno = ENOMEM; - return -1; - } - return libc_sendto(sockfd, buf, len, flags, dest_addr, addrlen); -} - -int socket(int domain, int type, int protocol) -{ - if (should_malloc_fail()) { - errno = ENOMEM; - return -1; - } - return libc_socket(domain, type, protocol); -} - -int listen(int sockfd, int backlog) -{ - if (should_malloc_fail()) { - errno = ENOMEM; - return -1; - } - return libc_listen(sockfd, backlog); -}