chore: Move from gcov to llvm source-based coverage.

2024-03-22 13:30:51 +08:00 · 2024-01-12 10:27:48 +00:00 · 2024-01-12 10:27:48 +00:00 · df76f5cf47
commit df76f5cf47
parent 072e3beb3f
9 changed files with 60 additions and 268 deletions
--- a/.github/scripts/coverage-linux
+++ b/.github/scripts/coverage-linux
@ -1,62 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-NPROC=$(nproc)
-
-sudo apt-get install -y --no-install-recommends \
-  libgmock-dev \
-  libgtest-dev \
-  libopus-dev \
-  libsodium-dev \
-  libvpx-dev \
-  llvm-14 \
-  ninja-build
-git clone --depth=1 https://github.com/ralight/mallocfail /tmp/mallocfail
-cd /tmp/mallocfail # pushd
-make
-sudo make install
-cd - # popd
-
-export CC=clang
-export CXX=clang++
-
-sudo install other/docker/coverage/run_mallocfail /usr/local/bin/run_mallocfail
-(cd other/proxy && go get && go build)
-other/proxy/proxy &
-
-. ".github/scripts/flags-coverage.sh"
-
-cmake -B_build -H. -GNinja \
-  -DCMAKE_C_FLAGS="$C_FLAGS" \
-  -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \
-  -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \
-  -DCMAKE_SHARED_LINKER_FLAGS="$LD_FLAGS" \
-  -DCMAKE_INSTALL_PREFIX:PATH="$PWD/_install" \
-  -DENABLE_SHARED=OFF \
-  -DMIN_LOGGER_LEVEL=TRACE \
-  -DMUST_BUILD_TOXAV=ON \
-  -DNON_HERMETIC_TESTS=OFF \
-  -DSTRICT_ABI=ON \
-  -DTEST_TIMEOUT_SECONDS=120 \
-  -DUSE_IPV6=OFF \
-  -DAUTOTEST=ON \
-  -DPROXY_TEST=ON
-
-cmake --build _build --parallel "$NPROC" --target install -- -k 0
-
-cd _build # pushd
-ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6 ||
-  ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6
-
-export PYTHONUNBUFFERED=1
-run_mallocfail --ctest=2 --jobs=8
-cd - # popd
-
-#coveralls \
-#  --exclude auto_tests \
-#  --exclude other \
-#  --exclude testing \
-#  --gcov-options '\-lp'
-
-bash <(curl -s https://codecov.io/bash) -x "llvm-cov-14 gcov"
--- a/.github/scripts/flags-coverage.sh
+++ b/.github/scripts/flags-coverage.sh
@ -5,10 +5,10 @@
 add_ld_flag -Wl,-z,defs

 # Make compilation error on a warning
-add_flag -Werror
+add_flag -Werror -Wno-unsafe-buffer-usage

 # Coverage flags.
-add_flag --coverage
+add_flag -fprofile-instr-generate -fcoverage-mapping

 # Optimisation, but keep stack traces useful.
 add_c_flag -fno-inline -fno-omit-frame-pointer
--- a/auto_tests/check_compat.h
+++ b/auto_tests/check_compat.h
@ -9,7 +9,7 @@
 #define ck_assert(ok) do {                                              \
  if (!(ok)) {                                                          \
    fprintf(stderr, "%s:%d: failed `%s'\n", __FILE__, __LINE__, #ok);   \
-    abort();                                                            \
+    exit(7);                                                            \
  }                                                                     \
 } while (0)

@ -18,7 +18,7 @@
    fprintf(stderr, "%s:%d: failed `%s': ", __FILE__, __LINE__, #ok);   \
    fprintf(stderr, __VA_ARGS__);                                       \
    fprintf(stderr, "\n");                                              \
-    abort();                                                            \
+    exit(7);                                                            \
  }                                                                     \
 } while (0)

@ -26,7 +26,7 @@
  fprintf(stderr, "%s:%d: ", __FILE__, __LINE__);                       \
  fprintf(stderr, __VA_ARGS__);                                         \
  fprintf(stderr, "\n");                                                \
-  abort();                                                              \
+  exit(7);                                                              \
 } while (0)

 #endif // C_TOXCORE_AUTO_TESTS_CHECK_COMPAT_H
--- a/heroku.yml
+++ b/heroku.yml
@ -1,4 +0,0 @@
---
-build:
-  docker:
-    web: other/bootstrap_daemon/websocket/Dockerfile
--- a/other/docker/coverage/Dockerfile
+++ b/other/docker/coverage/Dockerfile
@ -1,56 +1,57 @@
 FROM toxchat/c-toxcore:sources AS src
 FROM ubuntu:20.04 AS build

+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 RUN apt-get update && \
 DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
 ca-certificates \
- clang \
- cmake \
 curl \
- gcc \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main" >> /etc/apt/sources.list \
+ && curl -L https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc \
+ && apt-get update && \
+ DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
+ clang-17 \
+ cmake \
 git \
 golang-1.18 \
+ libclang-rt-17-dev \
 libconfig-dev \
 libgmock-dev \
 libgtest-dev \
 libopus-dev \
 libsodium-dev \
+ libunwind-17-dev \
 libvpx-dev \
- llvm-dev \
+ lld-17 \
+ llvm-17-dev \
 make \
 ninja-build \
 pkg-config \
- python3-lxml \
- python3-pip \
- python3-pygments \
 && apt-get clean \
- && rm -rf /var/lib/apt/lists/* \
- && pip3 install --no-cache-dir gcovr
-# strip gtest/gmock so it doesn't end up in stack traces. This speeds up the
-# mallocfail run below by a lot.
-RUN ["strip", "-g",\
- "/usr/lib/x86_64-linux-gnu/libgmock.a",\
- "/usr/lib/x86_64-linux-gnu/libgmock_main.a",\
- "/usr/lib/x86_64-linux-gnu/libgtest.a",\
- "/usr/lib/x86_64-linux-gnu/libgtest_main.a"]
+ && rm -rf /var/lib/apt/lists/*
 RUN ["curl", "-s", "https://codecov.io/bash", "-o", "/usr/local/bin/codecov"]
 RUN ["chmod", "+x", "/usr/local/bin/codecov"]

-ENV CC=clang \
-    CXX=clang++ \
+ENV CC=clang-17 \
+    CXX=clang++-17 \
    PYTHONUNBUFFERED=1 \
    PATH=$PATH:/usr/lib/go-1.18/bin
-SHELL ["/bin/bash", "-c"]
+
+COPY --from=src /src/ /work/

 WORKDIR /work
-COPY --from=src /src/ /work/
+RUN git clone --depth=1 https://github.com/TokTok/toktok-fuzzer /work/testing/fuzzing/toktok-fuzzer
 RUN source .github/scripts/flags-coverage.sh \
 && go version \
 && (cd other/proxy && go get github.com/things-go/go-socks5 && go build proxy_server.go) \
 && cmake -B_build -H. -GNinja \
 -DCMAKE_C_FLAGS="$C_FLAGS" \
 -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \
- -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \
+ -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS -fuse-ld=lld" \
 -DCMAKE_UNITY_BUILD=ON \
 -DENABLE_SHARED=OFF \
 -DMIN_LOGGER_LEVEL=TRACE \
@ -69,24 +70,15 @@ RUN /work/other/proxy/proxy_server \
    ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6)

 WORKDIR /work/mallocfail
-RUN ["git", "clone", "--depth=1", "https://github.com/ralight/mallocfail", "/work/mallocfail"]
-COPY other/docker/coverage/syscall_funcs.c src/
-RUN gcc -fPIC -shared -O2 -g3 -Wall -Ideps/uthash -Ideps/sha3 deps/*/*.c src/*.c -o mallocfail.so -ldl -lbacktrace \
+RUN ["git", "clone", "--depth=1", "https://github.com/TokTok/mallocfail", "/work/mallocfail"]
+RUN clang-17 -fuse-ld=lld -fPIC -shared -O2 -g3 -Wall -I/usr/lib/llvm-17/include -L/usr/lib/llvm-17/lib -Ideps/uthash -Ideps/sha3 deps/*/*.c src/*.c -o mallocfail.so -ldl -lunwind \
 && install mallocfail.so /usr/local/lib/mallocfail.so

 WORKDIR /work/_build
 COPY other/docker/coverage/run_mallocfail /usr/local/bin/
-RUN ["run_mallocfail", "--ctest=2", "--jobs=8"]
-RUN ["gcovr", \
- "--sort-percentage", \
- "--gcov-executable=llvm-cov gcov", \
- "--html-details=html/", \
- "--root=..", \
- "--exclude=CMakeFiles/", \
- "--exclude=_deps/", \
- "--exclude=(.+/)?auto_tests/", \
- "--exclude=.+_test.cc?$", \
- "--exclude=(.+/)?other/", \
- "--exclude=(.+/)?testing/"]
+RUN ["run_mallocfail", "--ctest=1", "--jobs=8"]
+RUN llvm-profdata-17 merge -sparse $(find . -name "*.profraw") -o toxcore.profdata
+RUN llvm-cov-17 show -format=text -instr-profile=toxcore.profdata -sources $(cmake --build . --target help | grep -o '[^:]*_test:' | grep -o '[^:]*' | xargs -n1 find . -type f -name | awk '{print "-object "$1}') > coverage.txt
+RUN llvm-cov-17 show -format=html -instr-profile=toxcore.profdata -sources $(cmake --build . --target help | grep -o '[^:]*_test:' | grep -o '[^:]*' | xargs -n1 find . -type f -name | awk '{print "-object "$1}') -output-dir=html

 WORKDIR /work
--- a/other/docker/coverage/Dockerfile.nginx
+++ b/other/docker/coverage/Dockerfile.nginx
@ -1,5 +1,4 @@
 # vim:ft=dockerfile
 FROM toxchat/c-toxcore:coverage AS build
 FROM nginx:alpine
-COPY --from=build /work/_build/html/coverage_details.html /usr/share/nginx/html/index.html
-COPY --from=build /work/_build/html/ /usr/share/nginx/html/
+COPY --from=build --chown=nginx:nginx /work/_build/html/ /usr/share/nginx/html/
--- a/other/docker/coverage/run
+++ b/other/docker/coverage/run
@ -6,4 +6,4 @@ read -a ci_env <<<"$(bash <(curl -s https://codecov.io/env))"

 docker build -t toxchat/c-toxcore:sources -f other/docker/sources/Dockerfile .
 docker build -t toxchat/c-toxcore:coverage -f other/docker/coverage/Dockerfile .
-docker run "${ci_env[@]}" -e CI=true --name toxcore-coverage --rm -t toxchat/c-toxcore:coverage /usr/local/bin/codecov -x "llvm-cov gcov"
+docker run "${ci_env[@]}" -e CI=true --name toxcore-coverage --rm -t toxchat/c-toxcore:coverage /usr/local/bin/codecov
--- a/other/docker/coverage/run_mallocfail
+++ b/other/docker/coverage/run_mallocfail
@ -27,11 +27,12 @@ from typing import NoReturn
 from typing import Optional
 from typing import Tuple

-_PRIMER = "./unit_util_test"
+_PRIMER = "auto_tests/auto_version_test"
 _MALLOCFAIL_SO = "/usr/local/lib/mallocfail.so"
 _HASHES = "mallocfail_hashes"
 _HASHES_PREV = "mallocfail_hashes.prev"
 _TIMEOUT = 3.0
+_BUILD_DIR = os.getcwd()

 _ENV = {
    "LD_PRELOAD": _MALLOCFAIL_SO,
@ -45,10 +46,19 @@ def run_mallocfail(tmpdir: str, timeout: float, exe: str, iteration: int,
    print(f"\x1b[1;33mmallocfail '{exe}' run #{iteration}\x1b[0m")
    hashes = os.path.join(tmpdir, _HASHES)
    hashes_prev = os.path.join(tmpdir, _HASHES_PREV)
+    profraw = os.path.join(_BUILD_DIR, "mallocfail.out", exe, "%p.profraw")
    if os.path.exists(hashes):
        shutil.copy(hashes, hashes_prev)
    try:
-        proc = subprocess.run([exe], timeout=timeout, env=_ENV, cwd=tmpdir)
+        proc = subprocess.run(
+            [exe],
+            timeout=timeout,
+            env={
+                "LLVM_PROFILE_FILE": profraw,
+                **_ENV,
+            },
+            cwd=tmpdir,
+        )
    except subprocess.TimeoutExpired:
        print(f"\x1b[1;34mProgram {exe} timed out\x1b[0m")
        return True
@ -65,13 +75,16 @@ def run_mallocfail(tmpdir: str, timeout: float, exe: str, iteration: int,
        # Process exited cleanly (success or failure).
        pass
    elif proc.returncode == -6:
-        # Assertion failed.
+        # abort(), we allow it.
+        pass
+    elif proc.returncode == 7:
+        # ck_assert failed, also fine for us.
        pass
    elif proc.returncode == -14:
        print(f"\x1b[0;34mProgram '{exe}' timed out\x1b[0m")
    else:
        print(
-            f"\x1b[1;32mProgram '{exe}' failed to handle OOM situation cleanly\x1b[0m"
+            f"\x1b[1;32mProgram '{exe}' failed to handle OOM situation cleanly (code {proc.returncode})\x1b[0m"
        )
        if not keep_going:
            raise Exception("Aborting test")
@ -96,8 +109,8 @@ def find_prog(name: str) -> Tuple[Optional[str], ...]:
            return path
        return None

-    return (attempt(f"./unit_{name}_test"),
-            attempt(f"auto_tests/auto_{name}_test"))
+    return (attempt(f"auto_tests/auto_{name}_test"),
+            )  # attempt(f"./unit_{name}_test"),


 def parse_flags(args: List[str]) -> Tuple[Dict[str, str], List[str]]:
@ -128,6 +141,9 @@ def isolated_mallocfail(timeout: int, exe: str) -> None:
        shutil.copy(exe, os.path.join(tmpdir, exe))
        shutil.copy(_HASHES, os.path.join(tmpdir, _HASHES))
        loop_mallocfail(tmpdir, timeout, exe)
+        profraw = os.path.join(tmpdir, "default.profraw")
+        if os.path.exists(profraw):
+            shutil.copy(profraw, exe + ".mallocfail.profraw")


 def main(args: List[str]) -> None:
@ -150,12 +166,12 @@ def main(args: List[str]) -> None:
    else:
        jobs = 1

-    # Start by running util_test, which allocates no memory of its own, just
+    # Start by running version_test, which allocates no memory of its own, just
    # to prime the mallocfail hashes so it doesn't make global initialisers
    # such as llvm_gcov_init fail.
    if os.path.exists(_PRIMER):
-        print(f"\x1b[1;33mPriming hashes with unit_util_test\x1b[0m")
-        loop_mallocfail(".", timeout, _PRIMER, keep_going=True)
+        print(f"\x1b[1;33mPriming hashes with {_PRIMER}\x1b[0m")
+        loop_mallocfail(os.getcwd(), timeout, _PRIMER, keep_going=True)

    print(f"\x1b[1;33m--------------------------------\x1b[0m")
    print(f"\x1b[1;33mStarting mallocfail for {len(exes)} programs:\x1b[0m")
--- a/other/docker/coverage/syscall_funcs.c
+++ b/other/docker/coverage/syscall_funcs.c
@ -1,149 +0,0 @@
-#define _GNU_SOURCE
-
-#include "mallocfail.h"
-
-#include <dlfcn.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <sys/ioctl.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-
-static int (*libc_ioctl)(int fd, unsigned long request, ...);
-static int (*libc_bind)(int sockfd, const struct sockaddr *addr,
-        socklen_t addrlen);
-static int (*libc_getsockopt)(int sockfd, int level, int optname,
-        void *optval, socklen_t *optlen);
-static int (*libc_setsockopt)(int sockfd, int level, int optname,
-        const void *optval, socklen_t optlen);
-static ssize_t (*libc_recv)(int sockfd, void *buf, size_t len, int flags);
-static ssize_t (*libc_recvfrom)(int sockfd, void *buf, size_t len, int flags,
-        struct sockaddr *src_addr, socklen_t *addrlen);
-static ssize_t (*libc_send)(int sockfd, const void *buf, size_t len, int flags);
-static ssize_t(*libc_sendto)(int sockfd, const void *buf, size_t len, int flags,
-        const struct sockaddr *dest_addr, socklen_t addrlen);
-static int (*libc_socket)(int domain, int type, int protocol);
-static int (*libc_listen)(int sockfd, int backlog);
-
-__attribute__((__constructor__))
-static void init(void)
-{
-    libc_ioctl = dlsym(RTLD_NEXT, "ioctl");
-    libc_bind = dlsym(RTLD_NEXT, "bind");
-    libc_getsockopt = dlsym(RTLD_NEXT, "getsockopt");
-    libc_setsockopt = dlsym(RTLD_NEXT, "setsockopt");
-    libc_recv = dlsym(RTLD_NEXT, "recv");
-    libc_recvfrom = dlsym(RTLD_NEXT, "recvfrom");
-    libc_send = dlsym(RTLD_NEXT, "send");
-    libc_sendto = dlsym(RTLD_NEXT, "sendto");
-    libc_socket = dlsym(RTLD_NEXT, "socket");
-    libc_listen = dlsym(RTLD_NEXT, "listen");
-}
-
-int ioctl(int fd, unsigned long request, ...)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-
-    va_list ap;
-    va_start(ap, request);
-    const int ret = libc_ioctl(fd, SIOCGIFCONF, va_arg(ap, void *));
-    va_end(ap);
-    return ret;
-}
-
-int bind(int sockfd, const struct sockaddr *addr,
-        socklen_t addrlen)
-{
-    // Unlike all others, if bind should fail once, it should fail always, because in toxcore we try
-    // many ports before giving up. If this only fails once, we'll never reach the code path where
-    // we give up.
-    static int should_fail = -1;
-    if (should_fail == -1) {
-        should_fail = should_malloc_fail();
-    }
-    if (should_fail) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_bind(sockfd, addr, addrlen);
-}
-
-int getsockopt(int sockfd, int level, int optname,
-        void *optval, socklen_t *optlen)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_getsockopt(sockfd, level, optname, optval, optlen);
-}
-
-int setsockopt(int sockfd, int level, int optname,
-        const void *optval, socklen_t optlen)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_setsockopt(sockfd, level, optname, optval, optlen);
-}
-
-ssize_t recv(int sockfd, void *buf, size_t len, int flags)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_recv(sockfd, buf, len, flags);
-}
-
-ssize_t recvfrom(int sockfd, void *buf, size_t len, int flags,
-        struct sockaddr *src_addr, socklen_t *addrlen)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_recvfrom(sockfd, buf, len, flags, src_addr, addrlen);
-}
-
-ssize_t send(int sockfd, const void *buf, size_t len, int flags)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_send(sockfd, buf, len, flags);
-}
-
-ssize_t sendto(int sockfd, const void *buf, size_t len, int flags,
-        const struct sockaddr *dest_addr, socklen_t addrlen)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_sendto(sockfd, buf, len, flags, dest_addr, addrlen);
-}
-
-int socket(int domain, int type, int protocol)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_socket(domain, type, protocol);
-}
-
-int listen(int sockfd, int backlog)
-{
-    if (should_malloc_fail()) {
-        errno = ENOMEM;
-        return -1;
-    }
-    return libc_listen(sockfd, backlog);
-}