diff --git a/.github/scripts/cmake-linux b/.github/scripts/coverage-linux similarity index 54% rename from .github/scripts/cmake-linux rename to .github/scripts/coverage-linux index 2956e711..5cb3babf 100755 --- a/.github/scripts/cmake-linux +++ b/.github/scripts/coverage-linux @@ -10,17 +10,23 @@ sudo apt-get install -y --no-install-recommends \ libopus-dev \ libsodium-dev \ libvpx-dev \ + llvm-11 \ ninja-build +git clone --depth=1 https://github.com/ralight/mallocfail /tmp/mallocfail +cd /tmp/mallocfail # pushd +make +sudo make install +cd - # popd -. ".github/scripts/flags-$CC.sh" +export CC=clang +export CXX=clang++ -add_ld_flag -Wl,-z,defs +sudo install other/docker/coverage/run_mallocfail /usr/local/bin/run_mallocfail +go get github.com/things-go/go-socks5 +go build other/proxy/proxy_server.go +./proxy_server & -# Make compilation error on a warning -add_flag -Werror - -# Coverage flags. -add_flag -fprofile-arcs -ftest-coverage +. ".github/scripts/flags-coverage.sh" cmake -B_build -H. -GNinja \ -DCMAKE_C_FLAGS="$C_FLAGS" \ @@ -28,17 +34,30 @@ cmake -B_build -H. -GNinja \ -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \ -DCMAKE_SHARED_LINKER_FLAGS="$LD_FLAGS" \ -DCMAKE_INSTALL_PREFIX:PATH="$PWD/_install" \ + -DENABLE_SHARED=OFF \ -DMIN_LOGGER_LEVEL=TRACE \ -DMUST_BUILD_TOXAV=ON \ -DNON_HERMETIC_TESTS=OFF \ -DSTRICT_ABI=ON \ -DTEST_TIMEOUT_SECONDS=120 \ -DUSE_IPV6=OFF \ - -DAUTOTEST=ON + -DAUTOTEST=ON \ + -DPROXY_TEST=ON cmake --build _build --parallel "$NPROC" --target install -- -k 0 cd _build # pushd ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6 || ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6 + +export PYTHONUNBUFFERED=1 +run_mallocfail --ctest=2 --jobs=8 cd - # popd + +#coveralls \ +# --exclude auto_tests \ +# --exclude other \ +# --exclude testing \ +# --gcov-options '\-lp' + +bash <(curl -s https://codecov.io/bash) -x "llvm-cov-11 gcov" diff --git a/.github/scripts/flags-coverage.sh b/.github/scripts/flags-coverage.sh new file mode 100644 index 00000000..3fa6b6bd --- /dev/null +++ b/.github/scripts/flags-coverage.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +. .github/scripts/flags-clang.sh + +add_ld_flag -Wl,-z,defs + +# Make compilation error on a warning +add_flag -Werror + +# Coverage flags. +add_flag --coverage + +# Optimisation, but keep stack traces useful. +add_c_flag -fno-inline -fno-omit-frame-pointer + +# Show useful stack traces on crash. +add_flag -fsanitize=undefined -fno-sanitize-recover=all + +# In test code (_test.cc and libgtest), throw away all debug information. +# We only care about stack frames inside toxcore (which is C). Without this, +# mallocfail will spend a lot of time finding all the ways in which gtest can +# fail to allocate memory, which is not interesting to us. +add_cxx_flag -g0 + +# Continue executing code in error paths so we can see it cleanly exit (and the +# test code itself may abort). +add_flag -DABORT_ON_LOG_ERROR=false diff --git a/.github/scripts/upload-coverage b/.github/scripts/upload-coverage deleted file mode 100755 index d49538d9..00000000 --- a/.github/scripts/upload-coverage +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# We only submit coverage from the Linux build. -coveralls \ - --exclude auto_tests \ - --exclude other \ - --exclude testing \ - --gcov-options '\-lp' - -bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c42185a..4462456c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,13 +81,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Build and test - run: .github/scripts/cmake-linux - env: - CC: gcc - CXX: g++ - - name: Upload coverage - run: .github/scripts/upload-coverage + - name: Build, test, and upload coverage + run: .github/scripts/coverage-linux build-tcc: runs-on: ubuntu-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c9cc444..062926b6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,8 @@ if(MIN_LOGGER_LEVEL) endif() endif() +option(PROXY_TEST "Enable proxy test (needs HTTP/SOCKS5 proxy on port 8080/8081)" OFF) + option(USE_IPV6 "Use IPv6 in tests" ON) if(NOT USE_IPV6) add_definitions(-DUSE_IPV6=0) @@ -459,6 +461,7 @@ auto_test(file_transfer) auto_test(file_saving) auto_test(friend_connection) auto_test(friend_request) +auto_test(friend_request_spam) auto_test(invalid_tcp_proxy) auto_test(invalid_udp_proxy) auto_test(lan_discovery) @@ -484,6 +487,10 @@ auto_test(typing) auto_test(version) auto_test(save_compatibility) +if(PROXY_TEST) + auto_test(proxy) +endif() + if(NON_HERMETIC_TESTS) auto_test(bootstrap) auto_test(tcp_relay) diff --git a/auto_tests/BUILD.bazel b/auto_tests/BUILD.bazel index 819b52b1..8efb4aa5 100644 --- a/auto_tests/BUILD.bazel +++ b/auto_tests/BUILD.bazel @@ -33,8 +33,8 @@ flaky_tests = { name = src[:-2], size = "small", srcs = [src], - args = ["$(location %s)" % src], - data = glob(["data/*"]), + args = ["$(location %s)" % src] + ["$(location //c-toxcore/other/proxy)"], + data = glob(["data/*"]) + ["//c-toxcore/other/proxy"], flaky = flaky_tests.get( src[:-2], False, diff --git a/auto_tests/auto_test_support.c b/auto_tests/auto_test_support.c index 5d7208b8..dd4237cd 100644 --- a/auto_tests/auto_test_support.c +++ b/auto_tests/auto_test_support.c @@ -8,6 +8,10 @@ #include "auto_test_support.h" +#ifndef ABORT_ON_LOG_ERROR +#define ABORT_ON_LOG_ERROR true +#endif + const Run_Auto_Options default_run_auto_options = { GRAPH_COMPLETE, nullptr }; // List of live bootstrap nodes. These nodes should have TCP server enabled. @@ -148,6 +152,7 @@ void set_mono_time_callback(AutoTox *autotox) Mono_Time *mono_time = ((Messenger *)autotox->tox)->mono_time; autotox->clock = current_time_monotonic(mono_time); + mono_time_set_current_time_callback(mono_time, nullptr, &autotox->clock); // set to default first mono_time_set_current_time_callback(mono_time, get_state_clock_callback, &autotox->clock); } @@ -357,7 +362,7 @@ void print_debug_log(Tox *m, Tox_Log_Level level, const char *file, uint32_t lin uint32_t index = user_data ? *(uint32_t *)user_data : 0; fprintf(stderr, "[#%u] %s %s:%u\t%s:\t%s\n", index, tox_log_level_name(level), file, line, func, message); - if (level == TOX_LOG_LEVEL_ERROR) { + if (level == TOX_LOG_LEVEL_ERROR && ABORT_ON_LOG_ERROR) { fputs("Aborting test program\n", stderr); abort(); } diff --git a/auto_tests/friend_request_spam_test.c b/auto_tests/friend_request_spam_test.c new file mode 100644 index 00000000..02e03a51 --- /dev/null +++ b/auto_tests/friend_request_spam_test.c @@ -0,0 +1,80 @@ +/* Tests what happens when spamming friend requests from lots of temporary toxes. + */ + +#include +#include +#include +#include + +#include "../toxcore/ccompat.h" +#include "../toxcore/tox.h" +#include "../toxcore/util.h" +#include "../testing/misc_tools.h" +#include "auto_test_support.h" +#include "check_compat.h" + +#define FR_MESSAGE "Gentoo" +// TODO(iphydf): Investigate friend request spam: receiving more than 32 at a time means any further +// friend requests are dropped on the floor and aren't seen again. +#define FR_TOX_COUNT 33 + +typedef struct State { + bool unused; +} State; + +static void accept_friend_request(Tox *tox, const uint8_t *public_key, const uint8_t *data, size_t length, + void *userdata) +{ + ck_assert_msg(length == sizeof(FR_MESSAGE) && memcmp(FR_MESSAGE, data, sizeof(FR_MESSAGE)) == 0, + "unexpected friend request message"); + tox_friend_add_norequest(tox, public_key, nullptr); +} + +static void test_friend_request(AutoTox *autotoxes) +{ + const time_t con_time = time(nullptr); + + printf("All toxes add tox1 as friend.\n"); + tox_callback_friend_request(autotoxes[0].tox, accept_friend_request); + + uint8_t address[TOX_ADDRESS_SIZE]; + tox_self_get_address(autotoxes[0].tox, address); + + for (uint32_t i = 2; i < FR_TOX_COUNT; ++i) { + Tox_Err_Friend_Add err; + tox_friend_add(autotoxes[i].tox, address, (const uint8_t *)FR_MESSAGE, sizeof(FR_MESSAGE), &err); + ck_assert_msg(err == TOX_ERR_FRIEND_ADD_OK, "tox %u failed to add friend error code: %d", autotoxes[i].index, err); + } + + for (uint32_t t = 0; t < 100; ++t) { + bool connected = true; + for (uint32_t i = 0; i < FR_TOX_COUNT; ++i) { + uint32_t size = tox_self_get_friend_list_size(autotoxes[i].tox); + for (uint32_t fn = 0; fn < size; ++fn) { + if (tox_friend_get_connection_status(autotoxes[i].tox, fn, nullptr) == TOX_CONNECTION_NONE) { + connected = false; + } + } + } + + if (connected) { + break; + } + + iterate_all_wait(autotoxes, FR_TOX_COUNT, ITERATION_INTERVAL); + } + + uint32_t size = tox_self_get_friend_list_size(autotoxes[0].tox); + printf("Tox clients connected took %lu seconds; tox1 has %u friends.\n", (unsigned long)(time(nullptr) - con_time), size); +} + +int main(void) +{ + setvbuf(stdout, nullptr, _IONBF, 0); + + Run_Auto_Options options = default_run_auto_options; + options.graph = GRAPH_LINEAR; + run_auto_test(nullptr, FR_TOX_COUNT, test_friend_request, sizeof(State), &options); + + return 0; +} diff --git a/auto_tests/messenger_test.c b/auto_tests/messenger_test.c index fe20e6b7..3feeed90 100644 --- a/auto_tests/messenger_test.c +++ b/auto_tests/messenger_test.c @@ -168,6 +168,7 @@ static void test_getself_name(void) const char *nickname = "testGallop"; size_t len = strlen(nickname); char *nick_check = (char *)calloc(len + 1, 1); + ck_assert(nick_check != nullptr); setname(m, (const uint8_t *)nickname, len); getself_name(m, (uint8_t *)nick_check); @@ -279,6 +280,10 @@ int main(void) good_id = hex_string_to_bin(good_id_str); bad_id = hex_string_to_bin(bad_id_str); + ck_assert(friend_id != nullptr); + ck_assert(good_id != nullptr); + ck_assert(bad_id != nullptr); + Mono_Time *mono_time = mono_time_new(); /* IPv6 status from global define */ diff --git a/auto_tests/proxy_test.c b/auto_tests/proxy_test.c new file mode 100644 index 00000000..d6f2d8cf --- /dev/null +++ b/auto_tests/proxy_test.c @@ -0,0 +1,129 @@ +/* Tests that we can send messages to friends. + */ + +#include +#include +#include + +#include "auto_test_support.h" + +static void *proxy_routine(void *arg) +{ + const char *proxy_bin = (const char *)arg; + ck_assert(proxy_bin != nullptr); + printf("starting http/sock5 proxy: %s\n", proxy_bin); + ck_assert(system(proxy_bin) == 0); + return nullptr; +} + +static bool try_bootstrap(Tox *tox1, Tox *tox2, Tox *tox3, Tox *tox4) +{ + for (uint32_t i = 0; i < 400; ++i) { + if (tox_self_get_connection_status(tox1) != TOX_CONNECTION_NONE && + tox_self_get_connection_status(tox2) != TOX_CONNECTION_NONE && + tox_self_get_connection_status(tox3) != TOX_CONNECTION_NONE && + tox_self_get_connection_status(tox4) != TOX_CONNECTION_NONE) { + printf("%d %d %d %d\n", + tox_self_get_connection_status(tox1), + tox_self_get_connection_status(tox2), + tox_self_get_connection_status(tox3), + tox_self_get_connection_status(tox4)); + return true; + } + + tox_iterate(tox1, nullptr); + tox_iterate(tox2, nullptr); + tox_iterate(tox3, nullptr); + tox_iterate(tox4, nullptr); + + if (i % 10 == 0) { + printf("%d %d %d %d\n", + tox_self_get_connection_status(tox1), + tox_self_get_connection_status(tox2), + tox_self_get_connection_status(tox3), + tox_self_get_connection_status(tox4)); + } + + c_sleep(tox_iteration_interval(tox1)); + } + + return false; +} + +int main(int argc, char **argv) +{ + setvbuf(stdout, nullptr, _IONBF, 0); + if (argc >= 3) { + char *proxy_bin = argv[2]; + pthread_t proxy_thread; + pthread_create(&proxy_thread, nullptr, proxy_routine, proxy_bin); + c_sleep(100); + } + + const uint16_t tcp_port = 8082; + uint32_t index[] = { 1, 2, 3, 4 }; + + struct Tox_Options *tox_options = tox_options_new(nullptr); + ck_assert(tox_options != nullptr); + + // tox1 is a TCP server and has UDP enabled. + tox_options_set_udp_enabled(tox_options, true); + tox_options_set_tcp_port(tox_options, tcp_port); + + Tox *tox1 = tox_new_log(tox_options, nullptr, &index[0]); + ck_assert(tox1 != nullptr); + + // Get tox1's DHT key and port. + uint8_t dht_pk[TOX_PUBLIC_KEY_SIZE]; + tox_self_get_dht_id(tox1, dht_pk); + uint16_t dht_port = tox_self_get_udp_port(tox1, nullptr); + ck_assert(dht_port != 0); + + // tox2 is a regular DHT node bootstrapping against tox1. + tox_options_set_udp_enabled(tox_options, true); + tox_options_set_tcp_port(tox_options, 0); + + Tox *tox2 = tox_new_log(tox_options, nullptr, &index[1]); + ck_assert(tox2 != nullptr); + + // tox2 bootstraps against tox1 with UDP. + ck_assert(tox_bootstrap(tox2, "127.0.0.1", dht_port, dht_pk, nullptr)); + + // tox3 has UDP disabled and connects to tox1 via an HTTP proxy + tox_options_set_udp_enabled(tox_options, false); + tox_options_set_proxy_host(tox_options, "127.0.0.1"); + tox_options_set_proxy_port(tox_options, 8080); + tox_options_set_proxy_type(tox_options, TOX_PROXY_TYPE_HTTP); + + Tox *tox3 = tox_new_log(tox_options, nullptr, &index[2]); + ck_assert(tox3 != nullptr); + + // tox4 has UDP disabled and connects to tox1 via a SOCKS5 proxy + tox_options_set_udp_enabled(tox_options, false); + tox_options_set_proxy_host(tox_options, "127.0.0.1"); + tox_options_set_proxy_port(tox_options, 8081); + tox_options_set_proxy_type(tox_options, TOX_PROXY_TYPE_SOCKS5); + + Tox *tox4 = tox_new_log(tox_options, nullptr, &index[3]); + ck_assert(tox4 != nullptr); + + // tox3 and tox4 bootstrap against tox1 and add it as a TCP relay + ck_assert(tox_bootstrap(tox3, "127.0.0.1", dht_port, dht_pk, nullptr)); + ck_assert(tox_add_tcp_relay(tox3, "127.0.0.1", tcp_port, dht_pk, nullptr)); + + ck_assert(tox_bootstrap(tox4, "127.0.0.1", dht_port, dht_pk, nullptr)); + ck_assert(tox_add_tcp_relay(tox4, "127.0.0.1", tcp_port, dht_pk, nullptr)); + + int ret = 1; + if (try_bootstrap(tox1, tox2, tox3, tox4)) { + ret = 0; + } + + tox_options_free(tox_options); + tox_kill(tox4); + tox_kill(tox3); + tox_kill(tox2); + tox_kill(tox1); + + return ret; +} diff --git a/other/analysis/gen-file.sh b/other/analysis/gen-file.sh index bf74112d..0e433212 100644 --- a/other/analysis/gen-file.sh +++ b/other/analysis/gen-file.sh @@ -49,6 +49,7 @@ callmain() { FIND_QUERY="find . '-(' -name '*.c' -or -name '*.cc' '-)'" # Excludes FIND_QUERY="$FIND_QUERY -and -not -wholename './_build/*'" +FIND_QUERY="$FIND_QUERY -and -not -wholename './other/docker/*'" FIND_QUERY="$FIND_QUERY -and -not -wholename './super_donators/*'" FIND_QUERY="$FIND_QUERY -and -not -name amalgamation.cc" FIND_QUERY="$FIND_QUERY -and -not -name av_test.c" diff --git a/other/bootstrap_daemon/docker/tox-bootstrapd.sha256 b/other/bootstrap_daemon/docker/tox-bootstrapd.sha256 index 560abdb7..2f8d32c7 100644 --- a/other/bootstrap_daemon/docker/tox-bootstrapd.sha256 +++ b/other/bootstrap_daemon/docker/tox-bootstrapd.sha256 @@ -1 +1 @@ -3b7df7850212de052071b23f6b8085852467e227c232f80c97a76e4cf28d0327 /usr/local/bin/tox-bootstrapd +aea24cfae8db82511a298e2dbedb6634145732190ebcad361690cb6ca6560d7c /usr/local/bin/tox-bootstrapd diff --git a/other/bootstrap_daemon/websocket/websockify/websockify.go b/other/bootstrap_daemon/websocket/websockify/websockify.go index 0fcad22b..067ddb7a 100644 --- a/other/bootstrap_daemon/websocket/websockify/websockify.go +++ b/other/bootstrap_daemon/websocket/websockify/websockify.go @@ -34,7 +34,7 @@ var upgrader = websocket.Upgrader{ // Should be enough to fit any Tox TCP packets. ReadBufferSize: 2048, WriteBufferSize: 2048, - Subprotocols: []string{"binary"}, + Subprotocols: []string{"binary"}, CheckOrigin: func(r *http.Request) bool { return true }, diff --git a/other/docker/coverage/Dockerfile b/other/docker/coverage/Dockerfile index 21e64dce..a94f4338 100644 --- a/other/docker/coverage/Dockerfile +++ b/other/docker/coverage/Dockerfile @@ -3,9 +3,11 @@ FROM ubuntu:20.04 AS build RUN apt-get update && \ DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ - build-essential \ + clang \ cmake \ + gcc \ git \ + golang \ libconfig-dev \ libgtest-dev \ libmsgpack-dev \ @@ -13,6 +15,7 @@ RUN apt-get update && \ libsodium-dev \ libvpx-dev \ llvm-dev \ + make \ ninja-build \ pkg-config \ python3-pip \ @@ -21,38 +24,52 @@ RUN apt-get update && \ && rm -rf /var/lib/apt/lists/* \ && pip3 install --no-cache-dir gcovr -WORKDIR /work/mallocfail -RUN ["git", "clone", "--depth=1", "https://github.com/ralight/mallocfail", "/work/mallocfail"] -RUN ["make", "install"] +ENV CC=clang \ + CXX=clang++ \ + PYTHONUNBUFFERED=1 +SHELL ["/bin/bash", "-c"] WORKDIR /work COPY --from=src /src/ /work/ -RUN ["cmake", "-B_build", "-H.", "-GNinja", \ - "-DCMAKE_C_FLAGS=-fsanitize=undefined -fno-sanitize-recover=all -fprofile-arcs -ftest-coverage -O2 -fno-omit-frame-pointer -fno-inline -g", \ - "-DENABLE_SHARED=OFF", \ - "-DMIN_LOGGER_LEVEL=TRACE", \ - "-DMUST_BUILD_TOXAV=ON", \ - "-DNON_HERMETIC_TESTS=ON", \ - "-DSTRICT_ABI=ON", \ - "-DAUTOTEST=ON"] -RUN ["cmake", "--build", "_build", "--parallel", "8", "--target", "install"] +RUN source .github/scripts/flags-coverage.sh \ + && go get github.com/things-go/go-socks5 \ + && go build other/proxy/proxy_server.go \ + && cmake -B_build -H. -GNinja \ + -DCMAKE_C_FLAGS="$C_FLAGS" \ + -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \ + -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \ + -DCMAKE_UNITY_BUILD=ON \ + -DENABLE_SHARED=OFF \ + -DMIN_LOGGER_LEVEL=TRACE \ + -DMUST_BUILD_TOXAV=ON \ + -DNON_HERMETIC_TESTS=ON \ + -DSTRICT_ABI=ON \ + -DAUTOTEST=ON \ + -DPROXY_TEST=ON \ + && cmake --build _build --parallel 8 --target install WORKDIR /work/_build -RUN ["ctest", "-j50", "--output-on-failure", "--rerun-failed", "--repeat", "until-pass:6"] +RUN /work/proxy_server \ + & ctest -j50 --output-on-failure --rerun-failed --repeat until-pass:6 + +WORKDIR /work/mallocfail +RUN ["git", "clone", "--depth=1", "https://github.com/ralight/mallocfail", "/work/mallocfail"] COPY run_mallocfail /usr/local/bin/ -ENV PYTHONUNBUFFERED=1 \ - UBSAN_OPTIONS=color=always,print_stacktrace=1 -RUN run_mallocfail ./unit_*_test -RUN run_mallocfail ./auto_send_message_test +COPY syscall_funcs.c src/ +RUN gcc -fPIC -shared -O2 -g3 -Wall -Ideps/uthash -Ideps/sha3 deps/*/*.c src/*.c -o mallocfail.so -ldl -lbacktrace \ + && install mallocfail.so /usr/local/lib/mallocfail.so + +WORKDIR /work/_build +RUN ["run_mallocfail", "--ctest=2", "--jobs=8"] RUN ["gcovr", \ "--sort-percentage", \ - "--gcov-executable=gcov", \ + "--gcov-executable=llvm-cov gcov", \ "--html-details=html/", \ - "--html-self-contained", \ "--root=..", \ "--exclude=CMakeFiles/", \ "--exclude=_deps/", \ "--exclude=(.+/)?auto_tests/", \ + "--exclude=.+_test.cc?$", \ "--exclude=(.+/)?other/", \ "--exclude=(.+/)?testing/"] diff --git a/other/docker/coverage/mallocfail.h b/other/docker/coverage/mallocfail.h new file mode 100644 index 00000000..e26f188e --- /dev/null +++ b/other/docker/coverage/mallocfail.h @@ -0,0 +1,3 @@ +// Dummy, actual one is here: https://github.com/ralight/mallocfail/blob/master/src/mallocfail.h + +int should_malloc_fail(void); diff --git a/other/docker/coverage/run_mallocfail b/other/docker/coverage/run_mallocfail index 280397cf..57f99e62 100755 --- a/other/docker/coverage/run_mallocfail +++ b/other/docker/coverage/run_mallocfail @@ -1,49 +1,67 @@ #!/usr/bin/env python3 """Run a test repeatedly with mallocfail. -Usage: run_mallocfail +Usage: run_mallocfail [--ctest=] [...] -This runs the program with mallocfail until there are no more additional stack -hashes for mallocfail to try out. +This runs the programs with mallocfail until there are no more additional +stack hashes for mallocfail to try out. + +Passing "--ctest" additionally runs all the tests with a cost lower than the +given flag value. Cost is measured in seconds of runtime. You need to build mallocfail (https://github.com/ralight/mallocfail) and install it to /usr/local/lib/mallocfail. Change _MALLOCFAIL_SO below if you want it elsewhere. """ +import glob +import multiprocessing import os import shutil import subprocess import sys +import tempfile +import time +from typing import Dict from typing import List +from typing import NoReturn +from typing import Optional +from typing import Tuple +_PRIMER = "./unit_util_test" _MALLOCFAIL_SO = "/usr/local/lib/mallocfail.so" _HASHES = "mallocfail_hashes" _HASHES_PREV = "mallocfail_hashes.prev" -_TIMEOUT = 3 +_TIMEOUT = 3.0 + +_ENV = { + "LD_PRELOAD": _MALLOCFAIL_SO, + "UBSAN_OPTIONS": "color=always,print_stacktrace=1,exitcode=11", +} -def run_mallocfail(exe: str, iteration: int) -> bool: +def run_mallocfail(tmpdir: str, timeout: float, exe: str, + iteration: int) -> bool: """Run a program with mallocfail.""" print(f"\x1b[1;33mmallocfail '{exe}' run #{iteration}\x1b[0m") - if os.path.exists(_HASHES): - shutil.copy(_HASHES, _HASHES_PREV) + hashes = os.path.join(tmpdir, _HASHES) + hashes_prev = os.path.join(tmpdir, _HASHES_PREV) + if os.path.exists(hashes): + shutil.copy(hashes, hashes_prev) try: - proc = subprocess.run([exe], - timeout=_TIMEOUT, - env={"LD_PRELOAD": _MALLOCFAIL_SO}) + proc = subprocess.run([exe], timeout=timeout, env=_ENV, cwd=tmpdir) except subprocess.TimeoutExpired: print(f"\x1b[1;34mProgram {exe} timed out\x1b[0m") return True finally: - assert os.path.exists(_HASHES) - if os.path.exists(_HASHES_PREV): - with open(_HASHES_PREV, "r") as prev: - with open(_HASHES, "r") as cur: + assert os.path.exists(hashes) + if os.path.exists(hashes_prev): + with open(hashes_prev, "r") as prev: + with open(hashes, "r") as cur: if prev.read() == cur.read(): # Done: no new stack hashes. return False - if proc.returncode >= 0: + if proc.returncode in (0, 1): # Process exited cleanly (success or failure). pass elif proc.returncode == -6: @@ -55,21 +73,93 @@ def run_mallocfail(exe: str, iteration: int) -> bool: print( f"\x1b[1;34mProgram '{exe}' failed to handle OOM situation cleanly\x1b[0m" ) - sys.exit(1) + raise Exception("Aborting test") return True +def ctest_costs() -> Dict[str, float]: + with open("Testing/Temporary/CTestCostData.txt", "r") as fh: + costs = {} + for line in fh.readlines(): + if line.startswith("---"): + break + prog, _, cost = line.rstrip().split(" ") + costs[prog] = float(cost) + return costs + + +def find_prog(name: str) -> Tuple[Optional[str], ...]: + def attempt(path: str) -> Optional[str]: + if os.path.exists(path): + return path + return None + + return (attempt(f"./unit_{name}_test"), attempt(f"./auto_{name}_test")) + + +def parse_flags(args: List[str]) -> Tuple[Dict[str, str], List[str]]: + flags: Dict[str, str] = {} + exes: List[str] = [] + for arg in args: + if arg.startswith("--"): + flag, value = arg.split("=", 1) + flags[flag] = value + else: + exes.append(arg) + return flags, exes + + +def loop_mallocfail(tmpdir: str, timeout: float, exe: str) -> None: + i = 1 + while run_mallocfail(tmpdir, timeout, exe, i): + i += 1 + + +def isolated_mallocfail(timeout: int, exe: str) -> None: + with tempfile.TemporaryDirectory(prefix="mallocfail") as tmpdir: + print(f"\x1b[1;33mRunning for {exe} in isolated path {tmpdir}\x1b[0m") + shutil.copy(exe, os.path.join(tmpdir, exe)) + shutil.copy(_HASHES, os.path.join(tmpdir, _HASHES)) + loop_mallocfail(tmpdir, timeout, exe) + + def main(args: List[str]) -> None: """Run a program repeatedly under mallocfail.""" if len(args) == 1: print(f"Usage: {args[0]} ") sys.exit(1) - for exe in args[1:]: - i = 1 - while run_mallocfail(exe, i): - i += 1 + flags, exes = parse_flags(args[1:]) + + timeout = _TIMEOUT + if "--ctest" in flags: + costs = ctest_costs() + max_cost = float(flags["--ctest"]) + timeout = max(max_cost + 1, timeout) + exes.extend(prog for test in costs.keys() for prog in find_prog(test) + if costs[test] <= max_cost and prog) + if "--jobs" in flags: + jobs = int(flags["--jobs"]) + else: + jobs = 1 + + # Start by running util_test, which allocates no memory of its own, just + # to prime the mallocfail hashes so it doesn't make global initialisers + # such as llvm_gcov_init fail. + if os.path.exists(_PRIMER): + print(f"\x1b[1;33mPriming hashes with unit_util_test\x1b[0m") + loop_mallocfail(".", timeout, _PRIMER) + + print(f"\x1b[1;33m--------------------------------\x1b[0m") + print(f"\x1b[1;33mStarting mallocfail for {len(exes)} programs:\x1b[0m") + print(f"\x1b[1;33m{exes}\x1b[0m") + print(f"\x1b[1;33m--------------------------------\x1b[0m") + time.sleep(1) + with multiprocessing.Pool(jobs) as p: + done = tuple( + p.starmap(isolated_mallocfail, ((timeout, exe) for exe in exes))) + print(f"\x1b[1;32mCompleted {len(done)} programs\x1b[0m") if __name__ == "__main__": diff --git a/other/docker/coverage/syscall_funcs.c b/other/docker/coverage/syscall_funcs.c new file mode 100644 index 00000000..3d768571 --- /dev/null +++ b/other/docker/coverage/syscall_funcs.c @@ -0,0 +1,149 @@ +#define _GNU_SOURCE + +#include "mallocfail.h" + +#include +#include +#include +#include +#include +#include +#include + +static int (*libc_ioctl)(int fd, unsigned long request, ...); +static int (*libc_bind)(int sockfd, const struct sockaddr *addr, + socklen_t addrlen); +static int (*libc_getsockopt)(int sockfd, int level, int optname, + void *optval, socklen_t *optlen); +static int (*libc_setsockopt)(int sockfd, int level, int optname, + const void *optval, socklen_t optlen); +static ssize_t (*libc_recv)(int sockfd, void *buf, size_t len, int flags); +static ssize_t (*libc_recvfrom)(int sockfd, void *buf, size_t len, int flags, + struct sockaddr *src_addr, socklen_t *addrlen); +static ssize_t (*libc_send)(int sockfd, const void *buf, size_t len, int flags); +static ssize_t(*libc_sendto)(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen); +static int (*libc_socket)(int domain, int type, int protocol); +static int (*libc_listen)(int sockfd, int backlog); + +__attribute__((__constructor__)) +static void init(void) +{ + libc_ioctl = dlsym(RTLD_NEXT, "ioctl"); + libc_bind = dlsym(RTLD_NEXT, "bind"); + libc_getsockopt = dlsym(RTLD_NEXT, "getsockopt"); + libc_setsockopt = dlsym(RTLD_NEXT, "setsockopt"); + libc_recv = dlsym(RTLD_NEXT, "recv"); + libc_recvfrom = dlsym(RTLD_NEXT, "recvfrom"); + libc_send = dlsym(RTLD_NEXT, "send"); + libc_sendto = dlsym(RTLD_NEXT, "sendto"); + libc_socket = dlsym(RTLD_NEXT, "socket"); + libc_listen = dlsym(RTLD_NEXT, "listen"); +} + +int ioctl(int fd, unsigned long request, ...) +{ + if (should_malloc_fail()) { + errno = ENOMEM; + return -1; + } + + va_list ap; + va_start(ap, request); + const int ret = libc_ioctl(fd, SIOCGIFCONF, va_arg(ap, void *)); + va_end(ap); + return ret; +} + +int bind(int sockfd, const struct sockaddr *addr, + socklen_t addrlen) +{ + // Unlike all others, if bind should fail once, it should fail always, because in toxcore we try + // many ports before giving up. If this only fails once, we'll never reach the code path where + // we give up. + static int should_fail = -1; + if (should_fail == -1) { + should_fail = should_malloc_fail(); + } + if (should_fail) { + errno = ENOMEM; + return -1; + } + return libc_bind(sockfd, addr, addrlen); +} + +int getsockopt(int sockfd, int level, int optname, + void *optval, socklen_t *optlen) +{ + if (should_malloc_fail()) { + errno = ENOMEM; + return -1; + } + return libc_getsockopt(sockfd, level, optname, optval, optlen); +} + +int setsockopt(int sockfd, int level, int optname, + const void *optval, socklen_t optlen) +{ + if (should_malloc_fail()) { + errno = ENOMEM; + return -1; + } + return libc_setsockopt(sockfd, level, optname, optval, optlen); +} + +ssize_t recv(int sockfd, void *buf, size_t len, int flags) +{ + if (should_malloc_fail()) { + errno = ENOMEM; + return -1; + } + return libc_recv(sockfd, buf, len, flags); +} + +ssize_t recvfrom(int sockfd, void *buf, size_t len, int flags, + struct sockaddr *src_addr, socklen_t *addrlen) +{ + if (should_malloc_fail()) { + errno = ENOMEM; + return -1; + } + return libc_recvfrom(sockfd, buf, len, flags, src_addr, addrlen); +} + +ssize_t send(int sockfd, const void *buf, size_t len, int flags) +{ + if (should_malloc_fail()) { + errno = ENOMEM; + return -1; + } + return libc_send(sockfd, buf, len, flags); +} + +ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen) +{ + if (should_malloc_fail()) { + errno = ENOMEM; + return -1; + } + return libc_sendto(sockfd, buf, len, flags, dest_addr, addrlen); +} + +int socket(int domain, int type, int protocol) +{ + if (should_malloc_fail()) { + errno = ENOMEM; + return -1; + } + return libc_socket(domain, type, protocol); +} + +int listen(int sockfd, int backlog) +{ + if (should_malloc_fail()) { + errno = ENOMEM; + return -1; + } + return libc_listen(sockfd, backlog); +} diff --git a/other/docker/sources/Dockerfile b/other/docker/sources/Dockerfile index 743a86e7..1d257161 100644 --- a/other/docker/sources/Dockerfile +++ b/other/docker/sources/Dockerfile @@ -1,6 +1,8 @@ FROM scratch # Roughly in order of change frequency. +COPY .github/scripts/flags*.sh /src/.github/scripts/ +COPY other/proxy/ /src/other/proxy/ COPY cmake/ /src/cmake/ COPY other/bootstrap_daemon/ /src/other/bootstrap_daemon/ COPY other/pkgconfig/ /src/other/pkgconfig/ diff --git a/other/proxy/BUILD.bazel b/other/proxy/BUILD.bazel new file mode 100644 index 00000000..aa17c6d2 --- /dev/null +++ b/other/proxy/BUILD.bazel @@ -0,0 +1,17 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +package(features = ["layering_check"]) + +go_library( + name = "go_default_library", + srcs = ["proxy_server.go"], + importpath = "github.com/TokTok/c-toxcore/other/proxy", + visibility = ["//visibility:private"], + deps = ["@com_github_things_go_go_socks5//:go_default_library"], +) + +go_binary( + name = "proxy", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/other/proxy/proxy_server.go b/other/proxy/proxy_server.go new file mode 100644 index 00000000..04d9b2f3 --- /dev/null +++ b/other/proxy/proxy_server.go @@ -0,0 +1,85 @@ +package main + +import ( + "crypto/tls" + "encoding/hex" + "io" + "log" + "net" + "net/http" + "os" + "time" + + "github.com/things-go/go-socks5" +) + +const ( + debug = false + httpAddr = ":8080" + socks5Addr = ":8081" +) + +func handleTunneling(w http.ResponseWriter, r *http.Request) { + log.Printf("opening tunnel to %q", r.Host) + destConn, err := net.DialTimeout("tcp", r.Host, 10*time.Second) + if err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + return + } + log.Printf("responding OK") + w.WriteHeader(http.StatusOK) + hijacker, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "Hijacking not supported", http.StatusInternalServerError) + return + } + log.Printf("hijacking HTTP connection") + clientConn, _, err := hijacker.Hijack() + if err != nil { + http.Error(w, err.Error(), http.StatusServiceUnavailable) + } + log.Printf("starting data transfer") + go transfer(">", destConn, clientConn) + go transfer("<", clientConn, destConn) +} + +func transfer(direction string, destination io.WriteCloser, source io.ReadCloser) { + defer destination.Close() + defer source.Close() + var buf [2048]byte + for { + n, err := source.Read(buf[:]) + if err != nil { + log.Printf("error: %s", err) + break + } + if debug { + log.Printf("%s %02x", direction, hex.EncodeToString(buf[:n])) + } + destination.Write(buf[:n]) + } +} + +func main() { + log.Printf("starting HTTP proxy server on %s", httpAddr) + httpServer := &http.Server{ + Addr: httpAddr, + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodConnect { + handleTunneling(w, r) + } else { + panic("invalid method, only CONNECT is allowed") + } + }), + // Disable HTTP/2. + TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), + } + + log.Printf("starting SOCKS5 proxy server on %s", socks5Addr) + socks5Server := socks5.NewServer( + socks5.WithLogger(socks5.NewLogger(log.New(os.Stdout, "socks5: ", log.LstdFlags))), + ) + + go socks5Server.ListenAndServe("tcp", socks5Addr) + log.Fatal(httpServer.ListenAndServe()) +} diff --git a/testing/misc_tools.c b/testing/misc_tools.c index 6e2f07f9..6188599b 100644 --- a/testing/misc_tools.c +++ b/testing/misc_tools.c @@ -57,6 +57,10 @@ uint8_t *hex_string_to_bin(const char *hex_string) uint8_t *ret = (uint8_t *)malloc(len); const char *pos = hex_string; + if (ret == nullptr) { + return nullptr; + } + for (i = 0; i < len; ++i, pos += 2) { unsigned int val; sscanf(pos, "%02x", &val); diff --git a/toxcore/TCP_client.c b/toxcore/TCP_client.c index 38a40b0f..1ff01d7f 100644 --- a/toxcore/TCP_client.c +++ b/toxcore/TCP_client.c @@ -114,7 +114,7 @@ static int connect_sock_to(const Logger *logger, Socket sock, const IP_Port *ip_ * return 0 on failure. */ non_null() -static int proxy_http_generate_connection_request(TCP_Client_Connection *tcp_conn) +static int proxy_http_generate_connection_request(const Logger *logger, TCP_Client_Connection *tcp_conn) { char one[] = "CONNECT "; char two[] = " HTTP/1.1\nHost: "; @@ -136,7 +136,6 @@ static int proxy_http_generate_connection_request(TCP_Client_Connection *tcp_con tcp_conn->con.last_packet_length = written; tcp_conn->con.last_packet_sent = 0; - return 1; } @@ -574,7 +573,7 @@ TCP_Client_Connection *new_TCP_connection(const Logger *logger, const Mono_Time switch (proxy_info->proxy_type) { case TCP_PROXY_HTTP: { temp->status = TCP_CLIENT_PROXY_HTTP_CONNECTING; - proxy_http_generate_connection_request(temp); + proxy_http_generate_connection_request(logger, temp); break; } diff --git a/toxcore/TCP_common.c b/toxcore/TCP_common.c index a91d4f7b..18a6c2df 100644 --- a/toxcore/TCP_common.c +++ b/toxcore/TCP_common.c @@ -272,6 +272,10 @@ int read_packet_TCP_secure_connection(const Logger *logger, Socket sock, uint16_ VLA(uint8_t, data_encrypted, *next_packet_length); const int len_packet = read_TCP_packet(logger, sock, data_encrypted, *next_packet_length, ip_port); + if (len_packet == -1) { + return 0; + } + if (len_packet != *next_packet_length) { LOGGER_ERROR(logger, "invalid packet length: %d, expected %d", len_packet, *next_packet_length); return 0; diff --git a/toxcore/TCP_server.c b/toxcore/TCP_server.c index d557d542..c8275b4c 100644 --- a/toxcore/TCP_server.c +++ b/toxcore/TCP_server.c @@ -828,6 +828,7 @@ static Socket new_listening_TCP_socket(const Logger *logger, Family family, uint return net_invalid_socket; } + LOGGER_DEBUG(logger, "successfully bound to TCP port %d", port); return sock; } diff --git a/toxcore/ping_array_test.cc b/toxcore/ping_array_test.cc index 41423f40..c4856a4e 100644 --- a/toxcore/ping_array_test.cc +++ b/toxcore/ping_array_test.cc @@ -50,6 +50,7 @@ TEST(PingArray, StoredDataCanBeRetrieved) { Ping_Array_Ptr const arr(ping_array_new(2, 1)); Mono_Time_Ptr const mono_time(mono_time_new()); + ASSERT_NE(mono_time, nullptr); uint64_t const ping_id = ping_array_add(arr.get(), mono_time.get(), std::vector{1, 2, 3, 4}.data(), 4); @@ -64,6 +65,7 @@ TEST(PingArray, RetrievingDataWithTooSmallOutputBufferHasNoEffect) { Ping_Array_Ptr const arr(ping_array_new(2, 1)); Mono_Time_Ptr const mono_time(mono_time_new()); + ASSERT_NE(mono_time, nullptr); uint64_t const ping_id = ping_array_add(arr.get(), mono_time.get(), (std::vector{1, 2, 3, 4}).data(), 4); @@ -82,6 +84,7 @@ TEST(PingArray, ZeroLengthDataCanBeAdded) { Ping_Array_Ptr const arr(ping_array_new(2, 1)); Mono_Time_Ptr const mono_time(mono_time_new()); + ASSERT_NE(mono_time, nullptr); uint8_t c = 0; uint64_t const ping_id = ping_array_add(arr.get(), mono_time.get(), &c, sizeof(c)); @@ -94,6 +97,7 @@ TEST(PingArray, PingId0IsInvalid) { Ping_Array_Ptr const arr(ping_array_new(2, 1)); Mono_Time_Ptr const mono_time(mono_time_new()); + ASSERT_NE(mono_time, nullptr); uint8_t c = 0; EXPECT_EQ(ping_array_check(arr.get(), mono_time.get(), &c, sizeof(c), 0), -1); @@ -104,6 +108,7 @@ TEST(PingArray, DataCanOnlyBeRetrievedOnce) { Ping_Array_Ptr const arr(ping_array_new(2, 1)); Mono_Time_Ptr const mono_time(mono_time_new()); + ASSERT_NE(mono_time, nullptr); uint8_t c = 0; uint64_t const ping_id = ping_array_add(arr.get(), mono_time.get(), &c, sizeof(c)); @@ -117,6 +122,7 @@ TEST(PingArray, PingIdMustMatchOnCheck) { Ping_Array_Ptr const arr(ping_array_new(1, 1)); Mono_Time_Ptr const mono_time(mono_time_new()); + ASSERT_NE(mono_time, nullptr); uint8_t c = 0; uint64_t const ping_id = ping_array_add(arr.get(), mono_time.get(), &c, sizeof(c));