From 4efe541814ec2ddd073428d6928497b50c48397a Mon Sep 17 00:00:00 2001 From: iphydf Date: Sun, 3 May 2020 15:36:57 +0100 Subject: [PATCH] Add a script to run Travis CI locally. This isn't quite Travis, but close enough for local testing. --- .restyled.yaml | 6 +++ .travis.yml | 2 +- .travis/cmake-linux | 22 +++++----- other/analysis/check_recursion | 29 +++++++++---- other/astyle/format-source | 49 +++++++++++++--------- other/docker/Dockerfile.ci | 75 ++++++++++++++++++++++++++++++++++ other/docker/run-ci | 15 +++++++ 7 files changed, 157 insertions(+), 41 deletions(-) create mode 100644 other/docker/Dockerfile.ci create mode 100755 other/docker/run-ci diff --git a/.restyled.yaml b/.restyled.yaml index 722ff510..f0060d75 100644 --- a/.restyled.yaml +++ b/.restyled.yaml @@ -2,4 +2,10 @@ restylers: - astyle: arguments: ["--options=other/astyle/astylerc"] + - autopep8 + - black - prettier-yaml + - reorder-python-imports + - shellharden + - shfmt + - yapf diff --git a/.travis.yml b/.travis.yml index 510fb984..93192a6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ jobs: - libgtest-dev # For unit tests. - libvpx-dev # For toxav. - ninja-build - - pylint + - pylint3 install: .travis/$JOB install script: .travis/$JOB script after_script: .travis/upload-coverage diff --git a/.travis/cmake-linux b/.travis/cmake-linux index ed06dc26..73b335fe 100755 --- a/.travis/cmake-linux +++ b/.travis/cmake-linux @@ -5,7 +5,7 @@ ACTION="$1" set -eu CACHEDIR="$HOME/cache" -NPROC=`nproc` +NPROC=$(nproc) ASTYLE="$CACHEDIR/astyle/build/gcc/bin/astyle" ASTYLE_VERSION=3.1 TRAVIS_TOOL="https://raw.githubusercontent.com/TokTok/ci-tools/master/bin/travis-haskell" @@ -21,7 +21,7 @@ travis_install() { # Work around https://github.com/eddyxu/cpp-coveralls/issues/108 by manually # installing the pyOpenSSL module and injecting it into urllib3 as per # https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 - sed -i -e '/^import sys$/a import urllib3.contrib.pyopenssl\nurllib3.contrib.pyopenssl.inject_into_urllib3()' `which coveralls` + sed -i -e '/^import sys$/a import urllib3.contrib.pyopenssl\nurllib3.contrib.pyopenssl.inject_into_urllib3()' "$(which coveralls)" } # Install astyle (version in ubuntu-precise too old). @@ -34,15 +34,15 @@ travis_install() { } run_static_analysis() { - pylint -E other/analysis/check_recursion + pylint3 -E other/analysis/check_recursion export CPPFLAGS="-isystem $CACHEDIR/include" export LDFLAGS="-L$CACHEDIR/lib" - cat toxav/*.c toxcore/*.c toxencryptsave/*.c \ - | clang `pkg-config --cflags libsodium opus vpx` \ - -Itoxav -Itoxcore -Itoxencryptsave -S -emit-llvm -xc - -o- \ - | opt -analyze -print-callgraph 2>&1 \ - | other/analysis/check_recursion + cat toxav/*.c toxcore/*.c toxencryptsave/*.c | + clang "$(pkg-config --cflags libsodium opus vpx)" \ + -Itoxav -Itoxcore -Itoxencryptsave -S -emit-llvm -xc - -o- | + opt -analyze -print-callgraph 2>&1 | + other/analysis/check_recursion other/analysis/run-clang other/analysis/run-clang-analyze } @@ -85,10 +85,10 @@ travis_script() { cmake --build _build --parallel "$NPROC" --target install -- -k 0 - cd _build # pushd - ctest -j50 --output-on-failure || \ + cd _build # pushd + ctest -j50 --output-on-failure || ctest -j50 --output-on-failure --rerun-failed - cd - # popd + cd - # popd } if [ "-z" "$ACTION" ]; then diff --git a/other/analysis/check_recursion b/other/analysis/check_recursion index 21063527..a8a6e0ef 100755 --- a/other/analysis/check_recursion +++ b/other/analysis/check_recursion @@ -1,8 +1,8 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Tool to check for recursive calls in toxcore C code. -Usage: +Usage: cat toxav/*.c toxcore/*.c toxencryptsave/*.c \ | clang `pkg-config --cflags libsodium opus vpx` \ @@ -10,16 +10,17 @@ cat toxav/*.c toxcore/*.c toxencryptsave/*.c \ | opt -analyze -print-callgraph 2>&1 \ | other/analysis/check_recursion """ - -from __future__ import print_function - import collections import fileinput import re import sys import time +from typing import Dict +from typing import List +from typing import Set -def load_callgraph(): + +def load_callgraph() -> Dict: """ Parses the output from opt -print-callgraph from stdin or argv. @@ -39,7 +40,14 @@ def load_callgraph(): return {k: sorted(v) for k, v in graph.items()} -def walk(visited, callgraph, cycles, stack, cur): + +def walk( + visited: Set, + callgraph: Dict, + cycles: Set, + stack: List, + cur: str, +) -> None: """ Detects cycles in the callgraph and adds them to the cycles parameter. """ @@ -54,13 +62,15 @@ def walk(visited, callgraph, cycles, stack, cur): visited.add(callee) stack.pop() -def get_time(): + +def get_time() -> int: """ Return the current time in milliseconds. """ return int(round(time.time() * 1000)) -def find_recursion(expected): + +def find_recursion(expected: Set) -> None: """ Main function: detects cycles and prints them. @@ -98,6 +108,7 @@ def find_recursion(expected): if expected or cycles: sys.exit(1) + find_recursion(expected={ "add_to_closest -> add_to_closest", "add_to_list -> add_to_list", diff --git a/other/astyle/format-source b/other/astyle/format-source index 193e335c..7de28d06 100755 --- a/other/astyle/format-source +++ b/other/astyle/format-source @@ -36,10 +36,11 @@ FROM_JSON='s/\\"/"/g;s/^"(.*)"$/$1/;s/\\\\/\\/g;s/\\n/\n/g' apidsl_request() { TMPFILE=$(mktemp /tmp/apidsl.XXXXXX) curl -s -o "$TMPFILE" -X POST --data @<( - echo '["Request",'; - cat $2; - echo ']') https://apidsl.herokuapp.com/$1 - if grep '\[1,"' "$TMPFILE" > /dev/null; then + echo '["Request",' + cat "$2" + echo ']' + ) "https://apidsl.herokuapp.com/$1" + if grep '\[1,"' "$TMPFILE" >/dev/null; then echo "Error: $(grep -o '".*"' /tmp/apidsl-$$ | perl -0777 -pe "$FROM_JSON")" >&2 rm "$TMPFILE" exit 1 @@ -52,34 +53,42 @@ apidsl_curl() { echo "apidsl_curl $*" >&2 apidsl_request "c" <( apidsl_request "parse" <( - perl -0777 -pe "$TO_JSON" $1)) | perl -0777 -pe "$FROM_JSON" + perl -0777 -pe "$TO_JSON" "$1" + ) + ) | perl -0777 -pe "$FROM_JSON" } # Check if apidsl generated sources are up to date. set +x -$APIDSL toxcore/LAN_discovery.api.h > toxcore/LAN_discovery.h & -$APIDSL toxcore/crypto_core.api.h > toxcore/crypto_core.h & -$APIDSL toxcore/ping.api.h > toxcore/ping.h & -$APIDSL toxcore/ping_array.api.h > toxcore/ping_array.h & -$APIDSL toxcore/tox.api.h > toxcore/tox.h & -$APIDSL toxav/toxav.api.h > toxav/toxav.h & -$APIDSL toxencryptsave/toxencryptsave.api.h > toxencryptsave/toxencryptsave.h & +"$APIDSL" toxcore/LAN_discovery.api.h >toxcore/LAN_discovery.h & +"$APIDSL" toxcore/crypto_core.api.h >toxcore/crypto_core.h & +"$APIDSL" toxcore/ping.api.h >toxcore/ping.h & +"$APIDSL" toxcore/ping_array.api.h >toxcore/ping_array.h & +"$APIDSL" toxcore/tox.api.h >toxcore/tox.h & +"$APIDSL" toxav/toxav.api.h >toxav/toxav.h & +"$APIDSL" toxencryptsave/toxencryptsave.api.h >toxencryptsave/toxencryptsave.h & set -x -wait; wait; wait; wait; wait; wait; wait +wait +wait +wait +wait +wait +wait +wait if grep '' ./*/*.h; then echo "error: some apidsl references were unresolved" exit 1 fi -CC_SOURCES=$(find . '(' -name '*.cc' ')') -CC_SOURCES="$CC_SOURCES toxcore/crypto_core.c" -CC_SOURCES="$CC_SOURCES toxcore/ping_array.c" +readarray -t CC_SOURCES <<<"$(find . '(' -name '*.cc' ')')" +CC_SOURCES+=(toxcore/crypto_core.c) +CC_SOURCES+=(toxcore/ping_array.c) for bin in clang-format-6.0 clang-format-5.0 clang-format; do if which "$bin"; then - "$bin" -i -style='{BasedOnStyle: Google, ColumnLimit: 100}' $CC_SOURCES + "$bin" -i -style='{BasedOnStyle: Google, ColumnLimit: 100}' "${CC_SOURCES[@]}" break fi done @@ -91,8 +100,8 @@ FIND="$FIND -and -not -wholename './super_donators/*'" FIND="$FIND -and -not -wholename './third_party/*'" FIND="$FIND -and -not -wholename './toxencryptsave/crypto_pwhash*'" -C_SOURCES=$(eval "$FIND") +readarray -t C_SOURCES <<<"$(eval "$FIND")" -$ASTYLE -n --options=other/astyle/astylerc $C_SOURCES +"$ASTYLE" -n --options=other/astyle/astylerc "${C_SOURCES[@]}" -git diff --exit-code +git diff --color=always --exit-code diff --git a/other/docker/Dockerfile.ci b/other/docker/Dockerfile.ci new file mode 100644 index 00000000..db7e5e2c --- /dev/null +++ b/other/docker/Dockerfile.ci @@ -0,0 +1,75 @@ +# This Docker build emulates roughly what Travis CI is doing. It is not exactly +# the same (different tool versions) and success in this image may not +# necessarily mean success on Travis. This image is also not automatically +# tested, so it may get out of date. Send PRs if you use it and it's broken. +# +# For one, we use bionic, not xenial, because xenial's clang is way too old. +FROM ubuntu:16.04 + +# Travis environment. +RUN apt-get update && apt-get install --no-install-recommends -y \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + git \ + pkg-config \ + python-pip \ + python-setuptools \ + python3 \ + software-properties-common \ + wget \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN curl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ + && apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" \ + && apt-get update && apt-get install --no-install-recommends -y \ + clang-6.0 \ + clang-format-6.0 \ + llvm-6.0 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN ls /usr/bin/clang-6.0 && ln -s /usr/bin/clang-6.0 /usr/bin/clang \ + && ls /usr/bin/clang++-6.0 && ln -s /usr/bin/clang++-6.0 /usr/bin/clang++ \ + && ls /usr/bin/clang-format-6.0 && ln -s /usr/bin/clang-format-6.0 /usr/bin/clang-format \ + && ls /usr/bin/opt-6.0 && ln -s /usr/bin/opt-6.0 /usr/bin/opt + +# Bionic's cmake is too old. +RUN pip install --upgrade pip cmake + +# .travis.yml +RUN apt-get update && apt-get install --no-install-recommends -y \ + libconfig-dev \ + libgtest-dev \ + libopus-dev \ + libsodium-dev \ + libvpx-dev \ + ninja-build \ + pylint3 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Set up travis user. +RUN groupadd -r -g 1000 travis \ + && useradd --no-log-init -r -g travis -u 1000 travis \ + && mkdir -p /src/workspace /home/travis \ + && chown travis:travis /home/travis +USER travis + +# Set up environment. +ENV CC=gcc CXX=g++ \ +PATH=/home/travis/.local/bin:$PATH \ +TRAVIS_REPO_SLUG=TokTok/c-toxcore + +# Copy minimal files to run "cmake-linux install", so we can avoid rebuilding +# astyle and other things when only source files change. +RUN mkdir -p /home/travis/build/c-toxcore /home/travis/cache +WORKDIR /home/travis/build/c-toxcore +COPY --chown=travis:travis c-toxcore/.travis/ /home/travis/build/c-toxcore/.travis/ +RUN .travis/cmake-linux install + +# Now copy the rest of the sources and run the build. +COPY --chown=travis:travis . /home/travis/build/ +RUN .travis/cmake-linux script diff --git a/other/docker/run-ci b/other/docker/run-ci new file mode 100755 index 00000000..9005cfbd --- /dev/null +++ b/other/docker/run-ci @@ -0,0 +1,15 @@ +#!/bin/bash + +set -eu + +readarray -t FILES <<<"$(git ls-files | sed -e 's,^,c-toxcore/,')" + +if [ -f .git ]; then + cd .. + tar -c "${FILES[@]}" "c-toxcore/.git" ".git/modules/c-toxcore" | + docker build -f c-toxcore/other/docker/Dockerfile.ci - +else + cd .. + tar -c "${FILES[@]}" "c-toxcore/.git" | + docker build -f c-toxcore/other/docker/Dockerfile.ci - +fi