From a2d586b78254f6ba3b12d20f2398f31e9ee66235 Mon Sep 17 00:00:00 2001 From: iphydf Date: Thu, 23 Apr 2020 16:18:15 +0000 Subject: [PATCH] Rework the toxchat/bootstrap-node Docker image. * Use fully static build for the bootstrap daemon. * Store a sha256sum of the binary in the repo. * Updated documentation for it. * Add support for fully static build in cmake. * Enable the docker build on every PR, so we catch changes to the checksum. I realise this is adding toil, but having the checksum is valuable for security of released binaries. --- .travis.yml | 9 +- .travis/tox-bootstrapd-docker | 47 +++---- cmake/ModulePackage.cmake | 10 ++ other/bootstrap_daemon/README.md | 61 +++++++-- other/bootstrap_daemon/docker/Dockerfile | 127 ++++++++++-------- other/bootstrap_daemon/docker/get-nodes.py | 24 ++-- .../docker/tox-bootstrapd.sha256 | 1 + other/fun/bootstrap_node_info.py | 2 +- 8 files changed, 173 insertions(+), 108 deletions(-) mode change 100644 => 100755 other/bootstrap_daemon/docker/get-nodes.py create mode 100644 other/bootstrap_daemon/docker/tox-bootstrapd.sha256 mode change 100644 => 100755 other/fun/bootstrap_node_info.py diff --git a/.travis.yml b/.travis.yml index bfebbd71..510fb984 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,6 +38,10 @@ jobs: - libvpx-dev # For toxav. install: .travis/$JOB install script: .travis/$JOB script + - stage: "Stage 1" + env: JOB=tox-bootstrapd-docker + services: [docker] + script: .travis/$JOB - stage: "Stage 1" if: type IN (push, api, cron) env: JOB=cmake-win32 @@ -61,11 +65,6 @@ jobs: os: osx install: .travis/$JOB install script: .travis/$JOB script - - stage: "Stage 1" - if: type IN (push, api, cron) - env: JOB=tox-bootstrapd-docker - services: [docker] - script: .travis/$JOB - stage: "Stage 2" if: type IN (push, api, cron) env: JOB=cmake-freebsd diff --git a/.travis/tox-bootstrapd-docker b/.travis/tox-bootstrapd-docker index 5dc39628..bd192f07 100755 --- a/.travis/tox-bootstrapd-docker +++ b/.travis/tox-bootstrapd-docker @@ -2,28 +2,23 @@ set -exu -# Copy source code to other/bootstrap_daemon/docker/c-toxcore -OLD_PWD=$PWD -cd /tmp -cp -a $OLD_PWD c-toxcore -mv c-toxcore $OLD_PWD/other/bootstrap_daemon/docker -cd $OLD_PWD -ls -lbh other/bootstrap_daemon/docker -ls -lbh other/bootstrap_daemon/docker/c-toxcore - -cd other/bootstrap_daemon - -# Make Docker container use our current source code instead of master branch -sed -i "s|WORKDIR /tmp/tox|WORKDIR /tmp/tox\nADD /c-toxcore ./c-toxcore/|g" docker/Dockerfile -sed -i 's|git clone|echo \\\#git clone|g' docker/Dockerfile -sed -i 's|git checkout|echo \\\#git checkout|g' docker/Dockerfile - -cat docker/Dockerfile - -sudo docker build -t tox-bootstrapd docker/ -sudo useradd --home-dir /var/lib/tox-bootstrapd --create-home --system --shell /sbin/nologin --comment "Account to run Tox's DHT bootstrap daemon" --user-group tox-bootstrapd +tar c $(git ls-files) | docker build -f other/bootstrap_daemon/docker/Dockerfile -t toxchat/bootstrap-node - +sudo useradd \ + --home-dir /var/lib/tox-bootstrapd \ + --create-home \ + --system \ + --shell /sbin/nologin \ + --comment "Account to run Tox's DHT bootstrap daemon" \ + --user-group tox-bootstrapd sudo chmod 700 /var/lib/tox-bootstrapd -sudo docker run -d --name tox-bootstrapd -v /var/lib/tox-bootstrapd/:/var/lib/tox-bootstrapd/ --ulimit nofile=32768:32768 -p 443:443 -p 3389:3389 -p 33445:33445 -p 33445:33445/udp tox-bootstrapd +docker run -d --name tox-bootstrapd \ + -v /var/lib/tox-bootstrapd/:/var/lib/tox-bootstrapd/ \ + --ulimit nofile=32768:32768 \ + -p 443:443 \ + -p 3389:3389 \ + -p 33445:33445 \ + -p 33445:33445/udp \ + toxchat/bootstrap-node sudo ls -lbh /var/lib/tox-bootstrapd @@ -35,14 +30,14 @@ fi COUNTER=0 COUNTER_END=120 while [ $COUNTER -lt $COUNTER_END ]; do - if sudo docker logs tox-bootstrapd | grep -q "Connected to another bootstrap node successfully" ; then + if docker logs tox-bootstrapd | grep -q "Connected to another bootstrap node successfully" ; then break fi sleep 1 COUNTER=$(($COUNTER+1)) done -sudo docker logs tox-bootstrapd +docker logs tox-bootstrapd if [ "$COUNTER" = "$COUNTER_END" ]; then echo "Error: Didn't connect to any nodes" @@ -52,9 +47,9 @@ fi # Wait a bit befrore testing if the container is still running sleep 30 -sudo docker ps -a +docker ps -a -if [ "`sudo docker inspect -f {{.State.Running}} tox-bootstrapd`" != "true" ]; then +if [ "`docker inspect -f {{.State.Running}} tox-bootstrapd`" != "true" ]; then echo "Error: Container is not running" exit 1 fi @@ -65,7 +60,7 @@ if ! cat /proc/$(pidof tox-bootstrapd)/limits | grep -P '^Max open files(\s+)327 exit 1 fi -if ! python3 ../fun/bootstrap_node_info.py ipv4 localhost 33445 ; then +if ! other/fun/bootstrap_node_info.py ipv4 localhost 33445 ; then echo "Error: Unable to get bootstrap node info" exit 1 fi diff --git a/cmake/ModulePackage.cmake b/cmake/ModulePackage.cmake index 77911b1d..9cc21863 100644 --- a/cmake/ModulePackage.cmake +++ b/cmake/ModulePackage.cmake @@ -9,6 +9,16 @@ if(NOT ENABLE_SHARED AND NOT ENABLE_STATIC) set(ENABLE_SHARED ON) endif() +option(FULLY_STATIC "Build fully static executables" OFF) +if(FULLY_STATIC) + set(CMAKE_EXE_LINKER_FLAGS "-static -no-pie") + # remove -Wl,-Bdynamic + set(CMAKE_EXE_LINK_DYNAMIC_C_FLAGS) + set(CMAKE_EXE_LINK_DYNAMIC_CXX_FLAGS) + set(ENABLE_SHARED OFF) + set(ENABLE_STATIC ON) +endif() + find_package(PkgConfig) function(pkg_use_module mod pkg) diff --git a/other/bootstrap_daemon/README.md b/other/bootstrap_daemon/README.md index d49b84d0..e108b415 100644 --- a/other/bootstrap_daemon/README.md +++ b/other/bootstrap_daemon/README.md @@ -216,22 +216,53 @@ sudo grep "tox-bootstrapd" /var/log/syslog ### Setting up -If you are familiar with Docker and would rather run the daemon in a Docker container, run the following from this directory: +If you are familiar with Docker and would rather run the daemon in a Docker container, you may download the latest official docker image. Check the GitHub [releases](https://github.com/TokTok/c-toxcore/releases) page for the latest version (e.g. v0.2.11), and run: ```sh -sudo docker build -t tox-bootstrapd docker/ +docker pull toxchat/bootstrap-node:0.2.11 +docker run --rm -it --entrypoint=sha256sum toxchat/bootstrap-node:latest /usr/local/bin/tox-bootstrapd +``` -sudo useradd --home-dir /var/lib/tox-bootstrapd --create-home --system --shell /sbin/nologin --comment "Account to run Tox's DHT bootstrap daemon" --user-group tox-bootstrapd +This will print the SHA256 checksum of the latest binary, which should agree with the SHA256 checksum in the Dockerfile. + +If you want to build the bootstrap node from source, check out the latest release: + +```sh +git checkout $(git tag --list | grep -P '^v(\d+).(\d+).(\d+)$' | \ + sed 's/v/v /g' | sed 's/\./ /g' | \ + sort -snk4,4 | sort -snk3,3 | sort -snk2,2 | tail -n 1 | \ + sed 's/v /v/g' | sed 's/ /\./g') +``` + +and run the following from the top level c-toxcore directory: + +```sh +tar c $(git ls-files) | docker build -f other/bootstrap_daemon/docker/Dockerfile -t toxchat/bootstrap-node - + +sudo useradd \ + --home-dir /var/lib/tox-bootstrapd \ + --create-home \ + --system \ + --shell /sbin/nologin \ + --comment "Account to run Tox's DHT bootstrap daemon" \ + --user-group tox-bootstrapd sudo chmod 700 /var/lib/tox-bootstrapd -sudo docker run -d --name tox-bootstrapd --restart always -v /var/lib/tox-bootstrapd/:/var/lib/tox-bootstrapd/ --ulimit nofile=32768:32768 -p 443:443 -p 3389:3389 -p 33445:33445 -p 33445:33445/udp tox-bootstrapd +docker run -d --name tox-bootstrapd --restart always \ + -v /var/lib/tox-bootstrapd/:/var/lib/tox-bootstrapd/ \ + --ulimit nofile=32768:32768 \ + -p 443:443 \ + -p 3389:3389 \ + -p 33445:33445 \ + -p 33445:33445/udp \ + toxchat/bootstrap-node ``` We create a new user and protect its home directory in order to mount it in the Docker image, so that the kyepair the daemon uses would be stored on the host system, which makes it less likely that you would loose the keypair while playing with or updating the Docker container. You can check logs for your public key or any errors: ```sh -sudo docker logs tox-bootstrapd +docker logs tox-bootstrapd ``` Note that the Docker container runs a script which pulls a list of bootstrap nodes off https://nodes.tox.chat/ and adds them in the config file. @@ -243,26 +274,32 @@ You want to make sure that the daemon uses the newest toxcore, as there might ha To update the daemon, all you need is to erase current container with its image: ```sh -sudo docker stop tox-bootstrapd -sudo docker rm tox-bootstrapd -sudo docker rmi tox-bootstrapd +docker stop tox-bootstrapd +docker rm tox-bootstrapd +docker rmi toxchat/bootstrap-node ``` Then rebuild and run the image again: ```sh -sudo docker build -t tox-bootstrapd docker/ -sudo docker run -d --name tox-bootstrapd --restart always -v /var/lib/tox-bootstrapd/:/var/lib/tox-bootstrapd/ -p 443:443 -p 3389:3389 -p 33445:33445 -p 33445:33445/udp tox-bootstrapd +tar c $(git ls-files) | docker build -f other/bootstrap_daemon/docker/Dockerfile -t toxchat/bootstrap-node - +docker run -d --name tox-bootstrapd --restart always \ + -v /var/lib/tox-bootstrapd/:/var/lib/tox-bootstrapd/ \ + -p 443:443 \ + -p 3389:3389 \ + -p 33445:33445 \ + -p 33445:33445/udp \ + toxchat/bootstrap-node ``` ### Troubleshooting - Check if the container is running: ```sh -sudo docker ps -a +docker ps -a ``` - Check the log for errors: ```sh -sudo docker logs tox-bootstrapd +docker logs tox-bootstrapd ``` diff --git a/other/bootstrap_daemon/docker/Dockerfile b/other/bootstrap_daemon/docker/Dockerfile index 5ef359cf..25a48b85 100644 --- a/other/bootstrap_daemon/docker/Dockerfile +++ b/other/bootstrap_daemon/docker/Dockerfile @@ -1,64 +1,81 @@ +########################################################### +# Builder image: we compile the code here (static build) +FROM alpine:3.11.5 AS build + +RUN ["apk", "--no-cache", "add",\ + "build-base",\ + "cmake",\ + "linux-headers",\ + "libconfig-dev",\ + "libconfig-static",\ + "libsodium-dev",\ + "libsodium-static",\ + "ninja",\ + "python3"\ +] + +WORKDIR /src/c-toxcore + +# Very selectively add files to the image, because we may have random stuff +# lying around. In particular, we don't need to rebuild the docker image when +# toxav changes or the Dockerfile changes down from the build. +COPY cmake cmake +COPY other/bootstrap_daemon/src other/bootstrap_daemon/src +COPY other/bootstrap_node_packets.[ch] other/ +COPY other/DHT_bootstrap.c other/ +COPY other/pkgconfig other/pkgconfig +COPY other/rpm other/rpm +COPY testing/misc_tools.[ch] testing/ +COPY toxcore toxcore +COPY toxencryptsave toxencryptsave +COPY CMakeLists.txt so.version ./ + +RUN ["cmake", "-B_build", "-H.",\ + "-GNinja",\ + "-DCMAKE_BUILD_TYPE=Release",\ + "-DFULLY_STATIC=ON",\ + "-DBUILD_TOXAV=OFF",\ + "-DBOOTSTRAP_DAEMON=ON"\ +] +RUN ["cmake", "--build", "_build", "--target", "install"] + +# Verify checksum from dev-built binary, so we can be sure Docker Hub doesn't +# mess with your binaries. +COPY other/bootstrap_daemon/docker/tox-bootstrapd.sha256 other/bootstrap_daemon/docker/ +RUN ["sha256sum", "/usr/local/bin/tox-bootstrapd"] +RUN ["sha256sum", "-c", "other/bootstrap_daemon/docker/tox-bootstrapd.sha256"] + +# Remove all the example bootstrap nodes from the config file. +COPY other/bootstrap_daemon/tox-bootstrapd.conf other/bootstrap_daemon/ +# hadolint ignore=SC2086,SC2154 +RUN ["sed", "-i", "/^bootstrap_nodes = /,$d", "other/bootstrap_daemon/tox-bootstrapd.conf"] + +# Add bootstrap nodes from https://nodes.tox.chat/. +COPY other/bootstrap_daemon/docker/get-nodes.py other/bootstrap_daemon/docker/ +RUN ["other/bootstrap_daemon/docker/get-nodes.py", "other/bootstrap_daemon/tox-bootstrapd.conf"] + +########################################################### +# Final image build: this is what runs the bootstrap node FROM debian:buster-slim -WORKDIR /tmp/tox - -RUN export BUILD_PACKAGES="\ - build-essential \ - cmake \ - git \ - libconfig-dev \ - libsodium-dev \ - python3" && \ - export RUNTIME_PACKAGES="\ - libconfig9 \ - libsodium23" && \ -# get all deps - apt-get update && apt-get install -y $BUILD_PACKAGES $RUNTIME_PACKAGES && \ -# install toxcore and daemon - git clone https://github.com/TokTok/c-toxcore && \ - cd c-toxcore && \ - # checkout latest release version - git checkout $(git tag --list | grep -P '^v(\d+).(\d+).(\d+)$' | \ - sed "s/v/v /g" | sed "s/\./ /g" | \ - sort -snk4,4 | sort -snk3,3 | sort -snk2,2 | tail -n 1 | \ - sed 's/v /v/g' | sed 's/ /\./g') && \ - mkdir _build && \ - cd _build && \ - cmake .. \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_SHARED=ON \ - -DENABLE_STATIC=OFF \ - -DBUILD_TOXAV=OFF \ - -DBOOTSTRAP_DAEMON=ON && \ - make -j`nproc` && \ - make install -j`nproc` && \ - cd .. && \ -# add new user - useradd --home-dir /var/lib/tox-bootstrapd --create-home \ - --system --shell /sbin/nologin \ - --comment "Account to run Tox's DHT bootstrap daemon" \ - --user-group tox-bootstrapd && \ - chmod 700 /var/lib/tox-bootstrapd && \ - cp other/bootstrap_daemon/tox-bootstrapd.conf /etc/tox-bootstrapd.conf && \ -# remove all the example bootstrap nodes from the config file - sed -i '/^bootstrap_nodes = /,$d' /etc/tox-bootstrapd.conf && \ -# add bootstrap nodes from https://nodes.tox.chat/ - python3 other/bootstrap_daemon/docker/get-nodes.py >> /etc/tox-bootstrapd.conf && \ -# perform cleanup - apt-get remove --purge -y $BUILD_PACKAGES && \ - apt-get clean && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* && \ - cd / && \ - rm -rf /tmp/* +COPY --from=build /usr/local/bin/tox-bootstrapd /usr/local/bin/ +COPY --from=build /src/c-toxcore/other/bootstrap_daemon/tox-bootstrapd.conf /etc/tox-bootstrapd.conf +RUN ["useradd", "--home-dir", "/var/lib/tox-bootstrapd", "--create-home",\ + "--system", "--shell", "/sbin/nologin",\ + "--comment", "Account to run the Tox DHT bootstrap daemon",\ + "--user-group", "tox-bootstrapd"\ +] +RUN ["chmod", "644", "/etc/tox-bootstrapd.conf"] +RUN ["chmod", "700", "/var/lib/tox-bootstrapd"] WORKDIR /var/lib/tox-bootstrapd USER tox-bootstrapd -ENTRYPOINT /usr/local/bin/tox-bootstrapd \ - --config /etc/tox-bootstrapd.conf \ - --log-backend stdout \ - --foreground +ENTRYPOINT ["/usr/local/bin/tox-bootstrapd",\ + "--config", "/etc/tox-bootstrapd.conf",\ + "--log-backend", "stdout",\ + "--foreground"\ +] EXPOSE 443/tcp 3389/tcp 33445/tcp 33445/udp diff --git a/other/bootstrap_daemon/docker/get-nodes.py b/other/bootstrap_daemon/docker/get-nodes.py old mode 100644 new mode 100755 index 821f089a..f38bd492 --- a/other/bootstrap_daemon/docker/get-nodes.py +++ b/other/bootstrap_daemon/docker/get-nodes.py @@ -24,26 +24,32 @@ THE SOFTWARE. # Gets a list of nodes from https://nodes.tox.chat/json and prints them out # in the format of tox-bootstrapd config file. -import urllib.request import json +import sys +import urllib.request response = urllib.request.urlopen('https://nodes.tox.chat/json') raw_json = response.read().decode('ascii', 'ignore') nodes = json.loads(raw_json)['nodes'] -output = 'bootstrap_nodes = (\n' - -for node in nodes: +def node_to_string(node): node_output = ' { // ' + node['maintainer'] + '\n' node_output += ' public_key = "' + node['public_key'] + '"\n' node_output += ' port = ' + str(node['port']) + '\n' node_output += ' address = "' if len(node['ipv4']) > 4: - output += node_output + node['ipv4'] + '"\n },\n' + return node_output + node['ipv4'] + '"\n }' if len(node['ipv6']) > 4: - output += node_output + node['ipv6'] + '"\n },\n' + return node_output + node['ipv6'] + '"\n }' + raise Exception('no IP address found for node ' + json.dumps(node)) -# remove last comma -output = output[:-2] + '\n)\n' +output = ('bootstrap_nodes = (\n' + + ',\n'.join(map(node_to_string, nodes)) + + '\n)') -print(output) +if len(sys.argv) > 1: + with open(sys.argv[1], 'a') as fh: + fh.write(output + '\n') + print("Wrote %d nodes to %s" % (len(nodes), sys.argv[1])) +else: + print(output) diff --git a/other/bootstrap_daemon/docker/tox-bootstrapd.sha256 b/other/bootstrap_daemon/docker/tox-bootstrapd.sha256 new file mode 100644 index 00000000..775b095e --- /dev/null +++ b/other/bootstrap_daemon/docker/tox-bootstrapd.sha256 @@ -0,0 +1 @@ +b99d22c87a2bcaa93eba33f2a140f1c1624f846b4f2d87ada9fc2d7788f13193 /usr/local/bin/tox-bootstrapd diff --git a/other/fun/bootstrap_node_info.py b/other/fun/bootstrap_node_info.py old mode 100644 new mode 100755 index ffda41e3..f4f5f73d --- a/other/fun/bootstrap_node_info.py +++ b/other/fun/bootstrap_node_info.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Copyright (c) 2014 by nurupo