From 5530e417428c7d5a3165db65ae4dc9f6086b3eb2 Mon Sep 17 00:00:00 2001 From: Maxim Biro Date: Thu, 18 Oct 2018 04:23:21 -0400 Subject: [PATCH] Increase NOFILE limit for tox-bootstrapd tox-bootstrapd can use around 600 TCP sockets during TCP server's normal functioning. Many systems default to having a soft limit of 1024 open file descriptors, which we are close to reaching, so it was suggested we bump that limit to a higher number. iphy suggested increasing it to 32768. --- .travis/tox-bootstrapd-docker | 8 ++++- other/bootstrap_daemon/README.md | 2 +- other/bootstrap_daemon/src/tox-bootstrapd.c | 29 ++++++++++++++- other/bootstrap_daemon/tox-bootstrapd.service | 3 ++ other/bootstrap_daemon/tox-bootstrapd.sh | 35 +++++++++++++++++-- 5 files changed, 72 insertions(+), 5 deletions(-) diff --git a/.travis/tox-bootstrapd-docker b/.travis/tox-bootstrapd-docker index 7c12a9d0..5dc39628 100755 --- a/.travis/tox-bootstrapd-docker +++ b/.travis/tox-bootstrapd-docker @@ -23,7 +23,7 @@ cat docker/Dockerfile sudo docker build -t tox-bootstrapd docker/ sudo useradd --home-dir /var/lib/tox-bootstrapd --create-home --system --shell /sbin/nologin --comment "Account to run Tox's DHT bootstrap daemon" --user-group tox-bootstrapd sudo chmod 700 /var/lib/tox-bootstrapd -sudo docker run -d --name tox-bootstrapd -v /var/lib/tox-bootstrapd/:/var/lib/tox-bootstrapd/ -p 443:443 -p 3389:3389 -p 33445:33445 -p 33445:33445/udp tox-bootstrapd +sudo docker run -d --name tox-bootstrapd -v /var/lib/tox-bootstrapd/:/var/lib/tox-bootstrapd/ --ulimit nofile=32768:32768 -p 443:443 -p 3389:3389 -p 33445:33445 -p 33445:33445/udp tox-bootstrapd sudo ls -lbh /var/lib/tox-bootstrapd @@ -59,6 +59,12 @@ if [ "`sudo docker inspect -f {{.State.Running}} tox-bootstrapd`" != "true" ]; t exit 1 fi +cat /proc/$(pidof tox-bootstrapd)/limits +if ! cat /proc/$(pidof tox-bootstrapd)/limits | grep -P '^Max open files(\s+)32768(\s+)32768(\s+)files'; then + echo "Error: ulimit is not set to the expected value" + exit 1 +fi + if ! python3 ../fun/bootstrap_node_info.py ipv4 localhost 33445 ; then echo "Error: Unable to get bootstrap node info" exit 1 diff --git a/other/bootstrap_daemon/README.md b/other/bootstrap_daemon/README.md index 848f9fa1..541305b1 100644 --- a/other/bootstrap_daemon/README.md +++ b/other/bootstrap_daemon/README.md @@ -219,7 +219,7 @@ sudo docker build -t tox-bootstrapd docker/ sudo useradd --home-dir /var/lib/tox-bootstrapd --create-home --system --shell /sbin/nologin --comment "Account to run Tox's DHT bootstrap daemon" --user-group tox-bootstrapd sudo chmod 700 /var/lib/tox-bootstrapd -sudo docker run -d --name tox-bootstrapd --restart always -v /var/lib/tox-bootstrapd/:/var/lib/tox-bootstrapd/ -p 443:443 -p 3389:3389 -p 33445:33445 -p 33445:33445/udp tox-bootstrapd +sudo docker run -d --name tox-bootstrapd --restart always -v /var/lib/tox-bootstrapd/:/var/lib/tox-bootstrapd/ --ulimit nofile=32768:32768 -p 443:443 -p 3389:3389 -p 33445:33445 -p 33445:33445/udp tox-bootstrapd ``` We create a new user and protect its home directory in order to mount it in the Docker image, so that the kyepair the daemon uses would be stored on the host system, which makes it less likely that you would loose the keypair while playing with or updating the Docker container. diff --git a/other/bootstrap_daemon/src/tox-bootstrapd.c b/other/bootstrap_daemon/src/tox-bootstrapd.c index 5dba9dcc..88f45494 100644 --- a/other/bootstrap_daemon/src/tox-bootstrapd.c +++ b/other/bootstrap_daemon/src/tox-bootstrapd.c @@ -27,11 +27,14 @@ #endif // system provided +#include #include -#include // for POSIX sigaction(2) +#include // system header, rather than C, because we need it for POSIX sigaction(2) #include // C +#include +#include #include #include #include @@ -424,6 +427,30 @@ int main(int argc, char *argv[]) if (tcp_server != nullptr) { log_write(LOG_LEVEL_INFO, "Initialized Tox TCP server successfully.\n"); + + struct rlimit limit; + + const rlim_t rlim_suggested = 32768; + const rlim_t rlim_min = 4096; + + assert(rlim_suggested >= rlim_min); + + if (!getrlimit(RLIMIT_NOFILE, &limit)) { + if (limit.rlim_cur < limit.rlim_max) { + // Some systems have a hard limit of over 1000000 open file descriptors, so let's cap it at something reasonable + // so that we don't set it to an unreasonably high number. + limit.rlim_cur = limit.rlim_max > rlim_suggested ? rlim_suggested : limit.rlim_max; + setrlimit(RLIMIT_NOFILE, &limit); + } + } + + if (!getrlimit(RLIMIT_NOFILE, &limit) && limit.rlim_cur < rlim_min) { + log_write(LOG_LEVEL_WARNING, + "Current limit on the number of files this process can open (%ju) is rather low for the proper functioning of the TCP server. " + "Consider raising the limit to at least %ju or the recommended %ju. " + "Continuing using the current limit (%ju).\n", + (uintmax_t)limit.rlim_cur, (uintmax_t)rlim_min, (uintmax_t)rlim_suggested, (uintmax_t)limit.rlim_cur); + } } else { log_write(LOG_LEVEL_ERROR, "Couldn't initialize Tox TCP server. Exiting.\n"); kill_onion_announce(onion_a); diff --git a/other/bootstrap_daemon/tox-bootstrapd.service b/other/bootstrap_daemon/tox-bootstrapd.service index 20f698d2..88ea9afa 100644 --- a/other/bootstrap_daemon/tox-bootstrapd.service +++ b/other/bootstrap_daemon/tox-bootstrapd.service @@ -11,6 +11,9 @@ WorkingDirectory=/var/lib/tox-bootstrapd ExecStart=/usr/local/bin/tox-bootstrapd --config /etc/tox-bootstrapd.conf User=tox-bootstrapd Group=tox-bootstrapd +# TCP Server needs to be able to have lots of TCP sockets open. +LimitNOFILE=32768 +# Uncomment to allow binding to ports < 1024, e.g. 443 (HTTPS) for TCP Server #CapabilityBoundingSet=CAP_NET_BIND_SERVICE [Install] diff --git a/other/bootstrap_daemon/tox-bootstrapd.sh b/other/bootstrap_daemon/tox-bootstrapd.sh index d33c38da..5f1e5c68 100644 --- a/other/bootstrap_daemon/tox-bootstrapd.sh +++ b/other/bootstrap_daemon/tox-bootstrapd.sh @@ -1,4 +1,4 @@ -#! /bin/sh +#! /bin/bash ### BEGIN INIT INFO # Provides: tox-bootstrapd # Required-Start: $remote_fs $syslog @@ -22,6 +22,35 @@ SCRIPTNAME=/etc/init.d/$NAME USER=tox-bootstrapd GROUP=tox-bootstrapd +# Set ulimit -n based on number of fds available. +# This check is borrowed from Debian's tor package, with a few modifications. +if [ -r /proc/sys/fs/file-max ]; then + system_max=$(cat /proc/sys/fs/file-max) + if [ "$system_max" -gt "80000" ] ; then + MAX_FILEDESCRIPTORS=32768 + elif [ "$system_max" -gt "40000" ] ; then + MAX_FILEDESCRIPTORS=16384 + elif [ "$system_max" -gt "20000" ] ; then + MAX_FILEDESCRIPTORS=8192 + elif [ "$system_max" -gt "10000" ] ; then + MAX_FILEDESCRIPTORS=4096 + else + MAX_FILEDESCRIPTORS=1024 + cat << EOF + +Warning: Your system has very few file descriptors available in total. + +Maybe you should try raising that by adding 'fs.file-max=100000' to your +/etc/sysctl.conf file. Feel free to pick any number that you deem appropriate. +Then run 'sysctl -p'. See /proc/sys/fs/file-max for the current value, and +file-nr in the same directory for how many of those are used at the moment. + +EOF + fi +else + MAX_FILEDESCRIPTORS=32768 +fi + # Exit if the package is not installed [ -x "$DAEMON" ] || exit 0 @@ -48,6 +77,8 @@ do_start() fi chown $USER:$GROUP $PIDDIR start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --test --chuid $USER > /dev/null || return 1 + # TCP Server needs to be able to have lots of TCP sockets open. + ulimit -n $MAX_FILEDESCRIPTORS start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON --chuid $USER -- $DAEMON_ARGS || return 2 } @@ -113,4 +144,4 @@ case "$1" in exit 3 ;; esac -exit 0 \ No newline at end of file +exit 0