Simplify Travis-CI FreeBSD build

This commit is contained in:
Maxim Biro 2018-07-08 10:22:32 -04:00
parent b4cf9808e9
commit c0b4cd156f
No known key found for this signature in database
GPG Key ID: FAF0E1CD60A4A4A0
10 changed files with 240 additions and 307 deletions

View File

@ -74,8 +74,8 @@ matrix:
install: .travis/$JOB install
script: .travis/$JOB script
- stage: "Stage 1"
if: type IN (push, api, cron)
env: JOB=cmake-freebsd
if: type IN (push, api, cron)
sudo: required
install: .travis/$JOB-stage1 install
script: .travis/$JOB-stage1 script
@ -85,8 +85,8 @@ matrix:
services: [docker]
script: .travis/$JOB
- stage: "Stage 2"
if: type IN (push, api, cron)
env: JOB=cmake-freebsd
if: type IN (push, api, cron)
sudo: required
install: .travis/$JOB-stage2 install
script: .travis/$JOB-stage2 script

View File

@ -0,0 +1,10 @@
#!/bin/sh
NPROC=`nproc`
SCREEN_SESSION=freebsd
SSH_PORT=10022
RUN() {
ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@"
}

View File

@ -30,7 +30,7 @@ mkdir -p /opt/freebsd/cache
cd /opt/freebsd/cache
# Make sure to update DL_SHA512 when bumping the version
FREEBSD_VERSION="11.1"
FREEBSD_VERSION="11.2"
IMAGE_NAME=FreeBSD-${FREEBSD_VERSION}-RELEASE-amd64.raw
# Sends keys to the VM as they are
@ -56,7 +56,7 @@ start_vm()
# Start emulator. 2000mb RAM should be enough, right? The build machine has over 7gb.
screen -L -S $SCREEN_SESSION -d -m \
qemu-system-x86_64 -curses -m 2000 -smp $NPROC \
-net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic $IMAGE_NAME
-net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic "$IMAGE_NAME"
# Wait for the boot screen options
wait_for "Autoboot in"
@ -73,7 +73,8 @@ start_vm()
stop_vm()
{
# Turn it off
RUN poweroff
send_keys 'poweroff
'
# Wait for qemu process to terminate
while ps aux | grep qemu | grep -vq grep

View File

@ -2,14 +2,174 @@
ACTION="$1"
set -eu
set -eux
. other/travis/env.sh
. other/travis/env-freebsd.sh
. .travis/cmake-freebsd-env.sh
travis_install() {
. other/travis/freebsd-install
. other/travis/freebsd-install-stage1
. .travis/cmake-freebsd-install.sh
git tag -l --sort=version:refname > GIT_TAGS
OLD_PWD="$PWD"
mkdir -p /opt/freebsd/cache
cd /opt/freebsd/cache
# === Get the VM image, set it up and cache ===
# Create image if it's not cached or if this build script has changed
sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh" > /tmp/sha
sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> /tmp/sha
sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1" >> /tmp/sha
if [ ! -f "./$IMAGE_NAME.tgz" ] || [ ! -f ./cmake-freebsd-stage1-all.sha256 ] || [ "`cat cmake-freebsd-stage1-all.sha256`" != "`cat /tmp/sha`" ]; then
rm -rf ./*
# https://download.freebsd.org/ftp/releases/VM-IMAGES/11.2-RELEASE/amd64/Latest/
DL_SHA512="0c3c232c7023c5036daeb5fbf68c2ddecf9703c74e317afcf19da91e83d0afcc526785571e2868894ce15cdb56b74fafa1ce9fd216469db91e021ac2ef8911e5"
# Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html
# Note that not all mirrors listed on that page are working, so we have removed them
# I'm so sorry, there are no arrays in sh and we are not using bash...
DL_MIRROR_1=1
DL_MIRROR_2=4
# There are 2 mirrors
DL_MIRROR_RANDOM=`expr $(date +%s) % 2 + 1`
DL_URL="ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz"
wget "$DL_URL"
if ! ( echo "$DL_SHA512 $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then
echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one"
exit 1
fi
unxz "$IMAGE_NAME.xz"
# With this we don't have to guess how long a command will run for and sleeping
# for that amount of time, risking either under sleeping or over sleeping, instead
# we will sleep exactly until the command is finished by printing out a unique
# string after the command is executed and then checking if it was printed.
execute_shell_and_wait()
{
# $RANDOM is a bash built-in, so we try to avoid name collision here by using ugly RANDOM_STR name
RANDOM_STR=$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c16)
send_keys "$1;echo $RANDOM_STR
"
# \[1B is a control escape sequence for a new line in the terminal.
# We want to wait for <new-line>$RANDOM_STR instead of just $RANDOM_STR because
# $RANDOM_STR we have inputted with send_keys above would appear in the screenlog.0
# file and we don't want to match our input, we want to match the echo's output.
# The .\? optionally matches any character. Sometimes it happens that there is some
# random character inserved between the new line control escape sequence and $RANDOM_STR.
wait_for "\[1B.\?$RANDOM_STR"
}
start_vm
# Login as root user
send_keys 'root
'
# Wait for the prompt
wait_for "root@.*:~"
# Configure network, ssh and start changing password
execute_shell_and_wait 'echo "ifconfig_em0=DHCP" >> /etc/rc.conf'
execute_shell_and_wait 'echo "Port 22" >> /etc/ssh/sshd_config'
execute_shell_and_wait 'echo "PermitRootLogin yes" >> /etc/ssh/sshd_config'
execute_shell_and_wait 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config'
execute_shell_and_wait 'echo "PermitEmptyPasswords yes" >> /etc/ssh/sshd_config'
execute_shell_and_wait 'echo "sshd_enable=YES" >> /etc/rc.conf'
send_keys 'sh /etc/rc.d/netif restart && sh /etc/rc.d/sshd start && passwd
'
# Wait for the password prompt
wait_for "Changing local password for root"
# Reset password to empty for the passwordless ssh to work
send_keys '
'
wait_for "New Password"
send_keys '
'
# Update system
RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
# It fails if there is nothing to install, so we make it always succeed with true
RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true
# Update packages
RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg upgrade
# Install and set bash as the default shell for the root user
RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg install bash
RUN chsh -s /usr/local/bin/bash root
# Install required toxcore dependencies
RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg install git \
opus \
libvpx \
libsodium \
gmake \
cmake \
pkgconf \
opencv \
portaudio \
libsndfile \
texinfo \
autotools
# === Cache the VM image ===
stop_vm
# Create cache
tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
rm "$IMAGE_NAME"
rm screenlog.0
cp "$OLD_PWD/GIT_TAGS" .
sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh" > cmake-freebsd-stage1-all.sha256
sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> cmake-freebsd-stage1-all.sha256
sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1" >> cmake-freebsd-stage1-all.sha256
ls -lh
fi
# === Update the image on new version (tag) of toxcore ===
if ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS" ; then
# Extract the cached image
tar -Sxzvf "$IMAGE_NAME.tgz"
start_vm
# Update system
RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true
# Update packages
RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg upgrade
# === Cache the updated VM image ===
stop_vm
# Create/Update cache
rm "$IMAGE_NAME.tgz"
tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
rm "$IMAGE_NAME"
rm screenlog.0
cp "$OLD_PWD/GIT_TAGS" .
ls -lh
fi
cd "$OLD_PWD"
}
travis_script() {

View File

@ -2,18 +2,71 @@
ACTION="$1"
set -eu
set -eux
. other/travis/env.sh
. other/travis/env-freebsd.sh
. .travis/cmake-freebsd-env.sh
travis_install() {
. other/travis/freebsd-install
. other/travis/freebsd-install-stage2
. .travis/cmake-freebsd-install.sh
OLD_PWD="$PWD"
mkdir -p /opt/freebsd/cache
cd /opt/freebsd/cache
# Extract the cached image
tar -Sxzvf "$IMAGE_NAME.tgz"
# Get the image we will be using out of the cached directory
mv "$IMAGE_NAME" ..
ls -lh
cd ..
ls -lh
# === Get VM ready to build the code ===
start_vm
# Display FreeBSD kernel info and last login
RUN uname -a
RUN last
cd "$OLD_PWD"
# Copy over toxcore code from Travis to qemu
scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -P $SSH_PORT -r ./* root@localhost:~
RUN ls -lh
}
travis_script() {
. other/travis/toxcore-script
CACHEDIR="none"
. ".travis/flags-$CC.sh"
add_ld_flag -Wl,-z,defs
# Make compilation error on a warning
add_flag -Werror
RUN 'cmake -B_build -H. \
-DCMAKE_C_FLAGS="$C_FLAGS" \
-DCMAKE_CXX_FLAGS="$CXX_FLAGS" \
-DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \
-DCMAKE_SHARED_LINKER_FLAGS="$LD_FLAGS" \
-DCMAKE_INSTALL_PREFIX:PATH="_install" \
-DTRACE=ON \
-DMUST_BUILD_TOXAV=ON \
-DSTRICT_ABI=ON \
-DTEST_TIMEOUT_SECONDS=120 \
-DUSE_IPV6=OFF'
# We created the VM with the same number of cores as the host, so the host-ran `nproc` here is fine
RUN 'gmake "-j$NPROC" -k install -C_build'
RUN 'gmake "-j$NPROC" test ARGS="-j50" -C_build || \
gmake "-j$NPROC" -C_build test ARGS="-j50 --rerun-failed" CTEST_OUTPUT_ON_FAILURE=1 || \
true'
}
if [ "-z" "$ACTION" ]; then

View File

@ -1,31 +0,0 @@
#!/bin/sh
CMAKE=cmake
NPROC=`nproc`
CURDIR=/root
RUN_TESTS=true
MAKE=gmake
# A lot of tests fail and run for the full 2 minutes allowed, resulting in
# Travis build timing out, so we restrict it to just 1 test run until enough
# tests are fixed so that they succeed and don't run the full 2 minutes.
MAX_TEST_RETRIES=1
SCREEN_SESSION=freebsd
SSH_PORT=10022
RUN() {
ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@"
}
TESTS() {
COUNT="$1"; shift
RUN "$@" || {
if [ $COUNT -gt 1 ]; then
TESTS `expr $COUNT - 1` "$@"
else
# FIXME: We allow the tests to fail for now, but this should be changed to
# "false" once we fix tests under FreeBSD
true
fi
}
}

View File

@ -1,17 +0,0 @@
#!/bin/sh
# Globally used environment variables.
export CACHE_DIR=$HOME/cache
export OPAMROOT=$CACHE_DIR/.opam
export LD_LIBRARY_PATH=$CACHE_DIR/lib
export PKG_CONFIG_PATH=$CACHE_DIR/lib/pkgconfig
export ASTYLE=$CACHE_DIR/astyle/build/gcc/bin/astyle
export CFLAGS="-O3 -DTRAVIS_ENV=1"
export CXXFLAGS="-O3 -DTRAVIS_ENV=1"
export MAKE=make
BUILD_DIR=_build
MAX_TEST_RETRIES=10
# Workaround for broken Travis image.
export TERM=xterm

View File

@ -1,164 +0,0 @@
#!/bin/sh
git tag -l --sort=version:refname > GIT_TAGS
OLD_PWD="$PWD"
mkdir -p /opt/freebsd/cache
cd /opt/freebsd/cache
# === Get the VM image, set it up and cache ===
# Create image if it's not cached or if this build script has changed
if [ ! -f ./$IMAGE_NAME.tgz ] || [ ! -f ./freebsd-install-stage1.sha256 ] || [ "`cat freebsd-install-stage1.sha256`" != "`sha256sum $OLD_PWD/other/travis/freebsd-install-stage1`" ]; then
rm -rf ./*
# https://download.freebsd.org/ftp/releases/VM-IMAGES/11.1-RELEASE/amd64/Latest/
DL_SHA512="c569776334131fdc85cd25a2a0d5aecafdc3e4b2e6e010dffaa2488d934293ce4f091f23481079dd91ad20dfd2dfc3d3487707096c59448f1d8914c5d7d6b582"
# Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html
# Note that not all mirrors listed on that page are working, so we have removed them
# I'm so sorry, there are no arrays in sh and we are not using bash...
DL_MIRROR_1=1
DL_MIRROR_2=4
DL_MIRROR_3=5
DL_MIRROR_4=6
DL_MIRROR_5=8
DL_MIRROR_6=10
DL_MIRROR_7=14
DL_MIRROR_8=15
# There are 8 mirrors
DL_MIRROR_RANDOM=`expr $(date +%s) % 8 + 1`
DL_URL=ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz
wget $DL_URL
if ! ( echo "$DL_SHA512 $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then
echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one"
exit 1
fi
unxz $IMAGE_NAME.xz
# With this we don't have to guess how long a command will run for and sleeping
# for that amount of time, risking either under sleeping or over sleeping, instead
# we will sleep exactly until the command is finished by printing out a unique
# string after the command is executed and then checking if it was printed.
execute_shell_and_wait()
{
# $RANDOM is a bash built-in, so we try to avoid name collision here by using ugly RANDOM_STR name
RANDOM_STR=$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c16)
send_keys "$1;echo $RANDOM_STR
"
# \[1B is a control escape sequence for a new line in the terminal.
# We want to wait for <new-line>$RANDOM_STR instead of just $RANDOM_STR because
# $RANDOM_STR we have inputted with send_keys above would appear in the screenlog.0
# file and we don't want to match our input, we want to match the echo's output.
# The .\? optionally matches any character. Sometimes it happens that there is some
# random character inserved between the new line control escape sequence and $RANDOM_STR.
wait_for "\[1B.\?$RANDOM_STR"
}
start_vm
# Login as root user
send_keys 'root
'
# Wait for the prompt
wait_for "root@:~"
# Configure network, ssh and start changing password
execute_shell_and_wait 'echo "ifconfig_em0=DHCP" >> /etc/rc.conf'
execute_shell_and_wait 'echo "Port 22" >> /etc/ssh/sshd_config'
execute_shell_and_wait 'echo "PermitRootLogin yes" >> /etc/ssh/sshd_config'
execute_shell_and_wait 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config'
execute_shell_and_wait 'echo "PermitEmptyPasswords yes" >> /etc/ssh/sshd_config'
execute_shell_and_wait 'echo "sshd_enable=YES" >> /etc/rc.conf'
send_keys 'sh /etc/rc.d/netif restart && sh /etc/rc.d/sshd start && passwd
'
# Wait for the password prompt
wait_for "Changing local password for root"
# Reset password to empty for the passwordless ssh to work
send_keys '
'
wait_for "New Password"
send_keys '
'
# Update system
RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
# It fails if there is nothing to install, so we make it always succeed with true
RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true
# Update packages
RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg upgrade
# Install and set bash as the default shell for the root user
RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg install bash
RUN chsh -s /usr/local/bin/bash root
# Install required toxcore dependencies
RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg install git \
opus \
libvpx \
libsodium \
gmake \
cmake \
pkgconf \
opencv \
portaudio \
libsndfile \
texinfo \
autotools
# === Cache the VM image ===
stop_vm
# Create cache
tar -Sczvf $IMAGE_NAME.tgz $IMAGE_NAME
rm $IMAGE_NAME
rm screenlog.0
cp "$OLD_PWD/GIT_TAGS" .
sha256sum "$OLD_PWD/other/travis/freebsd-install-stage1" > freebsd-install-stage1.sha256
ls -lh
fi
# === Update the image on new version (tag) of toxcore ===
if ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS" ; then
# Extract the cached image
tar -Sxzvf $IMAGE_NAME.tgz
start_vm
# Update system
RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true
# Update packages
RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg upgrade
# === Cache the updated VM image ===
stop_vm
# Create/Update cache
rm $IMAGE_NAME.tgz
tar -Sczvf $IMAGE_NAME.tgz $IMAGE_NAME
rm $IMAGE_NAME
rm screenlog.0
cp "$OLD_PWD/GIT_TAGS" .
ls -lh
fi
cd "$OLD_PWD"

View File

@ -1,32 +0,0 @@
#!/bin/sh
OLD_PWD="$PWD"
mkdir -p /opt/freebsd/cache
cd /opt/freebsd/cache
# Extract the cached image
tar -Sxzvf $IMAGE_NAME.tgz
# Get the image we will be using out of the cached directory
mv $IMAGE_NAME ..
ls -lh
cd ..
ls -lh
# === Get VM ready to build the code ===
start_vm
# Display FreeBSD kernel info and last login
RUN uname -a
RUN last
cd "$OLD_PWD"
# Copy over toxcore code from Travis to qemu
scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -P $SSH_PORT -r ./* root@localhost:~
RUN ls -lh

View File

@ -1,47 +0,0 @@
#!/bin/sh
# Enable test coverage recording.
export CFLAGS="$CFLAGS -fprofile-arcs -ftest-coverage"
export CXXFLAGS="$CXXFLAGS -fprofile-arcs -ftest-coverage"
set_opt() {
opts="$opts -D$1="`expr ON \& \( $i % 2 \) \| OFF`
set +e # result can be 0
i=`expr $i / 2`
set -e
}
if perl -e '
use Socket qw(PF_INET6 SOCK_STREAM pack_sockaddr_in6 IN6ADDR_LOOPBACK);
socket(S, PF_INET6, SOCK_STREAM, 0) || exit 1;
bind(S, pack_sockaddr_in6(0, IN6ADDR_LOOPBACK)) || exit 1;
'; then
USE_IPV6=yes
else
USE_IPV6=no
fi
# Build toxcore and run tests.
RUN $CMAKE \
-B$BUILD_DIR \
-H. \
-DCMAKE_INSTALL_PREFIX:PATH=$CURDIR/_install \
-DDEBUG=ON \
-DMUST_BUILD_TOXAV=ON \
-DSTRICT_ABI=ON \
-DTEST_TIMEOUT_SECONDS=120 \
-DTRACE=ON \
-DUSE_IPV6=$USE_IPV6
export CTEST_OUTPUT_ON_FAILURE=1
RUN $MAKE -C$BUILD_DIR -j$NPROC -k install
if $RUN_TESTS; then
TESTS $MAX_TEST_RETRIES $MAKE -C$BUILD_DIR -j$NPROC test ARGS="-j50 --rerun-failed" CTEST_OUTPUT_ON_FAILURE="$CTEST_OUTPUT_ON_FAILURE"
fi
if [ -f "$ASTYLE" ]; then
other/astyle/format-source . $ASTYLE
fi