mirror of
https://github.com/irungentoo/toxcore.git
synced 2024-03-22 13:30:51 +08:00
Update and fix FreeBSD setup on Travis-CI
- Bump FreeBSD to 12.1. - Simplify stage1 logic. - Re-try downloading the image from a different mirror if one fails. - Use the `expect` utility instead of dealing with screen's log file. - Re-run failed toxcore test one more time and in sequence.
This commit is contained in:
parent
bd7b7fadba
commit
26fd89437b
|
@ -1,10 +1,44 @@
|
|||
#!/bin/sh
|
||||
|
||||
NPROC=`nproc`
|
||||
# Common variables and functions
|
||||
|
||||
NPROC=$(nproc)
|
||||
|
||||
SCREEN_SESSION=freebsd
|
||||
SSH_PORT=10022
|
||||
|
||||
FREEBSD_VERSION="12.1"
|
||||
IMAGE_NAME=FreeBSD-${FREEBSD_VERSION}-RELEASE-amd64.raw
|
||||
# https://download.freebsd.org/ftp/releases/VM-IMAGES/12.1-RELEASE/amd64/Latest/
|
||||
IMAGE_SHA512="a65da6260f5f894fc86fbe1f27dad7800906da7cffaa5077f82682ab74b6dd46c4ce87158c14b726d74ca3c6d611bea3bb336164da3f1cb990550310b110da22"
|
||||
|
||||
RUN() {
|
||||
ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@"
|
||||
ssh -t -o ConnectionAttempts=120 -o ConnectTimeout=2 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@"
|
||||
}
|
||||
|
||||
start_vm() {
|
||||
screen -d -m qemu-system-x86_64 -curses -m 2048 -smp $NPROC -net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic "$IMAGE_NAME"
|
||||
|
||||
# Wait for ssh to start listening on the port
|
||||
while ! echo "exit" | nc localhost ${SSH_PORT} | grep 'OpenSSH'; do
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# Test that ssh works
|
||||
RUN uname -a
|
||||
RUN last
|
||||
}
|
||||
|
||||
stop_vm()
|
||||
{
|
||||
# Turn it off
|
||||
# We use this contraption because for some reason `shutdown -h now` and
|
||||
# `poweroff` result in FreeBSD not shutting down on Travis (they work on my
|
||||
# machine though)
|
||||
RUN "shutdown -p +5sec && sleep 30" || true
|
||||
|
||||
# Wait for the qemu process to terminate
|
||||
while pgrep qemu; do
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
|
|
@ -1,89 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
# Travis doesn't provide FreeBSD machines, so we just take a Linux one and run
|
||||
# FreeBSD in qemu virtual machine. qemu is being ran in curses mode inside a
|
||||
# screen session, because screen allows to easily send input and read output.
|
||||
# The input is sent using `screen -S session-name -X stuff ...` and the output
|
||||
# is read from the screen's log file. Note that for some reason you can't send
|
||||
# long input lines on Travis (it works just fine when I do it on my machine...),
|
||||
# but that limitation is not an issue, as we don't really need to send long
|
||||
# lines of input anyway. Also, note that since we run qemu in curses mode, the
|
||||
# output contains control characters intended for a terminal emulator telling
|
||||
# how to position and color the text, so it might be a little tricky to read it
|
||||
# sometimes. The only time when this script has to send input to and read the
|
||||
# output from the screen session is during the initial setup when we setup the
|
||||
# network, install and configure the ssh server, and update the system. After
|
||||
# this initial setup, ssh is used to communicate with the FreeBSD running in the
|
||||
# VM, which is a lot friendlier way of communication. Please note that Travis
|
||||
# doesn't seem to allow KVM passthrough, so qemu has to emulate all the
|
||||
# hardware, which makes it quite slow compared to the host machine. We cache
|
||||
# the qemu image since it takes a long time to run the initial system and
|
||||
# package updates, and we do incremental system and package updates on every
|
||||
# change to the list of git tags (i.e. on every toxcore release, presumably).
|
||||
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y qemu
|
||||
|
||||
OLD_PWD="$PWD"
|
||||
|
||||
mkdir -p /opt/freebsd/cache
|
||||
cd /opt/freebsd/cache
|
||||
|
||||
# Make sure to update DL_SHA512 when bumping the version
|
||||
FREEBSD_VERSION="11.2"
|
||||
IMAGE_NAME=FreeBSD-${FREEBSD_VERSION}-RELEASE-amd64.raw
|
||||
|
||||
# Sends keys to the VM as they are
|
||||
send_keys()
|
||||
{
|
||||
screen -S $SCREEN_SESSION -X stuff "$1"
|
||||
}
|
||||
|
||||
# Blocks until a specific text appears on VM's screen
|
||||
wait_for()
|
||||
{
|
||||
while ! grep -q "$1" screenlog.0
|
||||
do
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# Starts VM and waits until it's fully running (until a login prompt is shown)
|
||||
start_vm()
|
||||
{
|
||||
rm -f screenlog.0
|
||||
|
||||
# Start emulator. 2000mb RAM should be enough, right? The build machine has over 7gb.
|
||||
screen -L -S $SCREEN_SESSION -d -m \
|
||||
qemu-system-x86_64 -curses -m 2000 -smp $NPROC \
|
||||
-net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic "$IMAGE_NAME"
|
||||
|
||||
# Wait for the boot screen options
|
||||
wait_for "Autoboot in"
|
||||
|
||||
# Select the 1st option
|
||||
send_keys '
|
||||
'
|
||||
|
||||
# Wait for the system to boot and present the login prompt
|
||||
wait_for "FreeBSD/amd64 ("
|
||||
}
|
||||
|
||||
# Shuts VM down and waits until its process finishes
|
||||
stop_vm()
|
||||
{
|
||||
# Turn it off
|
||||
send_keys 'poweroff
|
||||
'
|
||||
|
||||
# Wait for qemu process to terminate
|
||||
while ps aux | grep qemu | grep -vq grep
|
||||
do
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# Let's see what's in the cache directory
|
||||
ls -lh
|
||||
|
||||
cd "$OLD_PWD"
|
|
@ -1,5 +1,7 @@
|
|||
#!/bin/sh
|
||||
|
||||
# Download and initial setup of the FreeBSD VM
|
||||
|
||||
ACTION="$1"
|
||||
|
||||
set -eux
|
||||
|
@ -7,8 +9,6 @@ set -eux
|
|||
. .travis/cmake-freebsd-env.sh
|
||||
|
||||
travis_install() {
|
||||
. .travis/cmake-freebsd-install.sh
|
||||
|
||||
git tag -l --sort=version:refname > GIT_TAGS
|
||||
|
||||
OLD_PWD="$PWD"
|
||||
|
@ -18,84 +18,56 @@ travis_install() {
|
|||
|
||||
# === Get the VM image, set it up and cache ===
|
||||
|
||||
# Create image if it's not cached or if this build script has changed
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh" > /tmp/sha
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> /tmp/sha
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1" >> /tmp/sha
|
||||
if [ ! -f "./$IMAGE_NAME.tgz" ] || [ ! -f ./cmake-freebsd-stage1-all.sha256 ] || [ "`cat cmake-freebsd-stage1-all.sha256`" != "`cat /tmp/sha`" ]; then
|
||||
|
||||
# Create image if it's not cached, or if this build script has changed, or a new toxcore tag was pushed
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh" > /tmp/sha
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1" >> /tmp/sha
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1.expect" >> /tmp/sha
|
||||
if [ ! -f "./$IMAGE_NAME.tgz" ] || [ ! -f ./cmake-freebsd-stage1-all.sha256 ] || [ "$(cat cmake-freebsd-stage1-all.sha256)" != "$(cat /tmp/sha)" ] || ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS"; then
|
||||
rm -rf ./*
|
||||
|
||||
# https://download.freebsd.org/ftp/releases/VM-IMAGES/11.2-RELEASE/amd64/Latest/
|
||||
DL_SHA512="0c3c232c7023c5036daeb5fbf68c2ddecf9703c74e317afcf19da91e83d0afcc526785571e2868894ce15cdb56b74fafa1ce9fd216469db91e021ac2ef8911e5"
|
||||
# Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html
|
||||
# Note that not all mirrors listed on that page are working, so we have removed them
|
||||
# I'm so sorry, there are no arrays in sh and we are not using bash...
|
||||
DL_MIRROR_1=1
|
||||
DL_MIRROR_2=4
|
||||
# There are 2 mirrors
|
||||
DL_MIRROR_RANDOM=`expr $(date +%s) % 2 + 1`
|
||||
DL_URL="ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz"
|
||||
while true; do
|
||||
# Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html
|
||||
# Note that not all mirrors listed on that page are working, so we have removed them
|
||||
# There are no arrays in sh so we get a bit clever
|
||||
DL_MIRROR_1=1
|
||||
DL_MIRROR_2=2
|
||||
DL_MIRROR_3=3
|
||||
DL_MIRROR_4=4
|
||||
DL_MIRROR_5=5
|
||||
DL_MIRROR_6=6
|
||||
DL_MIRROR_7=7
|
||||
DL_MIRROR_8=10
|
||||
DL_MIRROR_9=11
|
||||
DL_MIRROR_10=13
|
||||
DL_MIRROR_11=14
|
||||
|
||||
wget "$DL_URL"
|
||||
# There are 11 mirrors
|
||||
DL_MIRROR_RANDOM=`expr $(date +%s) % 11 + 1`
|
||||
DL_URL="ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz"
|
||||
|
||||
if ! ( echo "$DL_SHA512 $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then
|
||||
echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one"
|
||||
exit 1
|
||||
# Make sure there are no partial downloads from the previous loop iterations
|
||||
rm -rf ./*
|
||||
|
||||
wget --tries 1 "$DL_URL" && break
|
||||
done
|
||||
|
||||
if ! ( echo "$IMAGE_SHA512 $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then
|
||||
echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
unxz "$IMAGE_NAME.xz"
|
||||
|
||||
# With this we don't have to guess how long a command will run for and sleeping
|
||||
# for that amount of time, risking either under sleeping or over sleeping, instead
|
||||
# we will sleep exactly until the command is finished by printing out a unique
|
||||
# string after the command is executed and then checking if it was printed.
|
||||
execute_shell_and_wait()
|
||||
{
|
||||
# $RANDOM is a bash built-in, so we try to avoid name collision here by using ugly RANDOM_STR name
|
||||
RANDOM_STR=$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c16)
|
||||
send_keys "$1;echo $RANDOM_STR
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y qemu screen expect
|
||||
|
||||
"
|
||||
# \[1B is a control escape sequence for a new line in the terminal.
|
||||
# We want to wait for <new-line>$RANDOM_STR instead of just $RANDOM_STR because
|
||||
# $RANDOM_STR we have inputted with send_keys above would appear in the screenlog.0
|
||||
# file and we don't want to match our input, we want to match the echo's output.
|
||||
# The .\? optionally matches any character. Sometimes it happens that there is some
|
||||
# random character inserved between the new line control escape sequence and $RANDOM_STR.
|
||||
wait_for "\[1B.\?$RANDOM_STR"
|
||||
}
|
||||
# The downloaded image has little free disk space
|
||||
qemu-img resize -f raw "$IMAGE_NAME" +5G
|
||||
|
||||
NPROC=$NPROC SSH_PORT=$SSH_PORT IMAGE_NAME="$IMAGE_NAME" screen "$OLD_PWD/.travis/cmake-freebsd-stage1.expect"
|
||||
|
||||
start_vm
|
||||
|
||||
# Login as root user
|
||||
send_keys 'root
|
||||
|
||||
'
|
||||
|
||||
# Wait for the prompt
|
||||
wait_for "root@.*:~"
|
||||
|
||||
# Configure network, ssh and start changing password
|
||||
execute_shell_and_wait 'echo "ifconfig_em0=DHCP" >> /etc/rc.conf'
|
||||
execute_shell_and_wait 'echo "Port 22" >> /etc/ssh/sshd_config'
|
||||
execute_shell_and_wait 'echo "PermitRootLogin yes" >> /etc/ssh/sshd_config'
|
||||
execute_shell_and_wait 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config'
|
||||
execute_shell_and_wait 'echo "PermitEmptyPasswords yes" >> /etc/ssh/sshd_config'
|
||||
execute_shell_and_wait 'echo "sshd_enable=YES" >> /etc/rc.conf'
|
||||
send_keys 'sh /etc/rc.d/netif restart && sh /etc/rc.d/sshd start && passwd
|
||||
'
|
||||
|
||||
# Wait for the password prompt
|
||||
wait_for "Changing local password for root"
|
||||
|
||||
# Reset password to empty for the passwordless ssh to work
|
||||
send_keys '
|
||||
'
|
||||
wait_for "New Password"
|
||||
send_keys '
|
||||
'
|
||||
|
||||
# Update system
|
||||
RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
|
||||
# It fails if there is nothing to install, so we make it always succeed with true
|
||||
|
@ -117,7 +89,6 @@ travis_install() {
|
|||
gmake \
|
||||
cmake \
|
||||
pkgconf \
|
||||
opencv \
|
||||
portaudio \
|
||||
libsndfile \
|
||||
texinfo \
|
||||
|
@ -130,60 +101,11 @@ travis_install() {
|
|||
# Create cache
|
||||
tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
|
||||
rm "$IMAGE_NAME"
|
||||
rm screenlog.0
|
||||
|
||||
cp "$OLD_PWD/GIT_TAGS" .
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh" > cmake-freebsd-stage1-all.sha256
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> cmake-freebsd-stage1-all.sha256
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1" >> cmake-freebsd-stage1-all.sha256
|
||||
|
||||
ls -lh
|
||||
fi
|
||||
|
||||
# === Update the image on new version (tag) of toxcore ===
|
||||
|
||||
if ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS" ; then
|
||||
# Extract the cached image
|
||||
tar -Sxzvf "$IMAGE_NAME.tgz"
|
||||
|
||||
start_vm
|
||||
|
||||
# Log in.
|
||||
# Although qemu prints "login:" and "Password:" lines, they are not written into the screen's log
|
||||
# file for some reason (perhaps not enough text to flush the buffer to a file?), so we can't use
|
||||
# wait_for on them, we just use sleep to add hopefully enough delay.
|
||||
sleep 5
|
||||
# "login:"
|
||||
send_keys 'root
|
||||
|
||||
'
|
||||
sleep 5
|
||||
# "Password:"
|
||||
send_keys '
|
||||
|
||||
'
|
||||
# Wait for the prompt
|
||||
wait_for "root@.* ~"
|
||||
|
||||
# Update system
|
||||
RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
|
||||
RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true
|
||||
|
||||
# Update packages
|
||||
RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg upgrade
|
||||
|
||||
# === Cache the updated VM image ===
|
||||
|
||||
stop_vm
|
||||
|
||||
# Create/Update cache
|
||||
rm "$IMAGE_NAME.tgz"
|
||||
tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
|
||||
rm "$IMAGE_NAME"
|
||||
rm screenlog.0
|
||||
|
||||
cp "$OLD_PWD/GIT_TAGS" .
|
||||
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh" > cmake-freebsd-stage1-all.sha256
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1" >> cmake-freebsd-stage1-all.sha256
|
||||
sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1.expect" >> cmake-freebsd-stage1-all.sha256
|
||||
ls -lh
|
||||
fi
|
||||
|
||||
|
|
41
.travis/cmake-freebsd-stage1.expect
Executable file
41
.travis/cmake-freebsd-stage1.expect
Executable file
|
@ -0,0 +1,41 @@
|
|||
#!/usr/bin/expect -f
|
||||
|
||||
set timeout -1
|
||||
|
||||
# Note: doesn't work if -nographic is used instead of -curses
|
||||
spawn qemu-system-x86_64 -curses -m 2048 -smp $env(NPROC) -net user,hostfwd=tcp::$env(SSH_PORT)-:22 -net nic "$env(IMAGE_NAME)"
|
||||
|
||||
# Skip the boot menu
|
||||
expect "to boot or any other key to stop"
|
||||
send -- "\r"
|
||||
|
||||
expect "login: "
|
||||
send -- "root\r"
|
||||
|
||||
# Setup DHCP networking and paswordless ssh
|
||||
expect "root@freebsd:~ # "
|
||||
send -- "echo \"ifconfig_em0=DHCP\" >> /etc/rc.conf\r"
|
||||
expect "root@freebsd:~ # "
|
||||
send -- "echo \"Port 22\" >> /etc/ssh/sshd_config\r"
|
||||
expect "root@freebsd:~ # "
|
||||
send -- "echo \"PermitRootLogin yes\" >> /etc/ssh/sshd_config\r"
|
||||
expect "root@freebsd:~ # "
|
||||
send -- "echo \"PasswordAuthentication yes\" >> /etc/ssh/sshd_config\r"
|
||||
expect "root@freebsd:~ # "
|
||||
send -- "echo \"PermitEmptyPasswords yes\" >> /etc/ssh/sshd_config\r"
|
||||
expect "root@freebsd:~ # "
|
||||
send -- "echo \"sshd_enable=YES\" >> /etc/rc.conf\r"
|
||||
expect "root@freebsd:~ # "
|
||||
|
||||
# Set the empty password
|
||||
send -- "passwd\r"
|
||||
expect "New Password:"
|
||||
send -- "\r"
|
||||
expect "Retype New Password:"
|
||||
send -- "\r"
|
||||
expect "root@freebsd:~ # "
|
||||
|
||||
# Done
|
||||
send -- "poweroff\r"
|
||||
wait
|
||||
exit 0
|
|
@ -1,5 +1,7 @@
|
|||
#!/bin/sh
|
||||
|
||||
# Toxcore building
|
||||
|
||||
ACTION="$1"
|
||||
|
||||
set -eux
|
||||
|
@ -7,7 +9,8 @@ set -eux
|
|||
. .travis/cmake-freebsd-env.sh
|
||||
|
||||
travis_install() {
|
||||
. .travis/cmake-freebsd-install.sh
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y qemu screen
|
||||
|
||||
OLD_PWD="$PWD"
|
||||
|
||||
|
@ -29,10 +32,6 @@ travis_install() {
|
|||
|
||||
start_vm
|
||||
|
||||
# Display FreeBSD kernel info and last login
|
||||
RUN uname -a
|
||||
RUN last
|
||||
|
||||
cd "$OLD_PWD"
|
||||
|
||||
# Copy over toxcore code from Travis to qemu
|
||||
|
@ -59,14 +58,15 @@ travis_script() {
|
|||
-DMIN_LOGGER_LEVEL=TRACE \
|
||||
-DMUST_BUILD_TOXAV=ON \
|
||||
-DSTRICT_ABI=ON \
|
||||
-DTEST_TIMEOUT_SECONDS=120 \
|
||||
-DTEST_TIMEOUT_SECONDS=300 \
|
||||
-DUSE_IPV6=OFF \
|
||||
-DAUTOTEST=ON'
|
||||
|
||||
# We created the VM with the same number of cores as the host, so the host-ran `nproc` here is fine
|
||||
RUN 'gmake "-j$NPROC" -k install -C_build'
|
||||
RUN 'gmake "-j$NPROC" test ARGS="-j50" -C_build || \
|
||||
gmake "-j$NPROC" -C_build test ARGS="-j50 --rerun-failed" CTEST_OUTPUT_ON_FAILURE=1 || \
|
||||
gmake "-j1" -C_build test ARGS="-j1 --rerun-failed" || \
|
||||
gmake "-j1" -C_build test ARGS="-j1 --rerun-failed" CTEST_OUTPUT_ON_FAILURE=1 || \
|
||||
true'
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue
Block a user