Update and fix FreeBSD setup on Travis-CI

- Bump FreeBSD to 12.1.
- Simplify stage1 logic.
- Re-try downloading the image from a different mirror if one fails.
- Use the `expect` utility instead of dealing with screen's log file.
- Re-run failed toxcore test one more time and in sequence.

5 files changed, 129 insertions, 221 deletions

diff --git a/.travis/cmake-freebsd-env.sh b/.travis/cmake-freebsd-env.sh
index 8ae7e738..7d698940 100644
--- a/.travis/cmake-freebsd-env.sh
+++ b/.travis/cmake-freebsd-env.sh
@@ -1,10 +1,44 @@
 #!/bin/sh
 
-NPROC=`nproc`
+# Common variables and functions
+
+NPROC=$(nproc)
 
 SCREEN_SESSION=freebsd
 SSH_PORT=10022
 
+FREEBSD_VERSION="12.1"
+IMAGE_NAME=FreeBSD-${FREEBSD_VERSION}-RELEASE-amd64.raw
+# https://download.freebsd.org/ftp/releases/VM-IMAGES/12.1-RELEASE/amd64/Latest/
+IMAGE_SHA512="a65da6260f5f894fc86fbe1f27dad7800906da7cffaa5077f82682ab74b6dd46c4ce87158c14b726d74ca3c6d611bea3bb336164da3f1cb990550310b110da22"
+
 RUN() {
-  ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@"
+  ssh -t -o ConnectionAttempts=120 -o ConnectTimeout=2 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@"
+}
+
+start_vm() {
+  screen -d -m qemu-system-x86_64 -curses -m 2048 -smp $NPROC -net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic "$IMAGE_NAME"
+
+  # Wait for ssh to start listening on the port
+  while ! echo "exit" | nc localhost ${SSH_PORT} | grep 'OpenSSH'; do
+    sleep 5
+  done
+
+  # Test that ssh works
+  RUN uname -a
+  RUN last
+}
+
+stop_vm()
+{
+  # Turn it off
+  # We use this contraption because for some reason `shutdown -h now` and
+  # `poweroff` result in FreeBSD not shutting down on Travis (they work on my
+  # machine though)
+  RUN "shutdown -p +5sec && sleep 30" || true
+
+  # Wait for the qemu process to terminate
+  while pgrep qemu; do
+    sleep 5
+  done
 }
diff --git a/.travis/cmake-freebsd-install.sh b/.travis/cmake-freebsd-install.sh
deleted file mode 100755
index 59a78965..00000000
--- a/.travis/cmake-freebsd-install.sh
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/bin/sh
-
-# Travis doesn't provide FreeBSD machines, so we just take a Linux one and run
-# FreeBSD in qemu virtual machine. qemu is being ran in curses mode inside a
-# screen session, because screen allows to easily send input and read output.
-# The input is sent using `screen -S session-name -X stuff ...` and the output
-# is read from the screen's log file. Note that for some reason you can't send
-# long input lines on Travis (it works just fine when I do it on my machine...),
-# but that limitation is not an issue, as we don't really need to send long
-# lines of input anyway. Also, note that since we run qemu in curses mode, the
-# output contains control characters intended for a terminal emulator telling
-# how to position and color the text, so it might be a little tricky to read it
-# sometimes. The only time when this script has to send input to and read the
-# output from the screen session is during the initial setup when we setup the
-# network, install and configure the ssh server, and update the system. After
-# this initial setup, ssh is used to communicate with the FreeBSD running in the
-# VM, which is a lot friendlier way of communication. Please note that Travis
-# doesn't seem to allow KVM passthrough, so qemu has to emulate all the
-# hardware, which makes it quite slow compared to the host machine. We cache
-# the qemu image since it takes a long time to run the initial system and
-# package updates, and we do incremental system and package updates on every
-# change to the list of git tags (i.e. on every toxcore release, presumably).
-
-sudo apt-get update
-sudo apt-get install -y qemu
-
-OLD_PWD="$PWD"
-
-mkdir -p /opt/freebsd/cache
-cd /opt/freebsd/cache
-
-# Make sure to update DL_SHA512 when bumping the version
-FREEBSD_VERSION="11.2"
-IMAGE_NAME=FreeBSD-${FREEBSD_VERSION}-RELEASE-amd64.raw
-
-# Sends keys to the VM as they are
-send_keys()
-{
-  screen -S $SCREEN_SESSION -X stuff "$1"
-}
-
-# Blocks until a specific text appears on VM's screen
-wait_for()
-{
-  while ! grep -q "$1" screenlog.0
-  do
-    sleep 1
-  done
-}
-
-# Starts VM and waits until it's fully running (until a login prompt is shown)
-start_vm()
-{
-  rm -f screenlog.0
-
-  # Start emulator. 2000mb RAM should be enough, right? The build machine has over 7gb.
-  screen -L -S $SCREEN_SESSION -d -m \
-    qemu-system-x86_64 -curses -m 2000 -smp $NPROC \
-    -net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic "$IMAGE_NAME"
-
-  # Wait for the boot screen options
-  wait_for "Autoboot in"
-
-  # Select the 1st option
-  send_keys '
-'
-
-  # Wait for the system to boot and present the login prompt
-  wait_for "FreeBSD/amd64 ("
-}
-
-# Shuts VM down and waits until its process finishes
-stop_vm()
-{
-  # Turn it off
-  send_keys 'poweroff
-'
-
-  # Wait for qemu process to terminate
-  while ps aux | grep qemu | grep -vq grep
-  do
-    sleep 1
-  done
-}
-
-# Let's see what's in the cache directory
-ls -lh
-
-cd "$OLD_PWD"
diff --git a/.travis/cmake-freebsd-stage1 b/.travis/cmake-freebsd-stage1
index a9d4c10e..6a88ed44 100755
--- a/.travis/cmake-freebsd-stage1
+++ b/.travis/cmake-freebsd-stage1
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+# Download and initial setup of the FreeBSD VM
+
 ACTION="$1"
 
 set -eux
@@ -7,8 +9,6 @@ set -eux
 . .travis/cmake-freebsd-env.sh
 
 travis_install() {
-  . .travis/cmake-freebsd-install.sh
-
   git tag -l --sort=version:refname > GIT_TAGS
 
   OLD_PWD="$PWD"
@@ -18,83 +18,55 @@ travis_install() {
 
   # === Get the VM image, set it up and cache ===
 
-  # Create image if it's not cached or if this build script has changed
-  sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"      > /tmp/sha
-  sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> /tmp/sha
-  sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"     >> /tmp/sha
-  if [ ! -f "./$IMAGE_NAME.tgz" ] || [ ! -f ./cmake-freebsd-stage1-all.sha256 ] || [ "`cat cmake-freebsd-stage1-all.sha256`" != "`cat /tmp/sha`" ]; then
-
+  # Create image if it's not cached, or if this build script has changed, or a new toxcore tag was pushed
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"         > /tmp/sha
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"        >> /tmp/sha
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1.expect" >> /tmp/sha
+  if [ ! -f "./$IMAGE_NAME.tgz" ] || [ ! -f ./cmake-freebsd-stage1-all.sha256 ] || [ "$(cat cmake-freebsd-stage1-all.sha256)" != "$(cat /tmp/sha)" ] || ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS"; then
     rm -rf ./*
 
-    # https://download.freebsd.org/ftp/releases/VM-IMAGES/11.2-RELEASE/amd64/Latest/
-    DL_SHA512="0c3c232c7023c5036daeb5fbf68c2ddecf9703c74e317afcf19da91e83d0afcc526785571e2868894ce15cdb56b74fafa1ce9fd216469db91e021ac2ef8911e5"
-    # Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html
-    # Note that not all mirrors listed on that page are working, so we have removed them
-    # I'm so sorry, there are no arrays in sh and we are not using bash...
-    DL_MIRROR_1=1
-    DL_MIRROR_2=4
-    # There are 2 mirrors
-    DL_MIRROR_RANDOM=`expr $(date +%s) % 2 + 1`
-    DL_URL="ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz"
-
-    wget "$DL_URL"
-
-    if ! ( echo "$DL_SHA512  $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then
-        echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one"
-        exit 1
+    while true; do
+      # Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html
+      # Note that not all mirrors listed on that page are working, so we have removed them
+      # There are no arrays in sh so we get a bit clever
+      DL_MIRROR_1=1
+      DL_MIRROR_2=2
+      DL_MIRROR_3=3
+      DL_MIRROR_4=4
+      DL_MIRROR_5=5
+      DL_MIRROR_6=6
+      DL_MIRROR_7=7
+      DL_MIRROR_8=10
+      DL_MIRROR_9=11
+      DL_MIRROR_10=13
+      DL_MIRROR_11=14
+
+      # There are 11 mirrors
+      DL_MIRROR_RANDOM=`expr $(date +%s) % 11 + 1`
+      DL_URL="ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz"
+
+      # Make sure there are no partial downloads from the previous loop iterations
+      rm -rf ./*
+
+      wget --tries 1 "$DL_URL" && break
+    done
+
+    if ! ( echo "$IMAGE_SHA512  $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then
+      echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one"
+      exit 1
     fi
 
     unxz "$IMAGE_NAME.xz"
 
-    # With this we don't have to guess how long a command will run for and sleeping
-    # for that amount of time, risking either under sleeping or over sleeping, instead
-    # we will sleep exactly until the command is finished by printing out a unique
-    # string after the command is executed and then checking if it was printed.
-    execute_shell_and_wait()
-    {
-      # $RANDOM is a bash built-in, so we try to avoid name collision here by using ugly RANDOM_STR name
-      RANDOM_STR=$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c16)
-      send_keys "$1;echo $RANDOM_STR
-
-"
-      # \[1B is a control escape sequence for a new line in the terminal.
-      # We want to wait for <new-line>$RANDOM_STR instead of just $RANDOM_STR because
-      # $RANDOM_STR we have inputted with send_keys above would appear in the screenlog.0
-      # file and we don't want to match our input, we want to match the echo's output.
-      # The .\? optionally matches any character. Sometimes it happens that there is some
-      # random character inserved between the new line control escape sequence and $RANDOM_STR.
-      wait_for "\[1B.\?$RANDOM_STR"
-    }
-
-    start_vm
-
-    # Login as root user
-    send_keys 'root
-
-'
-
-    # Wait for the prompt
-    wait_for "root@.*:~"
+    sudo apt-get update
+    sudo apt-get install -y qemu screen expect
 
-    # Configure network, ssh and start changing password
-    execute_shell_and_wait 'echo "ifconfig_em0=DHCP"          >> /etc/rc.conf'
-    execute_shell_and_wait 'echo "Port 22"                    >> /etc/ssh/sshd_config'
-    execute_shell_and_wait 'echo "PermitRootLogin yes"        >> /etc/ssh/sshd_config'
-    execute_shell_and_wait 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config'
-    execute_shell_and_wait 'echo "PermitEmptyPasswords yes"   >> /etc/ssh/sshd_config'
-    execute_shell_and_wait 'echo "sshd_enable=YES"            >> /etc/rc.conf'
-    send_keys 'sh /etc/rc.d/netif restart && sh /etc/rc.d/sshd start && passwd
-'
+    # The downloaded image has little free disk space
+    qemu-img resize -f raw "$IMAGE_NAME" +5G
 
-    # Wait for the password prompt
-    wait_for "Changing local password for root"
+    NPROC=$NPROC SSH_PORT=$SSH_PORT IMAGE_NAME="$IMAGE_NAME" screen "$OLD_PWD/.travis/cmake-freebsd-stage1.expect"
 
-    # Reset password to empty for the passwordless ssh to work
-    send_keys '
-'
-    wait_for "New Password"
-    send_keys '
-'
+    start_vm
 
     # Update system
     RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
@@ -117,7 +89,6 @@ travis_install() {
                                                     gmake \
                                                     cmake \
                                                     pkgconf \
-                                                    opencv \
                                                     portaudio \
                                                     libsndfile \
                                                     texinfo \
@@ -130,60 +101,11 @@ travis_install() {
     # Create cache
     tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
     rm "$IMAGE_NAME"
-    rm screenlog.0
-
-    cp "$OLD_PWD/GIT_TAGS" .
-    sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"      > cmake-freebsd-stage1-all.sha256
-    sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> cmake-freebsd-stage1-all.sha256
-    sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"     >> cmake-freebsd-stage1-all.sha256
-
-    ls -lh
-  fi
-
-  # === Update the image on new version (tag) of toxcore ===
-
-  if ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS" ; then
-    # Extract the cached image
-    tar -Sxzvf "$IMAGE_NAME.tgz"
-
-    start_vm
-
-    # Log in.
-    # Although qemu prints "login:" and "Password:" lines, they are not written into the screen's log
-    # file for some reason (perhaps not enough text to flush the buffer to a file?), so we can't use
-    # wait_for on them, we just use sleep to add hopefully enough delay.
-    sleep 5
-    # "login:"
-    send_keys 'root
-
-'
-    sleep 5
-    # "Password:"
-    send_keys '
-
-'
-    # Wait for the prompt
-    wait_for "root@.* ~"
-
-    # Update system
-    RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
-    RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true
-
-    # Update packages
-    RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg upgrade
-
-    # === Cache the updated VM image ===
-
-    stop_vm
-
-    # Create/Update cache
-    rm "$IMAGE_NAME.tgz"
-    tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
-    rm "$IMAGE_NAME"
-    rm screenlog.0
 
     cp "$OLD_PWD/GIT_TAGS" .
-
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"         > cmake-freebsd-stage1-all.sha256
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"        >> cmake-freebsd-stage1-all.sha256
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1.expect" >> cmake-freebsd-stage1-all.sha256
     ls -lh
   fi
 
diff --git a/.travis/cmake-freebsd-stage1.expect b/.travis/cmake-freebsd-stage1.expect
new file mode 100755
index 00000000..3269e742
--- /dev/null
+++ b/.travis/cmake-freebsd-stage1.expect
@@ -0,0 +1,41 @@
+#!/usr/bin/expect -f
+
+set timeout -1
+
+# Note: doesn't work if -nographic is used instead of -curses
+spawn qemu-system-x86_64 -curses -m 2048 -smp $env(NPROC) -net user,hostfwd=tcp::$env(SSH_PORT)-:22 -net nic "$env(IMAGE_NAME)"
+
+# Skip the boot menu
+expect "to boot or any other key to stop"
+send -- "\r"
+
+expect "login: "
+send -- "root\r"
+
+# Setup DHCP networking and paswordless ssh
+expect "root@freebsd:~ # "
+send -- "echo \"ifconfig_em0=DHCP\"          >> /etc/rc.conf\r"
+expect "root@freebsd:~ # "
+send -- "echo \"Port 22\"                    >> /etc/ssh/sshd_config\r"
+expect "root@freebsd:~ # "
+send -- "echo \"PermitRootLogin yes\"        >> /etc/ssh/sshd_config\r"
+expect "root@freebsd:~ # "
+send -- "echo \"PasswordAuthentication yes\" >> /etc/ssh/sshd_config\r"
+expect "root@freebsd:~ # "
+send -- "echo \"PermitEmptyPasswords yes\"   >> /etc/ssh/sshd_config\r"
+expect "root@freebsd:~ # "
+send -- "echo \"sshd_enable=YES\"            >> /etc/rc.conf\r"
+expect "root@freebsd:~ # "
+
+# Set the empty password
+send -- "passwd\r"
+expect "New Password:"
+send -- "\r"
+expect "Retype New Password:"
+send -- "\r"
+expect "root@freebsd:~ # "
+
+# Done
+send -- "poweroff\r"
+wait
+exit 0
diff --git a/.travis/cmake-freebsd-stage2 b/.travis/cmake-freebsd-stage2
index 9750531e..609cd618 100755
--- a/.travis/cmake-freebsd-stage2
+++ b/.travis/cmake-freebsd-stage2
@@ -1,5 +1,7 @@
 #!/bin/sh
 
+# Toxcore building
+
 ACTION="$1"
 
 set -eux
@@ -7,7 +9,8 @@ set -eux
 . .travis/cmake-freebsd-env.sh
 
 travis_install() {
-  . .travis/cmake-freebsd-install.sh
+  sudo apt-get update
+  sudo apt-get install -y qemu screen
 
   OLD_PWD="$PWD"
 
@@ -29,10 +32,6 @@ travis_install() {
 
   start_vm
 
-  # Display FreeBSD kernel info and last login
-  RUN uname -a
-  RUN last
-
   cd "$OLD_PWD"
 
   # Copy over toxcore code from Travis to qemu
@@ -59,14 +58,15 @@ travis_script() {
     -DMIN_LOGGER_LEVEL=TRACE \
     -DMUST_BUILD_TOXAV=ON \
     -DSTRICT_ABI=ON \
-    -DTEST_TIMEOUT_SECONDS=120 \
+    -DTEST_TIMEOUT_SECONDS=300 \
     -DUSE_IPV6=OFF \
     -DAUTOTEST=ON'
 
   # We created the VM with the same number of cores as the host, so the host-ran `nproc` here is fine
   RUN 'gmake "-j$NPROC" -k install -C_build'
   RUN 'gmake "-j$NPROC" test ARGS="-j50" -C_build || \
-    gmake "-j$NPROC" -C_build test ARGS="-j50 --rerun-failed" CTEST_OUTPUT_ON_FAILURE=1 || \
+    gmake "-j1" -C_build test ARGS="-j1 --rerun-failed" || \
+    gmake "-j1" -C_build test ARGS="-j1 --rerun-failed" CTEST_OUTPUT_ON_FAILURE=1 || \
     true'
 }