Simplify Travis-CI FreeBSD build

4 files changed, 323 insertions, 11 deletions

diff --git a/.travis/cmake-freebsd-env.sh b/.travis/cmake-freebsd-env.sh
new file mode 100644
index 00000000..8ae7e738
--- /dev/null
+++ b/.travis/cmake-freebsd-env.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+NPROC=`nproc`
+
+SCREEN_SESSION=freebsd
+SSH_PORT=10022
+
+RUN() {
+  ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@"
+}
diff --git a/.travis/cmake-freebsd-install.sh b/.travis/cmake-freebsd-install.sh
new file mode 100755
index 00000000..59a78965
--- /dev/null
+++ b/.travis/cmake-freebsd-install.sh
@@ -0,0 +1,89 @@
+#!/bin/sh
+
+# Travis doesn't provide FreeBSD machines, so we just take a Linux one and run
+# FreeBSD in qemu virtual machine. qemu is being ran in curses mode inside a
+# screen session, because screen allows to easily send input and read output.
+# The input is sent using `screen -S session-name -X stuff ...` and the output
+# is read from the screen's log file. Note that for some reason you can't send
+# long input lines on Travis (it works just fine when I do it on my machine...),
+# but that limitation is not an issue, as we don't really need to send long
+# lines of input anyway. Also, note that since we run qemu in curses mode, the
+# output contains control characters intended for a terminal emulator telling
+# how to position and color the text, so it might be a little tricky to read it
+# sometimes. The only time when this script has to send input to and read the
+# output from the screen session is during the initial setup when we setup the
+# network, install and configure the ssh server, and update the system. After
+# this initial setup, ssh is used to communicate with the FreeBSD running in the
+# VM, which is a lot friendlier way of communication. Please note that Travis
+# doesn't seem to allow KVM passthrough, so qemu has to emulate all the
+# hardware, which makes it quite slow compared to the host machine. We cache
+# the qemu image since it takes a long time to run the initial system and
+# package updates, and we do incremental system and package updates on every
+# change to the list of git tags (i.e. on every toxcore release, presumably).
+
+sudo apt-get update
+sudo apt-get install -y qemu
+
+OLD_PWD="$PWD"
+
+mkdir -p /opt/freebsd/cache
+cd /opt/freebsd/cache
+
+# Make sure to update DL_SHA512 when bumping the version
+FREEBSD_VERSION="11.2"
+IMAGE_NAME=FreeBSD-${FREEBSD_VERSION}-RELEASE-amd64.raw
+
+# Sends keys to the VM as they are
+send_keys()
+{
+  screen -S $SCREEN_SESSION -X stuff "$1"
+}
+
+# Blocks until a specific text appears on VM's screen
+wait_for()
+{
+  while ! grep -q "$1" screenlog.0
+  do
+    sleep 1
+  done
+}
+
+# Starts VM and waits until it's fully running (until a login prompt is shown)
+start_vm()
+{
+  rm -f screenlog.0
+
+  # Start emulator. 2000mb RAM should be enough, right? The build machine has over 7gb.
+  screen -L -S $SCREEN_SESSION -d -m \
+    qemu-system-x86_64 -curses -m 2000 -smp $NPROC \
+    -net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic "$IMAGE_NAME"
+
+  # Wait for the boot screen options
+  wait_for "Autoboot in"
+
+  # Select the 1st option
+  send_keys '
+'
+
+  # Wait for the system to boot and present the login prompt
+  wait_for "FreeBSD/amd64 ("
+}
+
+# Shuts VM down and waits until its process finishes
+stop_vm()
+{
+  # Turn it off
+  send_keys 'poweroff
+'
+
+  # Wait for qemu process to terminate
+  while ps aux | grep qemu | grep -vq grep
+  do
+    sleep 1
+  done
+}
+
+# Let's see what's in the cache directory
+ls -lh
+
+cd "$OLD_PWD"
diff --git a/.travis/cmake-freebsd-stage1 b/.travis/cmake-freebsd-stage1
index 3954e2af..d1aeab62 100755
--- a/.travis/cmake-freebsd-stage1
+++ b/.travis/cmake-freebsd-stage1
@@ -2,14 +2,174 @@
 
 ACTION="$1"
 
-set -eu
+set -eux
 
-. other/travis/env.sh
-. other/travis/env-freebsd.sh
+. .travis/cmake-freebsd-env.sh
 
 travis_install() {
-  . other/travis/freebsd-install
-  . other/travis/freebsd-install-stage1
+  . .travis/cmake-freebsd-install.sh
+
+  git tag -l --sort=version:refname > GIT_TAGS
+
+  OLD_PWD="$PWD"
+
+  mkdir -p /opt/freebsd/cache
+  cd /opt/freebsd/cache
+
+  # === Get the VM image, set it up and cache ===
+
+  # Create image if it's not cached or if this build script has changed
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"      > /tmp/sha
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> /tmp/sha
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"     >> /tmp/sha
+  if [ ! -f "./$IMAGE_NAME.tgz" ] || [ ! -f ./cmake-freebsd-stage1-all.sha256 ] || [ "`cat cmake-freebsd-stage1-all.sha256`" != "`cat /tmp/sha`" ]; then
+
+    rm -rf ./*
+
+    # https://download.freebsd.org/ftp/releases/VM-IMAGES/11.2-RELEASE/amd64/Latest/
+    DL_SHA512="0c3c232c7023c5036daeb5fbf68c2ddecf9703c74e317afcf19da91e83d0afcc526785571e2868894ce15cdb56b74fafa1ce9fd216469db91e021ac2ef8911e5"
+    # Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html
+    # Note that not all mirrors listed on that page are working, so we have removed them
+    # I'm so sorry, there are no arrays in sh and we are not using bash...
+    DL_MIRROR_1=1
+    DL_MIRROR_2=4
+    # There are 2 mirrors
+    DL_MIRROR_RANDOM=`expr $(date +%s) % 2 + 1`
+    DL_URL="ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz"
+
+    wget "$DL_URL"
+
+    if ! ( echo "$DL_SHA512  $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then
+        echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one"
+        exit 1
+    fi
+
+    unxz "$IMAGE_NAME.xz"
+
+    # With this we don't have to guess how long a command will run for and sleeping
+    # for that amount of time, risking either under sleeping or over sleeping, instead
+    # we will sleep exactly until the command is finished by printing out a unique
+    # string after the command is executed and then checking if it was printed.
+    execute_shell_and_wait()
+    {
+      # $RANDOM is a bash built-in, so we try to avoid name collision here by using ugly RANDOM_STR name
+      RANDOM_STR=$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c16)
+      send_keys "$1;echo $RANDOM_STR
+
+"
+      # \[1B is a control escape sequence for a new line in the terminal.
+      # We want to wait for <new-line>$RANDOM_STR instead of just $RANDOM_STR because
+      # $RANDOM_STR we have inputted with send_keys above would appear in the screenlog.0
+      # file and we don't want to match our input, we want to match the echo's output.
+      # The .\? optionally matches any character. Sometimes it happens that there is some
+      # random character inserved between the new line control escape sequence and $RANDOM_STR.
+      wait_for "\[1B.\?$RANDOM_STR"
+    }
+
+    start_vm
+
+    # Login as root user
+    send_keys 'root
+
+'
+
+    # Wait for the prompt
+    wait_for "root@.*:~"
+
+    # Configure network, ssh and start changing password
+    execute_shell_and_wait 'echo "ifconfig_em0=DHCP"          >> /etc/rc.conf'
+    execute_shell_and_wait 'echo "Port 22"                    >> /etc/ssh/sshd_config'
+    execute_shell_and_wait 'echo "PermitRootLogin yes"        >> /etc/ssh/sshd_config'
+    execute_shell_and_wait 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config'
+    execute_shell_and_wait 'echo "PermitEmptyPasswords yes"   >> /etc/ssh/sshd_config'
+    execute_shell_and_wait 'echo "sshd_enable=YES"            >> /etc/rc.conf'
+    send_keys 'sh /etc/rc.d/netif restart && sh /etc/rc.d/sshd start && passwd
+'
+
+    # Wait for the password prompt
+    wait_for "Changing local password for root"
+
+    # Reset password to empty for the passwordless ssh to work
+    send_keys '
+'
+    wait_for "New Password"
+    send_keys '
+'
+
+    # Update system
+    RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
+    # It fails if there is nothing to install, so we make it always succeed with true
+    RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true
+
+    # Update packages
+    RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg upgrade
+
+    # Install and set bash as the default shell for the root user
+    RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES pkg install bash
+    RUN chsh -s /usr/local/bin/bash root
+
+    # Install required toxcore dependencies
+    RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg install git \
+                                                    opus \
+                                                    libvpx \
+                                                    libsodium \
+                                                    gmake \
+                                                    cmake \
+                                                    pkgconf \
+                                                    opencv \
+                                                    portaudio \
+                                                    libsndfile \
+                                                    texinfo \
+                                                    autotools
+
+    # === Cache the VM image ===
+
+    stop_vm
+
+    # Create cache
+    tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
+    rm "$IMAGE_NAME"
+    rm screenlog.0
+
+    cp "$OLD_PWD/GIT_TAGS" .
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"      > cmake-freebsd-stage1-all.sha256
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> cmake-freebsd-stage1-all.sha256
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"     >> cmake-freebsd-stage1-all.sha256
+
+    ls -lh
+  fi
+
+  # === Update the image on new version (tag) of toxcore ===
+
+  if ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS" ; then
+    # Extract the cached image
+    tar -Sxzvf "$IMAGE_NAME.tgz"
+
+    start_vm
+
+    # Update system
+    RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
+    RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true
+
+    # Update packages
+    RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg upgrade
+
+    # === Cache the updated VM image ===
+
+    stop_vm
+
+    # Create/Update cache
+    rm "$IMAGE_NAME.tgz"
+    tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
+    rm "$IMAGE_NAME"
+    rm screenlog.0
+
+    cp "$OLD_PWD/GIT_TAGS" .
+
+    ls -lh
+  fi
+
+  cd "$OLD_PWD"
 }
 
 travis_script() {
diff --git a/.travis/cmake-freebsd-stage2 b/.travis/cmake-freebsd-stage2
index 57b8a0f8..92e681f0 100755
--- a/.travis/cmake-freebsd-stage2
+++ b/.travis/cmake-freebsd-stage2
@@ -2,18 +2,71 @@
 
 ACTION="$1"
 
-set -eu
+set -eux
 
-. other/travis/env.sh
-. other/travis/env-freebsd.sh
+. .travis/cmake-freebsd-env.sh
 
 travis_install() {
-  . other/travis/freebsd-install
-  . other/travis/freebsd-install-stage2
+  . .travis/cmake-freebsd-install.sh
+
+  OLD_PWD="$PWD"
+
+  mkdir -p /opt/freebsd/cache
+  cd /opt/freebsd/cache
+
+  # Extract the cached image
+  tar -Sxzvf "$IMAGE_NAME.tgz"
+
+  # Get the image we will be using out of the cached directory
+  mv "$IMAGE_NAME" ..
+  ls -lh
+
+  cd ..
+
+  ls -lh
+
+  # === Get VM ready to build the code ===
+
+  start_vm
+
+  # Display FreeBSD kernel info and last login
+  RUN uname -a
+  RUN last
+
+  cd "$OLD_PWD"
+
+  # Copy over toxcore code from Travis to qemu
+  scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -P $SSH_PORT -r ./* root@localhost:~
+
+  RUN ls -lh
 }
 
 travis_script() {
-  . other/travis/toxcore-script
+  CACHEDIR="none"
+  . ".travis/flags-$CC.sh"
+
+  add_ld_flag -Wl,-z,defs
+
+  # Make compilation error on a warning
+  add_flag -Werror
+
+  RUN 'cmake -B_build -H. \
+    -DCMAKE_C_FLAGS="$C_FLAGS" \
+    -DCMAKE_CXX_FLAGS="$CXX_FLAGS" \
+    -DCMAKE_EXE_LINKER_FLAGS="$LD_FLAGS" \
+    -DCMAKE_SHARED_LINKER_FLAGS="$LD_FLAGS" \
+    -DCMAKE_INSTALL_PREFIX:PATH="_install" \
+    -DTRACE=ON \
+    -DMUST_BUILD_TOXAV=ON \
+    -DSTRICT_ABI=ON \
+    -DTEST_TIMEOUT_SECONDS=120 \
+    -DUSE_IPV6=OFF'
+
+  # We created the VM with the same number of cores as the host, so the host-ran `nproc` here is fine
+  RUN 'gmake "-j$NPROC" -k install -C_build'
+  RUN 'gmake "-j$NPROC" test ARGS="-j50" -C_build || \
+    gmake "-j$NPROC" -C_build test ARGS="-j50 --rerun-failed" CTEST_OUTPUT_ON_FAILURE=1 || \
+    true'
 }
 
 if [ "-z" "$ACTION" ]; then