cirrus: Generate jobs dynamically

Instead of having static job definitions for FreeBSD and macOS,
use a generic template for both and fill in the details that are
actually different, such as the list of packages to install, in
the GitLab CI job, right before calling cirrus-run.

The target-specific information are provided by lcitool, so that
keeping them up to date is just a matter of running the refresh
script when necessary.

Signed-off-by: Andrea Bolognani <abologna@redhat.com>
Reviewed-by: Erik Skultety <eskultet@redhat.com>

.gitlab-ci.yml

@@ -58,11 +58,35 @@ stages:
 # Jobs that we delegate to Cirrus CI because they require an operating
 # system other than Linux. These jobs will only run if the required
 # setup has been performed on the GitLab account (see ci/README.rst).
+#
+# The Cirrus CI configuration is generated by replacing target-specific
+# variables in a generic template: some of these variables are provided
+# when the GitLab CI job is defined, others are taken from a shell
+# snippet generated using lcitool.
+#
+# Note that the $PATH environment variable has to be treated with
+# special care, because we can't just override it at the GitLab CI job
+# definition level or we risk breaking it completely.
 .cirrus_build_job_template: &cirrus_build_job_definition
   stage: builds
   image: registry.gitlab.com/libvirt/libvirt-ci/cirrus-run:master
   script:
-    - cirrus-run ci/cirrus/$NAME.yml.j2
+    - source ci/cirrus/libvirt-$NAME.vars
+    - sed -e "s|[@]CI_REPOSITORY_URL@|$CI_REPOSITORY_URL|g"
+          -e "s|[@]CI_COMMIT_REF_NAME@|$CI_COMMIT_REF_NAME|g"
+          -e "s|[@]CI_COMMIT_SHA@|$CI_COMMIT_SHA|g"
+          -e "s|[@]CIRRUS_VM_INSTANCE_TYPE@|$CIRRUS_VM_INSTANCE_TYPE|g"
+          -e "s|[@]CIRRUS_VM_IMAGE_SELECTOR@|$CIRRUS_VM_IMAGE_SELECTOR|g"
+          -e "s|[@]CIRRUS_VM_IMAGE_NAME@|$CIRRUS_VM_IMAGE_NAME|g"
+          -e "s|[@]INSTALL_COMMAND@|$INSTALL_COMMAND|g"
+          -e "s|[@]PATH@|$PATH_EXTRA${PATH_EXTRA:+:}\$PATH|g"
+          -e "s|[@]PKG_CONFIG_PATH@|$PKG_CONFIG_PATH|g"
+          -e "s|[@]PKGS@|$PKGS|g"
+          -e "s|[@]MAKE@|$MAKE|g"
+          -e "s|[@]PYTHON@|$PYTHON|g"
+      <ci/cirrus/build.yml >ci/cirrus/$NAME.yml
+    - cat ci/cirrus/$NAME.yml
+    - cirrus-run ci/cirrus/$NAME.yml
   only:
     variables:
       - $CIRRUS_GITHUB_REPO
@@ -351,11 +375,21 @@ x64-freebsd-12-build:
   <<: *cirrus_build_job_definition
   variables:
     NAME: freebsd-12
+    CIRRUS_VM_INSTANCE_TYPE: freebsd_instance
+    CIRRUS_VM_IMAGE_SELECTOR: image_family
+    CIRRUS_VM_IMAGE_NAME: freebsd-12-1
+    INSTALL_COMMAND: pkg install -y
 
 x64-macos-1015-build:
   <<: *cirrus_build_job_definition
   variables:
     NAME: macos-1015
+    CIRRUS_VM_INSTANCE_TYPE: osx_instance
+    CIRRUS_VM_IMAGE_SELECTOR: image
+    CIRRUS_VM_IMAGE_NAME: catalina-base
+    INSTALL_COMMAND: brew install
+    PATH_EXTRA: /usr/local/opt/ccache/libexec:/usr/local/opt/gettext/bin:/usr/local/opt/libpcap/bin:/usr/local/opt/libxslt/bin:/usr/local/opt/rpcgen/bin
+    PKG_CONFIG_PATH: /usr/local/opt/curl/lib/pkgconfig:/usr/local/opt/libpcap/lib/pkgconfig:/usr/local/opt/libxml2/lib/pkgconfig:/usr/local/opt/ncurses/lib/pkgconfig:/usr/local/opt/readline/lib/pkgconfig
 
 
 # Cross compiled build jobs

ci/cirrus/build.yml

@@ -0,0 +1,26 @@
+@CIRRUS_VM_INSTANCE_TYPE@:
+  @CIRRUS_VM_IMAGE_SELECTOR@: @CIRRUS_VM_IMAGE_NAME@
+
+env:
+  CI_REPOSITORY_URL: "@CI_REPOSITORY_URL@"
+  CI_COMMIT_REF_NAME: "@CI_COMMIT_REF_NAME@"
+  CI_COMMIT_SHA: "@CI_COMMIT_SHA@"
+  PATH: "@PATH@"
+  PKG_CONFIG_PATH: "@PKG_CONFIG_PATH@"
+  PYTHON: "@PYTHON@"
+  MAKE: "@MAKE@"
+
+build_task:
+  install_script:
+    - @INSTALL_COMMAND@ @PKGS@
+  clone_script:
+    - git clone --depth 100 "$CI_REPOSITORY_URL" .
+    - git fetch origin "$CI_COMMIT_REF_NAME"
+    - git reset --hard "$CI_COMMIT_SHA"
+  build_script:
+    - mkdir build
+    - cd build
+    - ../autogen.sh --prefix=$(pwd)/install-root
+    - $MAKE -j3
+    - $MAKE -j3 install
+    - $MAKE -j3 dist

ci/cirrus/freebsd-12.yml.j2

@@ -1,73 +0,0 @@
-freebsd_instance:
-  image_family: freebsd-12-1
-
-env:
-  CI_REPOSITORY_URL: {{ CI_REPOSITORY_URL }}
-  CI_COMMIT_REF_NAME: {{ CI_COMMIT_REF_NAME }}
-  CI_COMMIT_SHA: {{ CI_COMMIT_SHA }}
-
-freebsd_12_task:
-  install_script:
-    - pkg install -y
-          augeas
-          autoconf
-          automake
-          avahi
-          bash
-          bash-completion
-          ca_root_nss
-          ccache
-          chrony
-          cppi
-          curl
-          cyrus-sasl
-          dbus
-          diskscrub
-          dnsmasq
-          fusefs-libs
-          gdb
-          gettext
-          gettext-tools
-          git
-          glib
-          gmake
-          gnutls
-          hal
-          libpcap
-          libpciaccess
-          libssh
-          libssh2
-          libtool
-          libxml2
-          libxslt
-          lsof
-          meson
-          ncurses
-          ninja
-          patch
-          perl5
-          pkgconf
-          polkit
-          py37-docutils
-          py37-flake8
-          py37-setuptools
-          py37-wheel
-          python3
-          qemu-utils
-          radvd
-          readline
-          screen
-          sudo
-          vim
-          yajl
-  clone_script:
-    - git clone --depth 100 "$CI_REPOSITORY_URL" .
-    - git fetch origin "$CI_COMMIT_REF_NAME"
-    - git reset --hard "$CI_COMMIT_SHA"
-  build_script:
-    - mkdir build
-    - cd build
-    - ../autogen.sh --prefix=$(pwd)/install-root
-    - gmake -j3
-    - gmake -j3 install
-    - gmake -j3 dist

ci/cirrus/libvirt-freebsd-12.vars

@@ -0,0 +1,7 @@
+PACKAGING_COMMAND='pkg'
+CC='/usr/bin/clang'
+CCACHE='/usr/local/bin/ccache'
+MAKE='/usr/local/bin/gmake'
+NINJA='/usr/local/bin/ninja'
+PYTHON='/usr/local/bin/python3'
+PKGS='augeas autoconf automake avahi bash bash-completion ca_root_nss ccache chrony cppi curl cyrus-sasl dbus diskscrub dnsmasq fusefs-libs gdb gettext gettext-tools git glib gmake gnutls hal libpcap libpciaccess libssh libssh2 libtool libxml2 libxslt lsof meson ncurses ninja p5-App-cpanminus patch perl5 pkgconf polkit py37-docutils py37-flake8 py37-pip py37-setuptools py37-wheel python3 qemu-utils radvd readline screen sudo vim yajl'

ci/cirrus/libvirt-macos-1015.vars

@@ -0,0 +1,7 @@
+PACKAGING_COMMAND='brew'
+CC='/usr/bin/clang'
+CCACHE='/usr/local/bin/ccache'
+MAKE='/usr/local/bin/gmake'
+NINJA='/usr/local/bin/ninja'
+PYTHON='/usr/local/bin/python3'
+PKGS='augeas autoconf automake bash bash-completion ccache cpanminus cppi curl dbus dnsmasq docutils flake8 gdb gettext git glib gnutls gpatch libiscsi libpcap libssh libssh2 libtool libxml2 libxslt lsof make meson ncurses ninja perl pkg-config python3 qemu readline rpcgen screen scrub vim xz yajl'

ci/cirrus/macos-1015.yml.j2

@@ -1,38 +0,0 @@
-osx_instance:
-  image: catalina-base
-
-env:
-  CI_REPOSITORY_URL: {{ CI_REPOSITORY_URL }}
-  CI_COMMIT_REF_NAME: {{ CI_COMMIT_REF_NAME }}
-  CI_COMMIT_SHA: {{ CI_COMMIT_SHA }}
-  PATH: /usr/local/opt/gettext/bin:/usr/local/opt/ccache/libexec:/usr/local/opt/rpcgen/bin:$PATH
-  PKG_CONFIG_PATH: /usr/local/opt/libxml2/lib/pkgconfig
-
-macos_1015_task:
-  install_script:
-    - brew install
-           autoconf
-           automake
-           ccache
-           docutils
-           glib
-           gnutls
-           libtool
-           libxml2
-           make
-           pkg-config
-           python
-           rpcgen
-           xz
-           yajl
-  clone_script:
-    - git clone --depth 100 "$CI_REPOSITORY_URL" .
-    - git fetch origin "$CI_COMMIT_REF_NAME"
-    - git reset --hard "$CI_COMMIT_SHA"
-  build_script:
-    - mkdir build
-    - cd build
-    - ../autogen.sh --prefix=$(pwd)/install-root
-    - gmake -j3
-    - gmake -j3 install
-    - gmake -j3 dist

ci/cirrus/refresh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+if test -z "$1"
+then
+    echo "syntax: $0 PATH-TO-LCITOOL"
+    exit 1
+fi
+
+LCITOOL=$1
+
+if ! test -x "$LCITOOL"
+then
+    echo "$LCITOOL is not executable"
+    exit 1
+fi
+
+HOSTS=$($LCITOOL hosts | grep -E 'freebsd-12|macos')
+
+for host in $HOSTS
+do
+    $LCITOOL variables "$host" libvirt >"$host.vars"
+done