ENT-13030: various fixes for build host setup scripts during jenkins node yaml refactoring and simplification

ci/buildhost-image.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -ex
+
+platform=$1
+c=$platform
+base=$platform-patched
+built=$platform-buildhost
+if buildah images --format '{{.Name}}:{{.Tag}}' | grep $built; then
+  echo "container image $built already exists, remove if you want to rebuild"
+  continue
+fi
+
+buildah rm $c || true
+buildah --name $c from $base
+buildah copy $c . /buildscripts/ci
+
+# This section is debian specific for now. TODO: add alternatives when we add more platforms.
+buildah run $c apt install -y procps wget sudo
+buildah run $c apt remove -y cfengine-nova || true
+
+buildah run $c rm -rf /var/cfengine || true
+# touch flag file for policy to know it is in a container and avoid some aspects of configuration
+buildah run $c touch /etc/cfengine-in-container.flag
+buildah run $c /buildscripts/ci/setup-cfengine-build-host.sh | tee setup-cfengine-build-host.log
+# the above, if errored out, is not causing an error, need to fix that.
+grep -i error setup-cfengine-build-host.log && exit 1
+buildah tag $c $c-$(date +%F)
+buildah commit $c $c-buildhost
+buildah rm $c

ci/cfengine-build-host-setup.cf

@@ -30,8 +30,6 @@ bundle agent cfengine_build_host_setup
       "fakeroot";
       "flex";
       "gdb";
-      "libncurses5" comment => "added for debian-10";
-      "libncurses5-dev" comment => "added for debian-10";
       "libexpat1-dev";
       "libmodule-load-conditional-perl";
       "libpam0g-dev";
@@ -43,6 +41,15 @@ bundle agent cfengine_build_host_setup
       "rsync" comment => "added for debian-10";
       "systemd-coredump" comment => "added step to jenkins testing-pr job to query for coredumps on failures";
 
+    debian_10|ubuntu_20::
+      "libncurses5";
+      "libncurses5-dev";
+
+    (debian|ubuntu).!(debian_9|debian_10|ubuntu_16|ubuntu_18|ubuntu_20)::
+      "libncurses6";
+      "libncurses-dev";
+
+
     mingw_build_host::
       "wine:i386";
       "mingw-w64";
@@ -148,7 +155,10 @@ bundle agent cfengine_build_host_setup
       "systemssl_build_host" expression => fileexists("/etc/cfengine-systemssl-build-host.flag");
       "bootstrap_pr_host" expression => fileexists("/etc/cfengine-bootstrap-pr-host.flag");
       "containers_host" expression => fileexists("/etc/cfengine-containers-host.flag");
+      "not_in_container" expression => not(fileexists("/etc/cfengine-in-container.flag")),
+        comment => "We use an explicit flag file that we control to avoid ambiguity about whether we are in a container or not.";
     linux::
+      "have_tmp_mount" expression => returnszero("mount | grep '/tmp'", "useshell");
       "have_coredumpctl" expression => returnszero("command -v coredumpctl", "useshell");
       "missing_opt_jdk21" expression => not(fileexists("/opt/jdk-21.0.1"));
     (redhat|centos).!(redhat_6|centos_6|redhat_7|centos_7)::
@@ -173,7 +183,11 @@ bundle agent cfengine_build_host_setup
         comment => "note: centos-7 has installed instead of --installed argument, and that works on rhel-8 and rhel-9 so go with the sub-command instead of option";
 
   commands:
-    have_coredumpctl::
+    have_tmp_mount::
+      "mount -o remount,size=5G /tmp"
+        comment => "We could check if /tmp was size 5G but not worth the trouble since this remount call just sets the maximum size of the tmpfs in virtual memory.",
+        contain => in_shell;
+    have_coredumpctl.not_in_container::
       "sysctl kernel.core_pattern='|/lib/systemd/systemd-coredump %p %u %g %s %t %e'" -> { "ENT-12669" }
         comment => "Ensure that core_pattern is proper for systemd-coredump if coredumpctl is present.",
         contain => in_shell;
@@ -200,7 +214,7 @@ bundle agent cfengine_build_host_setup
     ubuntu_16::
       "have_i386_architecture" expression => strcmp(execresult("${paths.dpkg} --print-foreign-architectures", "noshell"), "i386");
     ubuntu::
-      "have_localhost_localdomain_hostname" expression => strcmp(execresult("${paths.hostname} -f", "useshell"), "localhost.localdomain");
+      "localhost_localdomain_hostname_missing" expression => not(strcmp(execresult("${paths.hostname} -f", "useshell"), "localhost.localdomain"));
     opensuse|suse|sles::
       "have_$(suse_users_and_groups)_group" expression => returnszero("grep '^$(suse_users_and_groups):' /etc/group >/dev/null", "useshell");
       "have_$(suse_users_and_groups)_user" expression => returnszero("grep '^$(suse_users_and_groups):' /etc/passwd >/dev/null", "useshell");
@@ -271,7 +285,7 @@ jenkins_builds ALL=NOPASSWD: /usr/bin/podman
     mingw_build_host.!have_i386_architecture::
       "${paths.dpkg} --add-architecture i386";
 
-    ubuntu.!have_localhost_localdomain_hostname::
+    ubuntu.not_in_container.localhost_localdomain_hostname_missing::
       "/usr/bin/hostnamectl set-hostname localhost.localdomain"
         comment => "hack for aws ubuntu hosts having unique ip-n-n-n-n hostnames, we need localhost.localdomain";
     !have_daemon_group.(suse|sles|opensuse)::

ci/cfengine-masterfiles-3.21.7-1.pkg.tar.gz.sha256

@@ -0,0 +1 @@
+b9afe61210c43880aada212eb6119506c00bcc030b17c2cbdd0bf88a1514bd82  cfengine-masterfiles-3.21.7-1.pkg.tar.gz

ci/cfengine-masterfiles-3.26.0-1.pkg.tar.gz.sha256

@@ -0,0 +1 @@
+1217b72455f09776442c5abb40b66f49629175b298c0b7baf0d5ba5ffe94e5ed  cfengine-masterfiles-3.26.0-1.pkg.tar.gz

ci/clean-buildhost-images.sh

@@ -0,0 +1 @@
+buildah images | grep buildhost | awk '{print $3}' | xargs buildah rmi

ci/distribution-patched.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -ex
+if grep CODENAME=stretch /etc/os-release; then
+  echo "deb http://archive.debian.org/debian-archive/debian stretch main" >/etc/apt/sources.list
+  echo "deb http://archive.debian.org/debian-archive/debian stretch-backports main" >>/etc/apt/sources.list
+fi
+ if grep "CentOS Linux 7" /etc/os-release; then
+   sed -i 's/mirror.centos.org/vault.centos.org/;/^mirrorlist/d;s/^#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo
+ fi
+if command -v yum; then
+  yum -e 0 -d 0 -y update
+  yum -e 0 -d 0 -y install git rsync
+fi
+if command -v apt; then
+  DEBIAN_FRONTEND=noninteractive apt -yqq update
+  DEBIAN_FRONTEND=noninteractive apt -yqq upgrade
+  DEBIAN_FRONTEND=noninteractive apt install -yqq git rsync
+fi
+if command -v zypper; then
+  source /etc/os-release
+  rpm --import https://download.opensuse.org/distribution/leap/$VERSION_ID/repo/oss/repodata/repomd.xml.key
+  zypper ar -cfp 90 https://download.opensuse.org/distribution/leap/$VERSION_ID/repo/oss/ oss
+  for repo in oss sle backports; do
+    rpm --import https://download.opensuse.org/update/leap/$VERSION_ID/$repo/repodata/repomd.xml.key
+    zypper ar -cfp 70 https://download.opensuse.org/update/leap/$VERSION_ID/$repo/ update-$repo
+  done
+  zypper -qn ref
+  zypper lr # diagnostic to see what repos are enabled
+  zypper -qn update
+  zypper -qn rm libsnmp15
+  zypper -qn install git rsync
+  groupadd jenkins || true
+  useradd -m -u 1010 -g jenkins jenkins || true
+fi

ci/images.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -ex
+
+# todo: centos7, opensuse/leap:15 (no :12), registry.access.redhat.com/ubi9 (-minimal, -init, -micro (standard))
+# run this on x86 and arm hardware to cover "all the bases" :)
+for platform in $(cat platform-container-image.list); do
+  ./patched-image.sh $platform
+  ./buildhost-image.sh $platform
+done

ci/linux-install-jdk21.sh

@@ -1,29 +1,38 @@
 #!/usr/bin/env bash
 set -e
-# install jdk "manually"
-# depending on os, might want to do something like `apt remove default-jre openjdk-*-jre-*`
-cd /opt
-baseurl=https://download.oracle.com/java/21/latest/
-version=21.0.7
-if uname -m | grep aarch64; then
-  tarball=jdk-21_linux-aarch64_bin.tar.gz
-  sha=47372cfa9244dc74ec783a1b287381502419b564fbd0b18abc8f2d6b19ac865e
-else
-  tarball=jdk-21_linux-x64_bin.tar.gz
-  sha=267b10b14b4e5fada19aca3be3b961ce4f81f1bd3ffcd070e90a5586106125eb
-fi
-wget --quiet "$baseurl$tarball"
-echo "$sha" "$tarball" | sha256sum --check -
-sudo tar xf "$tarball"
-sudo tee /etc/profile.d/jdk.sh << EOF
+
+install_jdk() {
+  # install jdk "manually"
+  # depending on os, might want to do something like `apt remove default-jre openjdk-*-jre-*`
+  cd /opt
+  baseurl=https://download.oracle.com/java/21/latest/
+  version=21.0.7
+  if uname -m | grep aarch64; then
+    tarball=jdk-21_linux-aarch64_bin.tar.gz
+    sha=47372cfa9244dc74ec783a1b287381502419b564fbd0b18abc8f2d6b19ac865e
+  else
+    tarball=jdk-21_linux-x64_bin.tar.gz
+    sha=267b10b14b4e5fada19aca3be3b961ce4f81f1bd3ffcd070e90a5586106125eb
+  fi
+  wget --quiet "$baseurl$tarball"
+  echo "$sha" "$tarball" | sha256sum --check -
+  tar xf "$tarball"
+  tee /etc/profile.d/jdk.sh << EOF
 export JAVA_HOME="/opt/jdk-$version"
 export PATH=\$PATH:\$JAVA_HOME/bin
 EOF
-sudo chown -R root:jenkins "/opt/jdk-$version"
-sudo chmod -R g+rx "/opt/jdk-$version"
-if command -v update-alternatives; then
-  sudo update-alternatives --install /usr/bin/java java "/opt/jdk-$version/bin/java" 9999
+  chown -R root:jenkins "/opt/jdk-$version"
+  chmod -R g+rx "/opt/jdk-$version"
+  if command -v update-alternatives; then
+    update-alternatives --install /usr/bin/java java "/opt/jdk-$version/bin/java" 9999
+  else
+    ln -s "/opt/jdk-$version/bin/java" /usr/bin/java
+  fi
+  cd -
+}
+
+if [ "$(whoami)" = "root" ]; then
+  install_jdk
 else
-  sudo ln -s "/opt/jdk-$version/bin/java" /usr/bin/java
+  sudo bash -c install_jdk
 fi
-cd -

ci/patched-image.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -ex
+
+platform=$1
+patched=$platform-patched
+if buildah images --format '{{.Name}}:{{.Tag}}' | grep $patched; then
+  echo "container image $patched already exists. buildah rm $patched if you want to regenerate"
+  continue
+fi
+if ! buildah ps | grep $platform; then
+  buildah --name $platform from $platform
+fi
+buildah copy $platform . /buildscripts/ci
+buildah run $platform /buildscripts/ci/distribution-patched.sh
+#buildah tag $platform $platform-$(date +%F)
+#buildah commit $platform $patched
+#buildah rm $platform

ci/platform-container-image.list

@@ -0,0 +1,5 @@
+debian:11
+debian:12
+ubuntu:20.04
+ubuntu:22.04
+ubuntu:24.04

ci/setup-cfengine-build-host.sh

@@ -1,8 +1,37 @@
 #!/usr/bin/env bash
 shopt -s expand_aliases
 
-# TODO get latest LTS dynamically
-CFE_VERSION=3.24.2
+# Use the newest CFEngine version we can
+CFE_VERSION=3.26.0
+source /etc/os-release
+if [ "$ID" = "debian" ]; then
+  if [ "$VERSION_ID" -lt "9" ]; then
+    echo "Platform $ID $VERSION_ID is too old."
+    exit 9
+  fi
+  if [ "$VERSION_ID" -lt "11" ]; then
+    CFE_VERSION=3.21.7
+  fi
+fi
+if [ "$ID" = "redhat" ] || [ "$ID" = "centos" ]; then
+  if [ "$VERSION_ID" -lt "6" ]; then
+    echo "Platform $ID $VERSION_ID is too old."
+    exit 9
+  fi
+  if [ "$VERSION_ID" -lt "7" ]; then
+    CFE_VERSION=3.24.2
+  fi
+fi
+if [ "$ID" = "ubuntu" ]; then
+  _version=$(echo $VERSION_ID | cut -d. -f1)
+  if [ "$_version" -lt "16" ]; then
+    echo "Platform $ID $VERSION_ID is too old."
+    exit 9
+  fi
+  if [ "$_version" -lt "20" ]; then
+    CFE_VERSION=3.21.7
+  fi
+fi
 
 # install needed packages and software for a build host
 set -ex
@@ -12,18 +41,25 @@ if [ "$(id -u)" != "0" ]; then
 fi
 
 ls -la /home/
+if ! id -u jenkins; then
+  useradd jenkins -p jenkins
+fi
+mkdir -p /home/jenkins
 chown -R jenkins /home/jenkins
 
+echo "checking for CFEngine install..."
 if [ -d /var/cfengine ]; then
   echo "Error: CFEngine already installed on this host. Will not proceed trying to setup build host with CFEngine temporary install."
   exit 1
 fi
 
-
 function cleanup()
 {
   set -ex
   if command -v apt 2>/dev/null; then
+    # workaround for CFE-4544, remove scriptlets call systemctl even when systemctl is-system-running returns false
+    rm /bin/systemctl
+    ln -s /bin/echo /bin/systemctl
     apt remove -y cfengine-nova || true
   elif command -v yum 2>/dev/null; then
     yum erase -y cfengine-nova || true
@@ -34,7 +70,8 @@ function cleanup()
     exit 1
   fi
   echo "Ensuring CFEngine fully uninstalled/cleaned up"
-  rm -rf /var/cfengine /opt/cfengine /var/log/CFE* /var/log/postgresql.log || true
+# keep these logs around for debugging failed setup runs
+#  rm -rf /var/cfengine /opt/cfengine /var/log/CFE* /var/log/postgresql.log || true
   if command -v pkill; then
     pkill -9 cf-agent || true
     pkill -9 cf-serverd || true