ENT-14361, ENT-14362: Preparing build scripts for running build-in-container in Jenkins

.github/workflows/build-base-images.yml

@@ -21,10 +21,12 @@ jobs:
           - debian-11
           - debian-12
           - debian-13
+          - rhel-7
           - rhel-8
           - rhel-9
           - rhel-10
           - ubuntu-24-mingw
+          - tarballs
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6

build-in-container-inner.sh

@@ -33,23 +33,47 @@ for repo in $repos; do
     # over from previous test runs and are not needed for building.
     # Also skip node_modules/vendor for hub builds.
     # Also skip compilation results *.o, *.lo, *.la as the local copy is likely a different platform/OS than inside the container
+    # Skip revision files too: autogen only writes them when absent, so a
+    # leftover from an earlier host build would key the dependency cache to
+    # whatever commit that build saw.
+    # And skip output directories: --output-dir defaults to ./output, which lands
+    # inside buildscripts, and the collector at the end of this script would then
+    # pick an earlier build's packages up as if this build had made them.
     if [ -d "$src" ] || [ -L "$src" ]; then
         echo "Syncing $repo..."
         sudo rsync -aL --exclude='config.cache' --exclude='workdir' \
             --exclude='*.o' --exclude='*.lo' --exclude='*.la' \
             --exclude='node_modules' --exclude='vendor' \
+            --exclude='revision' \
+            --exclude='output' \
             --chown="$(id -u):$(id -g)" "$src/" "$BASEDIR/$repo/"
     else
         echo "ERROR: Required repository $repo not found" >&2
         exit 1
     fi
 done
 
+# The dependency cache is reached over sftp, so the key has to be in place
+# before install-dependencies runs. It arrives on a read-only mount owned by the
+# host user, and ssh refuses a key owned by anyone but us, hence the copy.
+if [ -f /run/secrets/sftp-cache-key ]; then
+    echo "Installing dependency cache key..."
+    install -d -m 700 "$HOME/.ssh"
+    install -m 600 /run/secrets/sftp-cache-key "$HOME/.ssh/id_rsa"
+    grep '^build-artifacts-cache' "$BASEDIR/buildscripts/ci/known_hosts" \
+        >> "$HOME/.ssh/known_hosts"
+
+    # Fail now rather than once every dependency has been built, which is when
+    # pkg-cache would first try to upload.
+    echo pwd | sftp -o BatchMode=yes -b - jenkins_sftp_cache@build-artifacts-cache.cloud.cfengine.com
+fi
+
 # Pin embedded build timestamps so two builds of the same source produce
 # identical binaries. Honored by OpenSSL, Apache httpd, Postgres, Python
 # (.pyc mtimes), dpkg-buildpackage, and rpmbuild.
 if [ -z "$SOURCE_DATE_EPOCH" ]; then
-    SOURCE_DATE_EPOCH=$(git -C "$BASEDIR/core" log -1 --format=%ct)
+    # cd rather than git -C: rhel-7 has git 1.8.3.1, which predates -C
+    SOURCE_DATE_EPOCH=$(cd "$BASEDIR/core" && git log -1 --format=%ct)
 fi
 export SOURCE_DATE_EPOCH
 echo "SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
@@ -93,21 +117,56 @@ install_mission_portal_deps() (
     find "$BASEDIR/mission-portal" "$BASEDIR/nova/api/http" -type d -name .git -path '*/vendor/*' -exec rm -rf {} +
 )
 
-# Build the masterfiles tarballs, mirroring build-scripts/bootstrap-tarballs.
-# Produces both the source tarball ("make dist") and the package tarball
-# ("make tar-package", files laid out as installed under prefix) and drops
-# them in /output alongside the platform packages.
-build_masterfiles_tarballs() (
+# Lets whoever consumes the output check that it arrived intact. Sorted in the C
+# locale so that the list itself comes out the same every time.
+write_sha256sums() (
+    cd /output
+    # shellcheck disable=SC2094
+    # > Make sure not to read and write the same file in the same pipeline.
+    # find leaves it out by name, so the list never covers itself.
+    find . -maxdepth 1 -type f ! -name sha256sums.txt -printf '%P\n' \
+        | LC_ALL=C sort | xargs -r sha256sum > sha256sums.txt
+)
+
+# Build the source tarballs. They are the same whichever platform builds them,
+# so only this image builds them, and nothing else here does. /output is
+# <output-dir>/tarballs on the host, as the packages' /output is per label.
+#
+# Each tarball's timestamps follow its own repository: Makefile.am in core and in
+# masterfiles clamps every mtime in the tarball to SOURCE_DATE_EPOCH, so taking
+# it from the last commit keeps a tarball identical until its own sources change.
+build_tarballs() (
     set -e
 
-    cd "$BASEDIR/masterfiles"
-    rm -f cfengine-masterfiles*.tar.gz
-    # Configure so the dist targets work, matching bootstrap-tarballs (no args).
-    ./configure
-    make dist        # source tarball:  cfengine-masterfiles-<version>.tar.gz
-    make tar-package # package tarball: cfengine-masterfiles-<version>.pkg.tar.gz
-    mv cfengine-masterfiles*.tar.gz /output/
-    make distclean
+    (
+        cd "$BASEDIR/core"
+        SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
+        export SOURCE_DATE_EPOCH
+        echo "core SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
+
+        rm -f cfengine-3.*.tar.gz
+        # Configure so the dist target exists, undone again below.
+        ./configure -C
+        make dist
+        mv cfengine-3.*.tar.gz /output/
+        make distclean
+    )
+
+    (
+        cd "$BASEDIR/masterfiles"
+        SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)
+        export SOURCE_DATE_EPOCH
+        echo "masterfiles SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH"
+
+        rm -f cfengine-masterfiles*.tar.gz
+        ./configure
+        make dist        # source tarball:  cfengine-masterfiles-<version>.tar.gz
+        make tar-package # package tarball: cfengine-masterfiles-<version>.pkg.tar.gz
+        mv cfengine-masterfiles*.tar.gz /output/
+        make distclean
+    )
+
+    write_sha256sums
 )
 
 # === Step runner with failure reporting ===
@@ -128,6 +187,15 @@ run_step() {
 
 # === Build steps ===
 run_step "01-autogen" "$BASEDIR/buildscripts/build-scripts/autogen"
+
+if [ "$TARBALLS" = yes ]; then
+    run_step "02-tarballs" build_tarballs
+    echo ""
+    echo "=== Build complete ==="
+    ls -lh /output/
+    exit 0
+fi
+
 run_step "02-install-dependencies" "$BASEDIR/buildscripts/build-scripts/install-dependencies"
 # Mission Portal is an Enterprise/nova-only component; its sources are only
 # synced when PROJECT=nova. Skip this step for community hubs.
@@ -137,11 +205,6 @@ fi
 run_step "04-configure" "$BASEDIR/buildscripts/build-scripts/configure"
 run_step "05-compile" "$BASEDIR/buildscripts/build-scripts/compile"
 run_step "06-package" "$BASEDIR/buildscripts/build-scripts/package"
-# Masterfiles tarballs are platform-independent and irrelevant to a Windows MSI
-# cross build, which only emits the .msi. Skip them when cross-compiling.
-if [ -z "$CROSS_TARGET" ]; then
-    run_step "07-masterfiles-tarballs" build_masterfiles_tarballs
-fi
 
 # === Copy output packages ===
 # Packages are created under $BASEDIR/<project>/ by dpkg-buildpackage / rpmbuild.
@@ -151,6 +214,8 @@ find "$BASEDIR" -maxdepth 4 \
     \( -name '*.deb' -o -name '*.rpm' -o -name '*.msi' -o -name '*.pkg.tar.gz' \) -print \
     -exec cp {} /output/ \;
 
+write_sha256sums
+
 echo ""
 echo "=== Build complete ==="
 ls -lh /output/

build-in-container.md

@@ -23,92 +23,45 @@ specified, defaults will:
   (`~/.cache/cfengine/buildscripts`).
 - Use the current working directory for output packages (`./output/`).
 
-## Usage
+### Usage
 
-```
-./build-in-container.py --platform PLATFORM --project PROJECT --role ROLE --build-type TYPE [OPTIONS]
+See:
+```bash
+$ ./build-in-container.py --help
 ```
 
-### Required arguments
-
-| Option         | Description                                             |
-| -------------- | ------------------------------------------------------- |
-| `--platform`   | Target platform (e.g. `ubuntu-22`, `debian-12`)         |
-| `--project`    | `community` or `nova` (not required for `--push-image`) |
-| `--role`       | `agent` or `hub` (not required for `--push-image`)      |
-| `--build-type` | `DEBUG` or `RELEASE` (not required for `--push-image`)  |
-
-None of the above arguments are required for `--update`.
-
-### Optional arguments
-
-| Option             | Default                          | Description                                                                        |
-| ------------------ | -------------------------------- | ---------------------------------------------------------------------------------- |
-| `--output-dir`     | `./output`                       | Where to write output packages                                                     |
-| `--cache-dir`      | `~/.cache/cfengine/buildscripts` | Dependency cache directory                                                         |
-| `--build-number`   | `1`                              | Build number for package versioning                                                |
-| `--version`        | auto                             | Override version string                                                            |
-| `--rebuild-image`  |                                  | Force rebuild of Docker image (bypasses Docker layer cache)                        |
-| `--push-image`     |                                  | Build image and push to registry, then exit                                        |
-| `--update`         |                                  | Fetch latest image versions from registry and update platforms.json                |
-| `--update-sha`     |                                  | Fetch latest base image manifest digests from Docker Hub and update platforms.json |
-| `--shell`          |                                  | Drop into a bash shell inside the container for debugging                          |
-| `--list-platforms` |                                  | List available platforms and exit                                                  |
-| `--source-dir`     | parent of `buildscripts/`        | Root directory containing repos                                                    |
-| `--arch`           | host architecture                | Override the container architecture (see [Architecture](#architecture))            |
-
 ## Supported platforms
 
-| Name        | Base image                 |
-| ----------- | -------------------------- |
-| `ubuntu-20` | `ubuntu:20.04`             |
-| `ubuntu-22` | `ubuntu:22.04`             |
-| `ubuntu-24` | `ubuntu:24.04`             |
-| `debian-11` | `debian:11`                |
-| `debian-12` | `debian:12`                |
-| `debian-13` | `debian:13`                |
-| `rhel-8`    | `rockylinux/rockylinux:8`  |
-| `rhel-9`    | `rockylinux/rockylinux:9`  |
-| `rhel-10`   | `rockylinux/rockylinux:10` |
-
-RHEL packages are built on Rocky Linux base images. The build scripts detect
-`OS=rhel` from `/etc/redhat-release` (which reports `Rocky Linux release ...`),
-so the produced `.rpm`s are ordinary Red Hat / rpm packages. AlmaLinux is _not_
-recognized by `build-scripts/detect-environment`, which is why Rocky is used.
-
-Adding a new Debian/Ubuntu platform requires a new entry in `platforms.json`
-and adding the platform name to the matrix in
+See:
+```bash
+$ ./build-in-container.py --list-platforms
+```
+
+Adding a new platform normally requires a new entry in `platforms.json` and
+adding the platform name to the matrix in
 `.github/workflows/build-base-images.yml` so the weekly job builds and
-pushes its image to `ghcr.io`. Without the matrix entry, no image is ever
-pushed and the `update-base-images.yml` workflow will fail with a 403 from
-`ghcr.io` when it queries tags for the missing repository.
+pushes its image to `ghcr.io`.
 
 The new entry in `platforms.json` needs:
 
 - `image_version`: set to `"latest"` as a placeholder. The
-  `update-base-images.yml` workflow (or `./build-in-container.py --update`
-  run locally) will replace it with the real ghcr.io tag after the first
-  image is pushed.
-- `base_image_sha`: the Docker Hub manifest digest for the `base_image`.
-  Don't copy this by hand — run `./build-in-container.py --update-sha
---platform <new-platform>` and it will fetch the current digest from
-  Docker Hub and write it into `platforms.json`.
-- `architectures` (optional): the list of docker platforms to publish, e.g.
-  `["linux/amd64", "linux/arm64"]`. Omit it to get the multi-arch default; set
-  it only to restrict a platform to specific architectures (see
-  [Architecture](#architecture)).
+  `update-base-images.yml` workflow will replace it with the real ghcr.io tag
+  after the first image is pushed.
+- `base_image_sha`: the Docker Hub manifest digest for the `base_image`. Don't
+  copy this by hand -- run
+  `./build-in-container.py --update-sha --platform <new-platform>` and it will
+  fetch the current digest from Docker Hub and write it into `platforms.json`.
 
-Adding another RHEL-family platform (a new Rocky/RHEL major version) works the
-same way: add a `platforms.json` entry with `"dockerfile": "Dockerfile.rhel"`
-and a matrix entry, then set any per-version `extra_build_args` — `CRB_REPO`
-(`powertools` on 8, `crb` on 9+), `PHP_MODULE_STREAM` (`remi-8.3` where the
-distro's default PHP is older than 8.3; RHEL 10 already ships 8.3), and
-`EXTRA_PKGS` for version-specific packages. Note that `--update-sha` also works
-for the namespaced `rockylinux/rockylinux` base images, not just official
-Docker Hub library images.
+Optionally, add the following entries:
 
-Adding an entirely different, non-RHEL/non-Debian platform family (e.g. SUSE)
-would require a new `container/Dockerfile.<family>` plus platform entries.
+- `architectures`: the list of docker platforms to publish, e.g.
+  `["linux/amd64", "linux/arm64"]`. Omit it to get the multi-arch default; set
+  it only to restrict or extend the architectures.
+- `extra_build_args`: allows you to add extra arguments through environment
+  variables.
+
+Adding an entirely different platform family (e.g. SUSE) would require a new
+`container/Dockerfile.<family>`.
 
 ## Architecture
 
@@ -127,7 +80,7 @@ The registry images are published as multi-arch manifests (`linux/amd64` and
 `linux/arm64`), so `--arch` normally just pulls the matching variant. If the
 registry does not provide the requested architecture (for example an older,
 single-arch image that predates multi-arch support), the script falls back to
-building the image locally for that architecture.
+building the image locally for that architecture only.
 
 Building a non-host architecture - whether locally or in CI - relies on
 QEMU/binfmt emulation being registered on the build host. If it isn't set up,
@@ -137,7 +90,7 @@ register it once with:
 docker run --privileged --rm tonistiigi/binfmt --install all
 ```
 
-Emulated builds are considerably slower than native ones.
+Please note that emulated builds are considerably slower than native ones.
 
 The set of architectures published for each platform defaults to `linux/amd64`
 and `linux/arm64`. A platform can override this with an `"architectures"` list
@@ -149,26 +102,25 @@ is pinned to `["linux/amd64"]`.
 
 The system has three components:
 
-1. **`build-in-container.py`** (Python) -- the orchestrator that runs on the host.
-   Parses arguments, builds the Docker image, and launches the container with
-   the correct mounts and environment variables.
+1. **`build-in-container.py`** (Python) -- the orchestrator that runs on the
+   host. Parses arguments, builds the Docker image, and launches the container
+   with the correct mounts and environment variables.
 
 2. **`build-in-container-inner.sh`** (Bash) -- runs inside the container. Copies
-   source repos from the read-only mount, then calls the existing build scripts
-   in order.
+   source repos from the read-only mount, then calls the build scripts in order.
 
-3. **`container/Dockerfile.debian`** and **`container/Dockerfile.rhel`** --
-   parameterized Dockerfiles shared across platforms of the same family via a
-   `BASE_IMAGE` build arg (plus per-platform `extra_build_args` in
-   `platforms.json`, e.g. the CRB repo name and PHP module stream for RHEL).
+3. **`container/Dockerfile.<family>`** -- parameterized Dockerfiles shared
+   across platforms of the same family via a `BASE_IMAGE` build arg (plus
+   per-platform `extra_build_args` in `platforms.json`.
 
 ### Container mounts
 
 | Host path                                | Container path                            | Mode       | Purpose                               |
 | ---------------------------------------- | ----------------------------------------- | ---------- | ------------------------------------- |
 | Source repos (parent of `buildscripts/`) | `/srv/source`                             | read-only  | Protects host repos from modification |
 | `~/.cache/cfengine/buildscripts/`        | `/home/builder/.cache/buildscripts_cache` | read-write | Dependency cache shared across builds |
-| `./output/`                              | `/output`                                 | read-write | Output packages copied here           |
+| `./output/{<label>,tarballs}/`           | `/output`                                 | read-write | Output packages copied here           |
+| `--sftp-key` (when given)                | `/run/secrets/sftp-cache-key`             | read-only  | Key for the remote dependency cache   |
 
 ### Build steps
 
@@ -181,6 +133,17 @@ The inner script runs these steps in order:
 5. **compile** -- compiles and installs to the dist tree
 6. **package** -- creates `.deb` or `.rpm` packages
 
+## Source tarballs
+
+The core and masterfiles tarballs are built in a dedicated container with:
+
+```bash
+./build-in-container.py --tarballs --build-type DEBUG
+```
+
+That builds them in the `tarballs` platform's image and writes them to
+`./output/tarballs/`.
+
 ## Docker image management
 
 By default, the script pulls a pre-built image from the container registry
@@ -239,8 +202,8 @@ specific Docker Hub manifest. To refresh them to the current digests:
 ./build-in-container.py --update-sha --platform ubuntu-22
 ```
 
-The `update-base-image-shas.yml` workflow automates this. It runs weekly
-(Monday at 01:00 UTC) and opens a pull request with any digest changes.
+The `update-base-image-shas.yml` workflow automates this. It runs weekly (Monday
+at 01:00 UTC) and opens a pull request with any digest changes.
 
 The workflow authenticates to `ghcr.io` using the automatic `GITHUB_TOKEN`
 provided by GitHub Actions. For this to work: