From 7013af7120d2ec358097bd4c990855febc8ee9d5 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 8 Apr 2026 17:10:08 +0200 Subject: [PATCH 1/6] Moved platform configuration to platforms.json Extracted PLATFORMS and IMAGE_VERSION from build-in-container.py into a flat platforms.json file, loaded via a cached get_config() function. Each platform now has separate image_name and image_version fields (split from image_tag), and extra_build_args can be omitted when empty. Signed-off-by: Lars Erik Wik --- build-in-container.md | 12 +++++----- build-in-container.py | 56 +++++++++++++------------------------------ platforms.json | 35 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 46 deletions(-) create mode 100644 platforms.json diff --git a/build-in-container.md b/build-in-container.md index 93af4ad60..613d81176 100644 --- a/build-in-container.md +++ b/build-in-container.md @@ -62,8 +62,8 @@ specified, defaults will: | `debian-11` | `debian:11` | | `debian-12` | `debian:12` | -Adding a new Debian/Ubuntu platform requires only a new entry in the `PLATFORMS` -dict in `build-in-container.py`. Adding a non-debian based platform (e.g., +Adding a new Debian/Ubuntu platform requires only a new entry in `platforms.json`. +Adding a non-debian based platform (e.g., RHEL/CentOS) requires a new `container/Dockerfile.rhel` plus platform entries. ## How it works @@ -112,8 +112,8 @@ hash and skips rebuilding when nothing has changed. ### Container registry -Images are hosted at `ghcr.io/cfengine` and versioned via `IMAGE_VERSION` in -`build-in-container.py`. To push a new image: +Images are hosted at `ghcr.io/cfengine` and versioned per-platform via +`image_version` in `platforms.json`. To push a new image: ```bash # Build and push a single platform @@ -140,14 +140,14 @@ provided by GitHub Actions. For this to work: - After the first push, each package defaults to private. To allow anonymous pulls, go to the package on GitHub (**your org → Packages**), open **Package settings**, and change the visibility to **Public**. This is a one-time step - per package — new tags (e.g. from bumping `IMAGE_VERSION`) inherit the + per package — new tags (e.g. from bumping `image_version`) inherit the existing visibility. ### Updating the toolchain 1. Edit `container/Dockerfile.debian` as needed 2. Test locally with `--rebuild-image` -3. Bump `IMAGE_VERSION` in `build-in-container.py` +3. Bump `image_version` in `platforms.json` 4. Commit the Dockerfile change + version bump 5. Push new images by triggering the GitHub Actions workflow diff --git a/build-in-container.py b/build-in-container.py index fe04d49ea..3bab2ff01 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -6,7 +6,9 @@ """ import argparse +import functools import hashlib +import json import logging import subprocess import sys @@ -15,40 +17,13 @@ log = logging.getLogger("build-in-container") IMAGE_REGISTRY = "ghcr.io/cfengine" -IMAGE_VERSION = "1" - -PLATFORMS = { - "ubuntu-20": { - "image_tag": f"cfengine-builder-ubuntu-20:{IMAGE_VERSION}", - "base_image": "ubuntu:20.04", - "dockerfile": "Dockerfile.debian", - "extra_build_args": {"NCURSES_PKGS": "libncurses5 libncurses5-dev"}, - }, - "ubuntu-22": { - "image_tag": f"cfengine-builder-ubuntu-22:{IMAGE_VERSION}", - "base_image": "ubuntu:22.04", - "dockerfile": "Dockerfile.debian", - "extra_build_args": {}, - }, - "ubuntu-24": { - "image_tag": f"cfengine-builder-ubuntu-24:{IMAGE_VERSION}", - "base_image": "ubuntu:24.04", - "dockerfile": "Dockerfile.debian", - "extra_build_args": {}, - }, - "debian-11": { - "image_tag": f"cfengine-builder-debian-11:{IMAGE_VERSION}", - "base_image": "debian:11", - "dockerfile": "Dockerfile.debian", - "extra_build_args": {}, - }, - "debian-12": { - "image_tag": f"cfengine-builder-debian-12:{IMAGE_VERSION}", - "base_image": "debian:12", - "dockerfile": "Dockerfile.debian", - "extra_build_args": {}, - }, -} + + +@functools.cache +def get_config(): + """Load and cache platform configuration from platforms.json.""" + config_path = Path(__file__).resolve().parent / "platforms.json" + return json.loads(config_path.read_text()) def detect_source_dir(): @@ -88,7 +63,7 @@ def image_needs_rebuild(image_tag, current_hash): def build_image(platform_name, platform_config, script_dir, rebuild=False): """Build the Docker image for the given platform.""" - image_tag = platform_config["image_tag"] + image_tag = f"{platform_config['image_name']}:{platform_config['image_version']}" dockerfile_name = platform_config["dockerfile"] dockerfile_path = script_dir / "container" / dockerfile_name current_hash = dockerfile_hash(dockerfile_path) @@ -132,7 +107,8 @@ def build_image(platform_name, platform_config, script_dir, rebuild=False): def registry_image_ref(platform_name): """Return the fully-qualified registry image reference for a platform.""" - return f"{IMAGE_REGISTRY}/{PLATFORMS[platform_name]['image_tag']}" + platform = get_config()[platform_name] + return f"{IMAGE_REGISTRY}/{platform['image_name']}:{platform['image_version']}" def pull_image(platform_name): @@ -168,7 +144,7 @@ def push_image(platform_name, local_tag): ref = registry_image_ref(platform_name) if image_exists_in_registry(platform_name): - log.error(f"Image {ref} already exists. Bump IMAGE_VERSION.") + log.error(f"Image {ref} already exists. Bump image_version in platforms.json.") sys.exit(1) log.info(f"Tagging {local_tag} as {ref}...") @@ -252,7 +228,7 @@ def parse_args(): ) parser.add_argument( "--platform", - choices=list(PLATFORMS.keys()), + choices=list(get_config().keys()), help="Target platform", ) parser.add_argument( @@ -318,7 +294,7 @@ def parse_args(): if args.list_platforms: print("Available platforms:") - for name, config in PLATFORMS.items(): + for name, config in get_config().items(): print(f" {name:15s} ({config['base_image']})") sys.exit(0) @@ -357,7 +333,7 @@ def main(): script_dir = source_dir / "buildscripts" - platform_config = PLATFORMS[args.platform] + platform_config = get_config()[args.platform] if args.push_image: image_tag = build_image( diff --git a/platforms.json b/platforms.json new file mode 100644 index 000000000..1c94e0a1d --- /dev/null +++ b/platforms.json @@ -0,0 +1,35 @@ +{ + "ubuntu-20": { + "image_name": "cfengine-builder-ubuntu-20", + "image_version": "1", + "base_image": "ubuntu:20.04", + "dockerfile": "Dockerfile.debian", + "extra_build_args": { + "NCURSES_PKGS": "libncurses5 libncurses5-dev" + } + }, + "ubuntu-22": { + "image_name": "cfengine-builder-ubuntu-22", + "image_version": "1", + "base_image": "ubuntu:22.04", + "dockerfile": "Dockerfile.debian" + }, + "ubuntu-24": { + "image_name": "cfengine-builder-ubuntu-24", + "image_version": "1", + "base_image": "ubuntu:24.04", + "dockerfile": "Dockerfile.debian" + }, + "debian-11": { + "image_name": "cfengine-builder-debian-11", + "image_version": "1", + "base_image": "debian:11", + "dockerfile": "Dockerfile.debian" + }, + "debian-12": { + "image_name": "cfengine-builder-debian-12", + "image_version": "1", + "base_image": "debian:12", + "dockerfile": "Dockerfile.debian" + } +} From bf4284d72905e7df6b5eeab30a75778212045c6c Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 8 Apr 2026 17:35:28 +0200 Subject: [PATCH 2/6] Added base image SHA256 digests to platforms.json Pinned each base image to a specific digest for reproducible builds. The digest is appended to the base_image reference when passed to docker build. Signed-off-by: Lars Erik Wik --- build-in-container.py | 2 +- platforms.json | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/build-in-container.py b/build-in-container.py index 3bab2ff01..10d46d51b 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -79,7 +79,7 @@ def build_image(platform_name, platform_config, script_dir, rebuild=False): "-f", str(dockerfile_path), "--build-arg", - f"BASE_IMAGE={platform_config['base_image']}", + f"BASE_IMAGE={platform_config['base_image']}@{platform_config['base_image_sha']}", "--label", f"dockerfile-hash={current_hash}", "-t", diff --git a/platforms.json b/platforms.json index 1c94e0a1d..bc41094b3 100644 --- a/platforms.json +++ b/platforms.json @@ -3,6 +3,7 @@ "image_name": "cfengine-builder-ubuntu-20", "image_version": "1", "base_image": "ubuntu:20.04", + "base_image_sha": "sha256:c664f8f86ed5a386b0a340d981b8f81714e21a8b9c73f658c4bea56aa179d54a", "dockerfile": "Dockerfile.debian", "extra_build_args": { "NCURSES_PKGS": "libncurses5 libncurses5-dev" @@ -12,24 +13,28 @@ "image_name": "cfengine-builder-ubuntu-22", "image_version": "1", "base_image": "ubuntu:22.04", + "base_image_sha": "sha256:4fff072216d2d3d6accc8bc09b57c33e474edd726f3f65fbadbb05647ab15fa5", "dockerfile": "Dockerfile.debian" }, "ubuntu-24": { "image_name": "cfengine-builder-ubuntu-24", "image_version": "1", "base_image": "ubuntu:24.04", + "base_image_sha": "sha256:e21f810fa78c09944446ec02048605eb3ab1e4e2e261c387ecc7456b38400d79", "dockerfile": "Dockerfile.debian" }, "debian-11": { "image_name": "cfengine-builder-debian-11", "image_version": "1", "base_image": "debian:11", + "base_image_sha": "sha256:b034993190d575d4173a405e347b7c210410bf9ac4e7a84734ae37a6cb488b04", "dockerfile": "Dockerfile.debian" }, "debian-12": { "image_name": "cfengine-builder-debian-12", "image_version": "1", "base_image": "debian:12", + "base_image_sha": "sha256:ad83e02b01f4bb0c3fa818396d8bf47c0e9f5803e98bf6cbd8f772ae9e2ec4e4", "dockerfile": "Dockerfile.debian" } } From 9d02c9052620f78d8f89945eb268e89951cc64f2 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 8 Apr 2026 18:12:47 +0200 Subject: [PATCH 3/6] Changed push_image to use UTC timestamp as image version Replaced the manual version bump check with automatic timestamp-based versioning (e.g., 20260408T153042Z) when pushing images. Removed the now-unused image_exists_in_registry function. Signed-off-by: Lars Erik Wik --- build-in-container.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/build-in-container.py b/build-in-container.py index 10d46d51b..564ae7c9f 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -6,6 +6,7 @@ """ import argparse +import datetime import functools import hashlib import json @@ -128,24 +129,11 @@ def pull_image(platform_name): return ref -def image_exists_in_registry(platform_name): - """Check if an image tag already exists in the registry.""" - ref = registry_image_ref(platform_name) - result = subprocess.run( - ["docker", "manifest", "inspect", ref], - capture_output=True, - text=True, - ) - return result.returncode == 0 - - def push_image(platform_name, local_tag): - """Tag a local image with the registry reference and push it.""" - ref = registry_image_ref(platform_name) - - if image_exists_in_registry(platform_name): - log.error(f"Image {ref} already exists. Bump image_version in platforms.json.") - sys.exit(1) + """Tag a local image with a timestamped version and push it.""" + image_name = get_config()[platform_name]["image_name"] + version = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + ref = f"{IMAGE_REGISTRY}/{image_name}:{version}" log.info(f"Tagging {local_tag} as {ref}...") result = subprocess.run(["docker", "tag", local_tag, ref]) @@ -159,6 +147,8 @@ def push_image(platform_name, local_tag): log.error("Docker push failed.") sys.exit(1) + log.info(f"Update image_version to \"{version}\" in platforms.json.") + def run_container(args, image_tag, source_dir, script_dir): """Run the build inside a Docker container.""" From 21e524a86fb0cd3f730a5f269f9b9f3fd8cf8034 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 8 Apr 2026 18:16:39 +0200 Subject: [PATCH 4/6] Added weekly cron schedule to build-base-images workflow The workflow now runs every Sunday at midnight UTC to keep base images up to date with the latest upstream packages. Signed-off-by: Lars Erik Wik --- .github/workflows/build-base-images.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-base-images.yml b/.github/workflows/build-base-images.yml index a7aeac4dd..ab105922e 100644 --- a/.github/workflows/build-base-images.yml +++ b/.github/workflows/build-base-images.yml @@ -1,7 +1,9 @@ name: Build base images on: - workflow_dispatch: + schedule: + - cron: "0 0 * * 0" # Every Sunday at midnight UTC + workflow_dispatch: # Allows manual trigger jobs: build-and-push: From 947a9a24c867a2e097314b26465c8430501141c4 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 8 Apr 2026 18:28:26 +0200 Subject: [PATCH 5/6] Added --update flag to fetch latest image versions from registry Queries the ghcr.io registry API for the latest tags and updates image_version in platforms.json. Supports --platform to update a single platform, or updates all when omitted. Signed-off-by: Lars Erik Wik --- build-in-container.md | 44 +++++++++++++++++++++++------------ build-in-container.py | 54 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 16 deletions(-) diff --git a/build-in-container.md b/build-in-container.md index 613d81176..ec821d9b5 100644 --- a/build-in-container.md +++ b/build-in-container.md @@ -38,19 +38,22 @@ specified, defaults will: | `--role` | `agent` or `hub` (not required for `--push-image`) | | `--build-type` | `DEBUG` or `RELEASE` (not required for `--push-image`) | +None of the above arguments are required for `--update`. + ### Optional arguments -| Option | Default | Description | -| ------------------ | -------------------------------- | ----------------------------------------------------------- | -| `--output-dir` | `./output` | Where to write output packages | -| `--cache-dir` | `~/.cache/cfengine/buildscripts` | Dependency cache directory | -| `--build-number` | `1` | Build number for package versioning | -| `--version` | auto | Override version string | -| `--rebuild-image` | | Force rebuild of Docker image (bypasses Docker layer cache) | -| `--push-image` | | Build image and push to registry, then exit | -| `--shell` | | Drop into a bash shell inside the container for debugging | -| `--list-platforms` | | List available platforms and exit | -| `--source-dir` | parent of `buildscripts/` | Root directory containing repos | +| Option | Default | Description | +| ------------------ | -------------------------------- | ------------------------------------------------------------------- | +| `--output-dir` | `./output` | Where to write output packages | +| `--cache-dir` | `~/.cache/cfengine/buildscripts` | Dependency cache directory | +| `--build-number` | `1` | Build number for package versioning | +| `--version` | auto | Override version string | +| `--rebuild-image` | | Force rebuild of Docker image (bypasses Docker layer cache) | +| `--push-image` | | Build image and push to registry, then exit | +| `--update` | | Fetch latest image versions from registry and update platforms.json | +| `--shell` | | Drop into a bash shell inside the container for debugging | +| `--list-platforms` | | List available platforms and exit | +| `--source-dir` | parent of `buildscripts/` | Root directory containing repos | ## Supported platforms @@ -129,7 +132,18 @@ which handles authentication automatically. #### GitHub Actions workflow The `build-base-images.yml` workflow builds and pushes images for every -supported platform. It is triggered manually via `workflow_dispatch`. +supported platform. It runs weekly (Sunday at midnight UTC) and can also be +triggered manually via `workflow_dispatch`. + +After the workflow pushes new images, update `platforms.json` to use them: + +```bash +# Update all platforms to the latest registry version +./build-in-container.py --update + +# Update a single platform +./build-in-container.py --update --platform ubuntu-22 +``` The workflow authenticates to `ghcr.io` using the automatic `GITHUB_TOKEN` provided by GitHub Actions. For this to work: @@ -147,9 +161,9 @@ provided by GitHub Actions. For this to work: 1. Edit `container/Dockerfile.debian` as needed 2. Test locally with `--rebuild-image` -3. Bump `image_version` in `platforms.json` -4. Commit the Dockerfile change + version bump -5. Push new images by triggering the GitHub Actions workflow +3. Push new images by triggering the GitHub Actions workflow +4. Run `./build-in-container.py --update` to update `platforms.json` +5. Commit the Dockerfile change + version update ## Debugging diff --git a/build-in-container.py b/build-in-container.py index 564ae7c9f..620d7a04d 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -13,6 +13,7 @@ import logging import subprocess import sys +import urllib.request from pathlib import Path log = logging.getLogger("build-in-container") @@ -150,6 +151,44 @@ def push_image(platform_name, local_tag): log.info(f"Update image_version to \"{version}\" in platforms.json.") +def latest_registry_version(image_name): + """Query ghcr.io for the latest tag of an image.""" + # Anonymous token — no credentials needed for public images + token_url = f"https://ghcr.io/token?scope=repository:cfengine/{image_name}:pull" + token = json.loads(urllib.request.urlopen(token_url).read())["token"] + + tags_url = f"https://ghcr.io/v2/cfengine/{image_name}/tags/list" + req = urllib.request.Request( + tags_url, headers={"Authorization": f"Bearer {token}"} + ) + tags = json.loads(urllib.request.urlopen(req).read()).get("tags", []) + if not tags: + return None + return sorted(tags)[-1] + + +def update_platform_versions(platform_name=None): + """Fetch latest image versions from the registry and update platforms.json.""" + config = get_config() + + platforms = [platform_name] if platform_name else list(config.keys()) + for name in platforms: + image_name = config[name]["image_name"] + latest = latest_registry_version(image_name) + if latest is None: + log.warning(f"No tags found for {image_name}, skipping.") + continue + old = config[name]["image_version"] + if old == latest: + log.info(f"{name}: already at {latest}") + else: + config[name]["image_version"] = latest + log.info(f"{name}: {old} -> {latest}") + + config_path = Path(__file__).resolve().parent / "platforms.json" + config_path.write_text(json.dumps(config, indent=2) + "\n") + + def run_container(args, image_tag, source_dir, script_dir): """Run the build inside a Docker container.""" output_dir = Path(args.output_dir).resolve() @@ -266,6 +305,11 @@ def parse_args(): action="store_true", help="Build image and push to registry, then exit", ) + parser.add_argument( + "--update", + action="store_true", + help="Fetch latest image version from registry and update platforms.json", + ) parser.add_argument( "--shell", action="store_true", @@ -288,7 +332,11 @@ def parse_args(): print(f" {name:15s} ({config['base_image']})") sys.exit(0) - # --platform is always required (except --list-platforms handled above) + if args.update: + # --platform is optional for --update; updates all if omitted + return args + + # --platform is always required (except --list-platforms/--update handled above) if not args.platform: parser.error("missing required argument --platform") @@ -315,6 +363,10 @@ def main(): format="%(message)s", ) + if args.update: + update_platform_versions(args.platform) + return + # Detect source directory if args.source_dir: source_dir = Path(args.source_dir).resolve() From a419325be7c49358fc3e906d0088a73f1fe93ad3 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Wed, 8 Apr 2026 18:39:30 +0200 Subject: [PATCH 6/6] Added workflow to update base image versions weekly Runs every Monday at midnight and creates a PR if platforms.json was updated with newer image tags. Signed-off-by: Lars Erik Wik --- .github/workflows/update-base-images.yml | 29 ++++++++++++++++++++++++ build-in-container.md | 12 +++++++--- 2 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/update-base-images.yml diff --git a/.github/workflows/update-base-images.yml b/.github/workflows/update-base-images.yml new file mode 100644 index 000000000..daed93478 --- /dev/null +++ b/.github/workflows/update-base-images.yml @@ -0,0 +1,29 @@ +name: Update base image versions + +on: + schedule: + - cron: "0 0 * * 1" # Every Monday at midnight UTC + workflow_dispatch: # Allows manual trigger + +jobs: + update: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Update platforms.json + run: ./build-in-container.py --update + + - name: Create pull request + uses: peter-evans/create-pull-request@v8 + with: + commit-message: "Updated base image versions in platforms.json" + branch: update-base-images + title: "Updated base image versions" + body: | + Automated update of `image_version` in `platforms.json` to the + latest tags from ghcr.io. diff --git a/build-in-container.md b/build-in-container.md index ec821d9b5..78687f336 100644 --- a/build-in-container.md +++ b/build-in-container.md @@ -145,6 +145,12 @@ After the workflow pushes new images, update `platforms.json` to use them: ./build-in-container.py --update --platform ubuntu-22 ``` +The `update-base-images.yml` workflow automates this step. It runs weekly +(Monday at midnight UTC) and can also be triggered manually. It calls +`./build-in-container.py --update` and opens a pull request with any +`platforms.json` changes. This workflow requires `contents: write` and +`pull-requests: write` permissions. + The workflow authenticates to `ghcr.io` using the automatic `GITHUB_TOKEN` provided by GitHub Actions. For this to work: @@ -161,9 +167,9 @@ provided by GitHub Actions. For this to work: 1. Edit `container/Dockerfile.debian` as needed 2. Test locally with `--rebuild-image` -3. Push new images by triggering the GitHub Actions workflow -4. Run `./build-in-container.py --update` to update `platforms.json` -5. Commit the Dockerfile change + version update +3. Commit and merge the Dockerfile change +4. Push new images by triggering the `build-base-images.yml` workflow +5. Trigger the `update-base-images.yml` workflow to open a PR updating `platforms.json` ## Debugging