diff --git a/.github/workflows/build-base-images.yml b/.github/workflows/build-base-images.yml index a7aeac4dd..ab105922e 100644 --- a/.github/workflows/build-base-images.yml +++ b/.github/workflows/build-base-images.yml @@ -1,7 +1,9 @@ name: Build base images on: - workflow_dispatch: + schedule: + - cron: "0 0 * * 0" # Every Sunday at midnight UTC + workflow_dispatch: # Allows manual trigger jobs: build-and-push: diff --git a/.github/workflows/update-base-images.yml b/.github/workflows/update-base-images.yml new file mode 100644 index 000000000..daed93478 --- /dev/null +++ b/.github/workflows/update-base-images.yml @@ -0,0 +1,29 @@ +name: Update base image versions + +on: + schedule: + - cron: "0 0 * * 1" # Every Monday at midnight UTC + workflow_dispatch: # Allows manual trigger + +jobs: + update: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Update platforms.json + run: ./build-in-container.py --update + + - name: Create pull request + uses: peter-evans/create-pull-request@v8 + with: + commit-message: "Updated base image versions in platforms.json" + branch: update-base-images + title: "Updated base image versions" + body: | + Automated update of `image_version` in `platforms.json` to the + latest tags from ghcr.io. diff --git a/build-in-container.md b/build-in-container.md index 93af4ad60..78687f336 100644 --- a/build-in-container.md +++ b/build-in-container.md @@ -38,19 +38,22 @@ specified, defaults will: | `--role` | `agent` or `hub` (not required for `--push-image`) | | `--build-type` | `DEBUG` or `RELEASE` (not required for `--push-image`) | +None of the above arguments are required for `--update`. + ### Optional arguments -| Option | Default | Description | -| ------------------ | -------------------------------- | ----------------------------------------------------------- | -| `--output-dir` | `./output` | Where to write output packages | -| `--cache-dir` | `~/.cache/cfengine/buildscripts` | Dependency cache directory | -| `--build-number` | `1` | Build number for package versioning | -| `--version` | auto | Override version string | -| `--rebuild-image` | | Force rebuild of Docker image (bypasses Docker layer cache) | -| `--push-image` | | Build image and push to registry, then exit | -| `--shell` | | Drop into a bash shell inside the container for debugging | -| `--list-platforms` | | List available platforms and exit | -| `--source-dir` | parent of `buildscripts/` | Root directory containing repos | +| Option | Default | Description | +| ------------------ | -------------------------------- | ------------------------------------------------------------------- | +| `--output-dir` | `./output` | Where to write output packages | +| `--cache-dir` | `~/.cache/cfengine/buildscripts` | Dependency cache directory | +| `--build-number` | `1` | Build number for package versioning | +| `--version` | auto | Override version string | +| `--rebuild-image` | | Force rebuild of Docker image (bypasses Docker layer cache) | +| `--push-image` | | Build image and push to registry, then exit | +| `--update` | | Fetch latest image versions from registry and update platforms.json | +| `--shell` | | Drop into a bash shell inside the container for debugging | +| `--list-platforms` | | List available platforms and exit | +| `--source-dir` | parent of `buildscripts/` | Root directory containing repos | ## Supported platforms @@ -62,8 +65,8 @@ specified, defaults will: | `debian-11` | `debian:11` | | `debian-12` | `debian:12` | -Adding a new Debian/Ubuntu platform requires only a new entry in the `PLATFORMS` -dict in `build-in-container.py`. Adding a non-debian based platform (e.g., +Adding a new Debian/Ubuntu platform requires only a new entry in `platforms.json`. +Adding a non-debian based platform (e.g., RHEL/CentOS) requires a new `container/Dockerfile.rhel` plus platform entries. ## How it works @@ -112,8 +115,8 @@ hash and skips rebuilding when nothing has changed. ### Container registry -Images are hosted at `ghcr.io/cfengine` and versioned via `IMAGE_VERSION` in -`build-in-container.py`. To push a new image: +Images are hosted at `ghcr.io/cfengine` and versioned per-platform via +`image_version` in `platforms.json`. To push a new image: ```bash # Build and push a single platform @@ -129,7 +132,24 @@ which handles authentication automatically. #### GitHub Actions workflow The `build-base-images.yml` workflow builds and pushes images for every -supported platform. It is triggered manually via `workflow_dispatch`. +supported platform. It runs weekly (Sunday at midnight UTC) and can also be +triggered manually via `workflow_dispatch`. + +After the workflow pushes new images, update `platforms.json` to use them: + +```bash +# Update all platforms to the latest registry version +./build-in-container.py --update + +# Update a single platform +./build-in-container.py --update --platform ubuntu-22 +``` + +The `update-base-images.yml` workflow automates this step. It runs weekly +(Monday at midnight UTC) and can also be triggered manually. It calls +`./build-in-container.py --update` and opens a pull request with any +`platforms.json` changes. This workflow requires `contents: write` and +`pull-requests: write` permissions. The workflow authenticates to `ghcr.io` using the automatic `GITHUB_TOKEN` provided by GitHub Actions. For this to work: @@ -140,16 +160,16 @@ provided by GitHub Actions. For this to work: - After the first push, each package defaults to private. To allow anonymous pulls, go to the package on GitHub (**your org → Packages**), open **Package settings**, and change the visibility to **Public**. This is a one-time step - per package — new tags (e.g. from bumping `IMAGE_VERSION`) inherit the + per package — new tags (e.g. from bumping `image_version`) inherit the existing visibility. ### Updating the toolchain 1. Edit `container/Dockerfile.debian` as needed 2. Test locally with `--rebuild-image` -3. Bump `IMAGE_VERSION` in `build-in-container.py` -4. Commit the Dockerfile change + version bump -5. Push new images by triggering the GitHub Actions workflow +3. Commit and merge the Dockerfile change +4. Push new images by triggering the `build-base-images.yml` workflow +5. Trigger the `update-base-images.yml` workflow to open a PR updating `platforms.json` ## Debugging diff --git a/build-in-container.py b/build-in-container.py index fe04d49ea..620d7a04d 100755 --- a/build-in-container.py +++ b/build-in-container.py @@ -6,49 +6,26 @@ """ import argparse +import datetime +import functools import hashlib +import json import logging import subprocess import sys +import urllib.request from pathlib import Path log = logging.getLogger("build-in-container") IMAGE_REGISTRY = "ghcr.io/cfengine" -IMAGE_VERSION = "1" - -PLATFORMS = { - "ubuntu-20": { - "image_tag": f"cfengine-builder-ubuntu-20:{IMAGE_VERSION}", - "base_image": "ubuntu:20.04", - "dockerfile": "Dockerfile.debian", - "extra_build_args": {"NCURSES_PKGS": "libncurses5 libncurses5-dev"}, - }, - "ubuntu-22": { - "image_tag": f"cfengine-builder-ubuntu-22:{IMAGE_VERSION}", - "base_image": "ubuntu:22.04", - "dockerfile": "Dockerfile.debian", - "extra_build_args": {}, - }, - "ubuntu-24": { - "image_tag": f"cfengine-builder-ubuntu-24:{IMAGE_VERSION}", - "base_image": "ubuntu:24.04", - "dockerfile": "Dockerfile.debian", - "extra_build_args": {}, - }, - "debian-11": { - "image_tag": f"cfengine-builder-debian-11:{IMAGE_VERSION}", - "base_image": "debian:11", - "dockerfile": "Dockerfile.debian", - "extra_build_args": {}, - }, - "debian-12": { - "image_tag": f"cfengine-builder-debian-12:{IMAGE_VERSION}", - "base_image": "debian:12", - "dockerfile": "Dockerfile.debian", - "extra_build_args": {}, - }, -} + + +@functools.cache +def get_config(): + """Load and cache platform configuration from platforms.json.""" + config_path = Path(__file__).resolve().parent / "platforms.json" + return json.loads(config_path.read_text()) def detect_source_dir(): @@ -88,7 +65,7 @@ def image_needs_rebuild(image_tag, current_hash): def build_image(platform_name, platform_config, script_dir, rebuild=False): """Build the Docker image for the given platform.""" - image_tag = platform_config["image_tag"] + image_tag = f"{platform_config['image_name']}:{platform_config['image_version']}" dockerfile_name = platform_config["dockerfile"] dockerfile_path = script_dir / "container" / dockerfile_name current_hash = dockerfile_hash(dockerfile_path) @@ -104,7 +81,7 @@ def build_image(platform_name, platform_config, script_dir, rebuild=False): "-f", str(dockerfile_path), "--build-arg", - f"BASE_IMAGE={platform_config['base_image']}", + f"BASE_IMAGE={platform_config['base_image']}@{platform_config['base_image_sha']}", "--label", f"dockerfile-hash={current_hash}", "-t", @@ -132,7 +109,8 @@ def build_image(platform_name, platform_config, script_dir, rebuild=False): def registry_image_ref(platform_name): """Return the fully-qualified registry image reference for a platform.""" - return f"{IMAGE_REGISTRY}/{PLATFORMS[platform_name]['image_tag']}" + platform = get_config()[platform_name] + return f"{IMAGE_REGISTRY}/{platform['image_name']}:{platform['image_version']}" def pull_image(platform_name): @@ -152,24 +130,11 @@ def pull_image(platform_name): return ref -def image_exists_in_registry(platform_name): - """Check if an image tag already exists in the registry.""" - ref = registry_image_ref(platform_name) - result = subprocess.run( - ["docker", "manifest", "inspect", ref], - capture_output=True, - text=True, - ) - return result.returncode == 0 - - def push_image(platform_name, local_tag): - """Tag a local image with the registry reference and push it.""" - ref = registry_image_ref(platform_name) - - if image_exists_in_registry(platform_name): - log.error(f"Image {ref} already exists. Bump IMAGE_VERSION.") - sys.exit(1) + """Tag a local image with a timestamped version and push it.""" + image_name = get_config()[platform_name]["image_name"] + version = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + ref = f"{IMAGE_REGISTRY}/{image_name}:{version}" log.info(f"Tagging {local_tag} as {ref}...") result = subprocess.run(["docker", "tag", local_tag, ref]) @@ -183,6 +148,46 @@ def push_image(platform_name, local_tag): log.error("Docker push failed.") sys.exit(1) + log.info(f"Update image_version to \"{version}\" in platforms.json.") + + +def latest_registry_version(image_name): + """Query ghcr.io for the latest tag of an image.""" + # Anonymous token — no credentials needed for public images + token_url = f"https://ghcr.io/token?scope=repository:cfengine/{image_name}:pull" + token = json.loads(urllib.request.urlopen(token_url).read())["token"] + + tags_url = f"https://ghcr.io/v2/cfengine/{image_name}/tags/list" + req = urllib.request.Request( + tags_url, headers={"Authorization": f"Bearer {token}"} + ) + tags = json.loads(urllib.request.urlopen(req).read()).get("tags", []) + if not tags: + return None + return sorted(tags)[-1] + + +def update_platform_versions(platform_name=None): + """Fetch latest image versions from the registry and update platforms.json.""" + config = get_config() + + platforms = [platform_name] if platform_name else list(config.keys()) + for name in platforms: + image_name = config[name]["image_name"] + latest = latest_registry_version(image_name) + if latest is None: + log.warning(f"No tags found for {image_name}, skipping.") + continue + old = config[name]["image_version"] + if old == latest: + log.info(f"{name}: already at {latest}") + else: + config[name]["image_version"] = latest + log.info(f"{name}: {old} -> {latest}") + + config_path = Path(__file__).resolve().parent / "platforms.json" + config_path.write_text(json.dumps(config, indent=2) + "\n") + def run_container(args, image_tag, source_dir, script_dir): """Run the build inside a Docker container.""" @@ -252,7 +257,7 @@ def parse_args(): ) parser.add_argument( "--platform", - choices=list(PLATFORMS.keys()), + choices=list(get_config().keys()), help="Target platform", ) parser.add_argument( @@ -300,6 +305,11 @@ def parse_args(): action="store_true", help="Build image and push to registry, then exit", ) + parser.add_argument( + "--update", + action="store_true", + help="Fetch latest image version from registry and update platforms.json", + ) parser.add_argument( "--shell", action="store_true", @@ -318,11 +328,15 @@ def parse_args(): if args.list_platforms: print("Available platforms:") - for name, config in PLATFORMS.items(): + for name, config in get_config().items(): print(f" {name:15s} ({config['base_image']})") sys.exit(0) - # --platform is always required (except --list-platforms handled above) + if args.update: + # --platform is optional for --update; updates all if omitted + return args + + # --platform is always required (except --list-platforms/--update handled above) if not args.platform: parser.error("missing required argument --platform") @@ -349,6 +363,10 @@ def main(): format="%(message)s", ) + if args.update: + update_platform_versions(args.platform) + return + # Detect source directory if args.source_dir: source_dir = Path(args.source_dir).resolve() @@ -357,7 +375,7 @@ def main(): script_dir = source_dir / "buildscripts" - platform_config = PLATFORMS[args.platform] + platform_config = get_config()[args.platform] if args.push_image: image_tag = build_image( diff --git a/platforms.json b/platforms.json new file mode 100644 index 000000000..bc41094b3 --- /dev/null +++ b/platforms.json @@ -0,0 +1,40 @@ +{ + "ubuntu-20": { + "image_name": "cfengine-builder-ubuntu-20", + "image_version": "1", + "base_image": "ubuntu:20.04", + "base_image_sha": "sha256:c664f8f86ed5a386b0a340d981b8f81714e21a8b9c73f658c4bea56aa179d54a", + "dockerfile": "Dockerfile.debian", + "extra_build_args": { + "NCURSES_PKGS": "libncurses5 libncurses5-dev" + } + }, + "ubuntu-22": { + "image_name": "cfengine-builder-ubuntu-22", + "image_version": "1", + "base_image": "ubuntu:22.04", + "base_image_sha": "sha256:4fff072216d2d3d6accc8bc09b57c33e474edd726f3f65fbadbb05647ab15fa5", + "dockerfile": "Dockerfile.debian" + }, + "ubuntu-24": { + "image_name": "cfengine-builder-ubuntu-24", + "image_version": "1", + "base_image": "ubuntu:24.04", + "base_image_sha": "sha256:e21f810fa78c09944446ec02048605eb3ab1e4e2e261c387ecc7456b38400d79", + "dockerfile": "Dockerfile.debian" + }, + "debian-11": { + "image_name": "cfengine-builder-debian-11", + "image_version": "1", + "base_image": "debian:11", + "base_image_sha": "sha256:b034993190d575d4173a405e347b7c210410bf9ac4e7a84734ae37a6cb488b04", + "dockerfile": "Dockerfile.debian" + }, + "debian-12": { + "image_name": "cfengine-builder-debian-12", + "image_version": "1", + "base_image": "debian:12", + "base_image_sha": "sha256:ad83e02b01f4bb0c3fa818396d8bf47c0e9f5803e98bf6cbd8f772ae9e2ec4e4", + "dockerfile": "Dockerfile.debian" + } +}