Almalinux 9 (#162)

Author: majst01

.github/workflows/master.yaml

@@ -96,3 +96,43 @@ jobs:
 
     - name: Upload image tarballs to GCS
       run: cd images && gsutil -m -h "Cache-Control:no-store" cp -r . gs://$GCS_BUCKET/metal-os
+
+  almalinux:
+    name: Build Almalinux based OS image
+    runs-on: self-hosted
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - uses: google-github-actions/auth@v2
+      with:
+        credentials_json: '${{ secrets.GCP_SA_KEY }}'
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v2
+
+    - name: Set up Go 1.22
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.22.x'
+        cache: false
+
+    - name: build install
+      run: make
+    - name: Prepare build environment
+      shell: bash
+      run: ./prepare.sh almalinux
+    - name: Build docker image for almalinux based workers and export tarball
+      run: |
+        DOCKER_MAKE_REGISTRY_LOGIN_USER="metalstack+ci" \
+        DOCKER_MAKE_REGISTRY_LOGIN_PASSWORD="${{ secrets.QUAY_IO_TOKEN }}" \
+        TMPDIR=/var/tmp \
+        docker-make \
+          --work-dir almalinux \
+          --file docker-make.yaml \
+          --no-cache \
+          --summary \
+          --no-lint \
+          --no-push
+    - name: Upload image tarballs to GCS
+      run: cd images && gsutil -m -h "Cache-Control:no-store" cp -r . gs://$GCS_BUCKET/metal-os

.github/workflows/pr.yaml

@@ -98,3 +98,50 @@ jobs:
 
     - name: Upload image tarballs to GCS
       run: cd images && gsutil -m -h "Cache-Control:no-store" cp -r . gs://$GCS_BUCKET/metal-os/pull_requests/
+
+  almalinux:
+    name: Build Almalinux based OS image
+    runs-on: self-hosted
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - uses: google-github-actions/auth@v2
+      with:
+        credentials_json: '${{ secrets.GCP_SA_KEY }}'
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v2
+
+    - name: Set up Go 1.22
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.22.x'
+        cache: false
+
+    - name: build install
+      run: make
+    - name: Prepare build environment
+      shell: bash
+      run: ./prepare.sh almalinux
+    - name: Build docker image for almalinux based workers and export tarball
+      run: |
+        DOCKER_MAKE_REGISTRY_LOGIN_USER="metalstack+ci" \
+        DOCKER_MAKE_REGISTRY_LOGIN_PASSWORD="${{ secrets.QUAY_IO_TOKEN }}" \
+        TMPDIR=/var/tmp \
+        docker-make \
+          --work-dir almalinux \
+          --file docker-make.yaml \
+          --no-cache \
+          --no-push \
+          --summary \
+          --no-lint \
+          --no-push
+
+    - uses: google-github-actions/setup-gcloud@v0
+      with:
+          service_account_email: ${{ secrets.GCP_SA_EMAIL }}
+          service_account_key: ${{ secrets.GCP_SA_KEY }}
+
+    - name: Upload image tarballs to GCS
+      run: cd images && gsutil -m -h "Cache-Control:no-store" cp -r . gs://$GCS_BUCKET/metal-os/pull_requests/

.github/workflows/release.yaml

@@ -33,10 +33,10 @@ jobs:
 
       - run: |
           # copy supported images (entire stable folder cannot be copied as a whole because it contains obsolete images as well)
+          gsutil -m cp -r gs://$GCS_BUCKET/metal-os/stable/almalinux/9 gs://$GCS_BUCKET/metal-os/${GITHUB_REF##*/}/almalinux/9
           gsutil -m cp -r gs://$GCS_BUCKET/metal-os/stable/debian/12 gs://$GCS_BUCKET/metal-os/${GITHUB_REF##*/}/debian/12
           gsutil -m cp -r gs://$GCS_BUCKET/metal-os/stable/debian-nvidia/12 gs://$GCS_BUCKET/metal-os/${GITHUB_REF##*/}/debian-nvidia/12
           gsutil -m cp -r gs://$GCS_BUCKET/metal-os/stable/firewall/3.0-ubuntu gs://$GCS_BUCKET/metal-os/${GITHUB_REF##*/}/firewall/3.0-ubuntu
-          gsutil -m cp -r gs://$GCS_BUCKET/metal-os/stable/ubuntu/22.04 gs://$GCS_BUCKET/metal-os/${GITHUB_REF##*/}/ubuntu/22.04
           gsutil -m cp -r gs://$GCS_BUCKET/metal-os/stable/ubuntu/24.04 gs://$GCS_BUCKET/metal-os/${GITHUB_REF##*/}/ubuntu/24.04
 
           PREFIX=metal-os/${GITHUB_REF##*/} go run ./cmd/tools/generate-table > downloads.md

IMAGE_STORE.md

@@ -11,7 +11,7 @@ The actual directory layout should look like:
 
 Where `<imagesdir>` is `/` for the master branch and `/${CI_COMMIT_REF_SLUG}/` for branches and merge requests.
 
-`<os>` is the name of the os in use, we currently only have `ubuntu` and `firewall`, where `firewall` is derived from the `ubuntu` image.
+`<os>` is the name of the os in use, some images like `firewall` are derived from another os image (in this case the `ubuntu` image).
 
 `<major.minor>` specifies the major and minor number of the OS, which is case of ubuntu "19.10", "19.10", "20.04" and so on. This version must follow the semantic versioning specification, whereas we tolerate a leading zero for the minor version which is quite common for some OSes.
 

Makefile

@@ -18,6 +18,7 @@ all: clean binary
 clean:
 	rm -f debian/context/install-go
 	rm -f centos/context/install-go
+	rm -f almalinux/context/install-go
 
 .PHONY: binary
 binary: test
@@ -31,6 +32,7 @@ binary: test
 	strip bin/$(BINARY)
 	cp bin/$(BINARY) debian/context/install-go
 	cp bin/$(BINARY) centos/context/install-go
+	cp bin/$(BINARY) almalinux/context/install-go
 
 .PHONY: test
 test:
@@ -55,3 +57,7 @@ firewall: ubuntu
 .PHONY: centos
 centos: binary
 	docker-make -nNL -w centos -f docker-make.yaml
+
+.PHONY: almalinux
+almalinux: binary
+	docker-make -nNL -w almalinux -f docker-make.yaml

README.md

@@ -1,17 +1,55 @@
 # metal-images
 
-This project builds operating system images usable for bare metal server provisioning with [metal-stack](https://metal-stack.io).
-Every OS image is build from a Dockerfile, exported to a lz4 compressed tarball, and uploaded to <https://images.metal-stack.io/>.
+This project builds operating system images that can be used for bare metal server deployments with [metal-stack](https://metal-stack.io).
+Every OS image is built from a Dockerfile, exported to a lz4 compressed tarball, and uploaded to <https://images.metal-stack.io/>.
 
-For security scanning those images are also pushed to [quay.io/metalstack](https://quay.io/user/metalstack).
+More information about the image store is available in [IMAGE_STORE.md](./IMAGE_STORE.md).
 
-Further information about the image store is available at [IMAGE_STORE.md](./IMAGE_STORE.md).
+Information about our initial architectural decisions can be found in [ARCHITECTURE.md](./ARCHITECTURE.md).
 
-Information about our initial architectural decisions may be found in [ARCHITECTURE.md](./ARCHITECTURE.md).
+## Supported Images
+
+Currently these images are supported:
+
+1. Debian 12
+1. Ubuntu 22.04
+1. Firewall 3.0-ubuntu (based on Ubuntu 22.04)
+1. Nvidia (based on Debian 12)
+
+## Unsupported Images
+
+We also publish images that we need for special purposes but do not officially support. Use at your own risk.
+
+1. CentOS 7
+1. Almalinux 9
+
+### GPU Support
+
+With the nvidia image a worker has GPU support. Please check our official documentation on [docs.metal-stack.io](https://docs.metal-stack.io/stable/overview/gpu-support/) on how to get this running on Kubernetes.
+
+## How new images become usable in a metal-stack partition
+
+Images are synchronized to partitions using a service called [metal-image-cache-sync](https://github.com/metal-stack/metal-image-cache-sync). The service mirrors the public operating system images to the management servers and transparently serves the metal images within a partition.
+
+Released images are tagged with the release date and can be accessed using the following image URL pattern:
+
+`https://images.metal-stack.io/metal-os/20230710/debian/12/img.tar.lz4`
+
+Images built from the master branch are accessible at an image URL like this:
+
+`https://images.metal-stack.io/metal-os/stable/debian/12/img.tar.lz4`
+
+For other branches, the URL pattern is as follows:
+
+`https://images.metal-stack.io/metal-os/pull_requests/${CI_COMMIT_REF_SLUG}/debian/12/img.tar.lz4`
+
+These URLs can be used to define an image at the metal-api.
 
 ## Local development and integration testing
 
-Before you can start developing changes for metal-images or even introduce new operating systems, you have to install the following tools:
+Please also refer to our documentation on docs.metal-stack.io on [Build Your Own Images](https://docs.metal-stack.io/stable/overview/os/#Building-Your-Own-Images) to check for the contract an OS image is expected to fulfill.
+
+Before you can start developing changes for metal-images or even introduce new operating systems, you should install the following tools:
 
 - **docker**: for sure
 - **kvm**: hypervisor used for integration tests
@@ -36,76 +74,45 @@ make centos
 
 # for nvidia images
 make nvidia
+
+# for almalinux images
+make almalinux
 ```
 
 For integration testing the images are started as [firecracker vm](https://firecracker-microvm.github.io/) with [weaveworks/ignite](https://github.com/weaveworks/ignite) and basic properties like interfaces to other metal-stack components, kernel parameters, internet reachability, DNS resolution etc. are checked with [goss](https://github.com/aelsabbahy/goss) in a GitHub action workflow. The integration tests are also executed when you build an image locally with.
 
-## Supported Images
-
-Currently these images are supported:
+### Debugging Image Provisioning
 
-1. Debian 12
-1. Ubuntu 22.04
-1. Firewall 3.0-ubuntu (based on Ubuntu 22.04)
-1. Nvidia (based on Debian 12)
+In some cases it may be necessary to manually figure out the commands for provisioning a machine image. To do this in a real server environment, it is possible to hook into the metal-hammer through the machine's serial console.
 
-### GPU Support
+You can interrupt the metal-hammer at any time by sending a keyboard interrupt. The metal-hammer takes a short pause before booting into the operating system kernel, which is a good time to send the interrupt.
 
-With the nvidia image a worker has GPU support. The cluster user must execute the following commands to get GPU support in Kubernetes:
+To prevent the machine from rebooting, you should immediately issue the following command:
 
 ```bash
-helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
-helm repo update
-
-kubectl create ns gpu-operator
-kubectl label --overwrite ns gpu-operator pod-security.kubernetes.io/enforce=privileged
-
-helm install --wait \
-  --generate-name \
-  --namespace gpu-operator \
-  --create-namespace \
-    nvidia/gpu-operator \
-    --set driver.enabled=false \
-    --set toolkit.enabled=false
-```
-
-After that `kubectl describe node` must show the gpu in the capacity like so:
-
-```plain
-...
-Capacity:
-  cpu:                64
-  ephemeral-storage:  100205640Ki
-  hugepages-1Gi:      0
-  hugepages-2Mi:      0
-  memory:             263802860Ki
-  nvidia.com/gpu:     1
-  pods:               510
-...
+while true; do echo "1" > /dev/watchdog && sleep 55; done &
 ```
 
-Unsupported images:
+If you want to enter the operating system through `chroot`, you need to remount some file systems that were mounted by the metal-hammer during provisioning:
 
-1. Centos 7.0
-
-## Schedule
-
-Builds from the master branch are scheduled on every sunday night at 1:10 o'clock to get fresh metal-images every week.
-
-## How new images get usable in a metal-stack partition
-
-Images are synced to partitions with a service that mirrors the public bucket and which runs on the management servers of partitions.
-
-Released Images are accessible with under this image URL, where `20230710` here is the tag of this repository.
-
-`http://images.metal-stack.io/metal-os/20230710/debian/12/img.tar.lz4`
-
-Images built from the master branch are accessible with an image URL like this:
+```bash
+# the mount points also depend on the file system layout of the machine, so please only take this as an example:
+mount /dev/sda2 /rootfs
+mount -t vfat /dev/sda1 /rootfs/boot/efi
+mount -t proc /proc /rootfs/proc
+mount -t sysfs /sys /rootfs/sys
+mount -t efivarfs /sys/firmware/efi/efivars /rootfs/sys/firmware/efi/efivars
+mount -t devtmpfs /dev /rootfs/dev
+```
 
-`http://images.metal-stack.io/metal-os/stable/debian/12/img.tar.lz4`
+Finally, you can then enter the provisioned OS image.
 
-For other branches, the URL pattern is this:
+```bash
+chroot /rootfs
 
-`http://images.metal-stack.io/metal-os/pull_requests/${CI_COMMIT_REF_SLUG}/debian/12/img.tar.lz4`
+# maybe you can mount further file systems here, which was not possible in the u-root environment of the metal-hammer
+vgchange -ay
+mount /dev/csi-lvm/varlib /var/lib/
+```
 
-Those URLs can be used to define an image at the metal-api.
+Keep in mind that you are still running on the metal-hammer kernel, which is different from the kernel that will be run in the operating system after provisioning. For further information on the metal-stack machine provisioning sequence, check out documentation on [docs.metal-stack.io](https://docs.metal-stack.io/stable/overview/architecture/#Machine-Provisioning-Sequence). The kernel used by the metal-hammer is built on our own inside the [kernel repository](https://github.com/metal-stack/kernel).