feat(logging): Enables syslog logging for iPXE and IPA

.github/workflows/build-ironic-ipxe.yaml

@@ -11,6 +11,11 @@ on:
     tags:
       - "ironic-ipxe/v*"
   pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - closed
     paths:
       - "containers/ironic-ipxe/**"
       - ".github/workflows/build-ironic-ipxe.yaml"
@@ -21,6 +26,7 @@ concurrency:
 
 jobs:
   build:
+    if: github.event_name != 'pull_request' || github.event.action != 'closed'
     runs-on: ubuntu-latest
 
     permissions:
@@ -32,7 +38,6 @@ jobs:
         uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
 
       - name: login to ghcr.io
-        if: startsWith(github.ref, 'refs/tags/ironic-ipxe/')
         uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
         with:
           registry: ghcr.io
@@ -53,6 +58,7 @@ jobs:
             type=raw,value=${{ steps.version.outputs.tag }},enable=${{ startsWith(github.ref, 'refs/tags/ironic-ipxe/') }}
             type=raw,value=latest,enable={{is_default_branch}}
             type=raw,value=latest,enable=${{ github.event_name == 'workflow_dispatch' }}
+            type=ref,event=pr
         env:
           DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
 
@@ -64,7 +70,47 @@ jobs:
           context: containers/ironic-ipxe
           file: containers/ironic-ipxe/Dockerfile
           pull: true
-          push: ${{ startsWith(github.ref, 'refs/tags/ironic-ipxe/') }}
+          push: ${{ startsWith(github.ref, 'refs/tags/ironic-ipxe/') || github.event_name == 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           annotations: ${{ steps.meta.outputs.annotations }}
+
+  clean:
+    if: github.event_name == 'pull_request' && github.event.action == 'closed'
+    runs-on: ubuntu-latest
+
+    permissions:
+      packages: write
+
+    steps:
+      - name: clean up PR container
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9
+        with:
+          script: |
+            const container_name = `${context.repo.repo}/ironic-ipxe`;
+            const response = await github.rest.packages.getAllPackageVersionsForPackageOwnedByOrg({
+              package_type: "container",
+              package_name: container_name,
+              org: context.repo.owner,
+            });
+
+            const target_tag = `pr-${context.payload.pull_request.number}`;
+            console.log(`Looking for tag ${target_tag} for container ${container_name}`);
+
+            const versions = response.data || [];
+            const matchingVersion = versions.find(version =>
+              version.metadata.container.tags.includes(target_tag)
+            );
+
+            if (matchingVersion) {
+              console.log(`Found tag to delete "${target_tag}":`, matchingVersion.html_url);
+              await github.rest.packages.deletePackageVersionForOrg({
+                package_type: "container",
+                package_name: container_name,
+                org: context.repo.owner,
+                package_version_id: matchingVersion.id,
+              });
+              console.log("Tag deleted");
+            } else {
+              console.log(`No package version found with the tag "${target_tag}".`);
+            }

containers/ironic-ipxe/Dockerfile

@@ -16,6 +16,7 @@ RUN git checkout ${IPXE_GIT_REF}
 # Copy config files
 COPY ipxe-general.h /src/ipxe/src/config/local/general.h
 COPY ipxe-branding.h /src/ipxe/src/config/local/branding.h
+COPY ipxe-console.h /src/ipxe/src/config/local/console.h
 
 WORKDIR /src/ipxe/src
 RUN make bin-x86_64-efi/snponly.efi ${IPXE_FLAGS}

containers/ironic-ipxe/ipxe-console.h

@@ -0,0 +1,13 @@
+// overrides for
+// https://github.com/ipxe/ipxe/blob/master/src/config/console.h
+
+/*
+ * Send iPXE console output to a remote syslog server.
+ *
+ * iPXE populates its `syslog` setting from DHCP option 7 (log-server) when
+ * present, so the destination is controlled by dnsmasq rather than compiled in.
+ * This only covers the window while iPXE itself is running (chainload + boot
+ * script execution), which is exactly the boot/cleaning/deploy phase we want
+ * visibility into.
+ */
+#define CONSOLE_SYSLOG

containers/ironic/Dockerfile

@@ -51,7 +51,8 @@ RUN \
 COPY containers/ironic/ironic-console-pod.yaml.template /var/lib/openstack/lib/python3.12/site-packages/ironic/console/container/
 
 # renovate: datasource=docker depName=ghcr.io/rackerlabs/understack/ironic-ipxe versioning=semver
-FROM ghcr.io/rackerlabs/understack/ironic-ipxe:1.0.0 AS ipxe
+# FROM ghcr.io/rackerlabs/understack/ironic-ipxe:1.0.0 AS ipxe
+FROM ghcr.io/rackerlabs/understack/ironic-ipxe:pr-2527 AS ipxe
 
 FROM quay.io/airshipit/ironic:${OPENSTACK_VERSION}-ubuntu_noble AS final
 

ironic-images/custom_elements/undercloud-rsyslog/install.d/60-undercloud-rsyslog-enable

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# Enable rsyslog and the boot-time configuration unit so remote log forwarding
+# is wired up on every ramdisk boot. The actual destination is resolved at boot
+# from the kernel cmdline by undercloud-rsyslog-configure.
+
+# dib-lint: disable=set setu setpipefail indent
+if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
+    set -x
+fi
+set -e
+
+# rsyslog's Debian package usually enables itself via systemd presets, but make
+# it explicit so the ramdisk always starts it.
+systemctl enable rsyslog.service || true
+systemctl enable undercloud-rsyslog-configure.service

ironic-images/custom_elements/undercloud-rsyslog/package-installs.yaml

@@ -0,0 +1 @@
+rsyslog:

ironic-images/custom_elements/undercloud-rsyslog/static/etc/systemd/system/undercloud-rsyslog-configure.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Configure rsyslog remote forwarding from kernel cmdline (undercloud)
+# Needs the network up so the first forwarded messages can reach the collector.
+After=network-online.target systemd-journald.service
+Wants=network-online.target
+# Configure before rsyslog starts so the generated drop-in is present on first start.
+Before=rsyslog.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/bin/undercloud-rsyslog-configure
+
+[Install]
+WantedBy=multi-user.target

ironic-images/custom_elements/undercloud-rsyslog/static/usr/local/bin/undercloud-rsyslog-configure

@@ -0,0 +1,71 @@
+#!/bin/bash
+#
+# Configure rsyslog to forward the IPA ramdisk's logs to a remote syslog
+# collector during cleaning/inspection/deploy.
+#
+# The destination is read from the kernel command line so the image stays
+# site-agnostic (one shared IPA image across all sites). Ironic passes it via
+# [redfish]kernel_append_params / [pxe]pxe_append_params in each deploy site's
+# ironic/values.yaml, e.g.  syslog_server=10.46.100.83  (optionally with
+# syslog_port=514). This mirrors how the stock dynamic-login element reads
+# rootpwd=/sshkey= from /proc/cmdline.
+#
+# rsyslog reads the systemd journal (imjournal), so everything IPA logs to the
+# journal is forwarded. Forwarding is RFC3164 over TCP by default (reliable);
+# the OTEL syslog receiver listens on both UDP and TCP 514.
+
+# dib-lint: disable=set setu setpipefail indent
+if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
+    set -x
+fi
+set -e
+
+CMDLINE=$(</proc/cmdline)
+
+SYSLOG_SERVER=""
+if [[ $CMDLINE =~ syslog_server=\"?([^\"[:space:]]+)\"? ]]; then
+    SYSLOG_SERVER="${BASH_REMATCH[1]}"
+fi
+
+# No target configured -> leave rsyslog at its defaults (local only) and exit
+# cleanly so this never blocks boot.
+if [ -z "$SYSLOG_SERVER" ]; then
+    echo "undercloud-rsyslog: no syslog_server= on kernel cmdline, skipping remote forwarding"
+    exit 0
+fi
+
+SYSLOG_PORT="514"
+if [[ $CMDLINE =~ syslog_port=\"?([0-9]+)\"? ]]; then
+    SYSLOG_PORT="${BASH_REMATCH[1]}"
+fi
+
+# Protocol: @@ = TCP (reliable), @ = UDP. Default to TCP; allow syslog_proto=udp.
+PREFIX="@@"
+if [[ $CMDLINE =~ syslog_proto=udp ]]; then
+    PREFIX="@"
+fi
+
+cat > /etc/rsyslog.d/99-undercloud-remote.conf <<EOF
+# Managed by undercloud-rsyslog element (generated at boot from kernel cmdline).
+# Pull from the systemd journal and forward everything to the remote collector.
+module(load="imjournal" StateFile="imjournal.state")
+
+# Queue so transient collector/network blips during cleaning don't wedge boot.
+action(
+    type="omfwd"
+    target="${SYSLOG_SERVER}"
+    port="${SYSLOG_PORT}"
+    protocol="$([ "$PREFIX" = "@@" ] && echo tcp || echo udp)"
+    action.resumeRetryCount="-1"
+    queue.type="linkedList"
+    queue.saveOnShutdown="off"
+    queue.size="10000"
+)
+EOF
+
+echo "undercloud-rsyslog: forwarding journal to ${PREFIX}${SYSLOG_SERVER}:${SYSLOG_PORT}"
+
+# Restart rsyslog if it is already running so the new config takes effect.
+if systemctl is-enabled rsyslog >/dev/null 2>&1; then
+    systemctl restart rsyslog || true
+fi

ironic-images/ipa-debian-bookworm.yaml

@@ -9,5 +9,6 @@
   - journal-to-console
   - package-installs
   - undercloud-ipa
+  - undercloud-rsyslog
   - install-static
   - undercloud-proliant-tools

ironic-images/ipa-debian-trixie.yaml

@@ -13,6 +13,7 @@
   - journal-to-console
   - package-installs
   - undercloud-ipa
+  - undercloud-rsyslog
   - install-static
   - undercloud-proliant-tools
   # Intel E810 ice DDP (firmware-intel-misc). Trixie-only: the package name