From eda0fce799ae06c34e72088528777a6f832e1dae Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 30 Sep 2026 08:57:59 +0000 Subject: [PATCH 1/8] Fix Ironic inspection kernel options When inspecting nodes using the agent inspect interface, the node may boot either via ironic_dnsmasq (discovery) or via neutron-dhcp-agent (conductor-managed boot). When conductor prepares the PXE configuration for managed inspection, [inspector] extra_kernel_params was not configured, causing IPA to boot without inspection collectors (ipa-inspection-collectors). IPA then sent an empty data payload to ironic-api, failing schema validation with HTTP 400 and causing inspection to time out. This change: - Configures [inspector] extra_kernel_params in ironic.conf from inspector_extra_kernel_options so conductor-managed inspection boot supplies the required IPA collectors. Depends-On: https://review.opendev.org/c/openstack/kayobe-config-dev/+/1003925 Change-Id: I44773ec8a17314b745f286edf20217a84eeee69b Signed-off-by: Will Szumski --- ansible/kolla-openstack.yml | 1 + ansible/roles/kolla-openstack/defaults/main.yml | 3 +++ .../kolla-openstack/templates/kolla/config/ironic.conf | 4 ++++ .../notes/fix-inspection-reliability-4d2b6c8e9f01a357.yaml | 6 ++++++ 4 files changed, 14 insertions(+) create mode 100644 releasenotes/notes/fix-inspection-reliability-4d2b6c8e9f01a357.yaml diff --git a/ansible/kolla-openstack.yml b/ansible/kolla-openstack.yml index c02fafe66..ea87f0bf0 100644 --- a/ansible/kolla-openstack.yml +++ b/ansible/kolla-openstack.yml @@ -140,6 +140,7 @@ kolla_inspector_keep_ports: "{{ inspector_keep_ports }}" kolla_inspector_enable_discovery: "{{ inspector_enable_discovery }}" kolla_inspector_discovery_enroll_node_driver: "{{ inspector_discovery_enroll_node_driver }}" + kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}" kolla_inspector_ipa_host: "{{ groups['controllers_with_ironic_enabled_True'][0] }}" kolla_openstack_custom_config_paths_extra_multi_env_static: - "{{ kayobe_config_path }}" diff --git a/ansible/roles/kolla-openstack/defaults/main.yml b/ansible/roles/kolla-openstack/defaults/main.yml index 3b63e766d..433180bd5 100644 --- a/ansible/roles/kolla-openstack/defaults/main.yml +++ b/ansible/roles/kolla-openstack/defaults/main.yml @@ -563,6 +563,9 @@ kolla_inspector_enable_discovery: # The Ironic driver with which to register newly discovered nodes. kolla_inspector_discovery_enroll_node_driver: +# List of extra kernel parameters for the inspection ramdisk when booted by Ironic. +kolla_inspector_extra_kernel_options: + # URL of Ironic Python Agent (IPA) kernel image for Ironic Inspector. # Mutually exclusive with kolla_inspector_ipa_kernel_path. kolla_inspector_ipa_kernel_upstream_url: diff --git a/ansible/roles/kolla-openstack/templates/kolla/config/ironic.conf b/ansible/roles/kolla-openstack/templates/kolla/config/ironic.conf index 3961eb631..e0bb64057 100644 --- a/ansible/roles/kolla-openstack/templates/kolla/config/ironic.conf +++ b/ansible/roles/kolla-openstack/templates/kolla/config/ironic.conf @@ -69,6 +69,10 @@ keep_ports = {{ kolla_inspector_keep_ports }} hooks = {{ kolla_inspector_hooks | join(',') }} {% endif %} +{% if kolla_inspector_extra_kernel_options %} +extra_kernel_params = {{ kolla_inspector_extra_kernel_options if kolla_inspector_extra_kernel_options is string else kolla_inspector_extra_kernel_options | join(' ') }} +{% endif %} + {% if kolla_extra_ironic %} ####################### # Extra configuration diff --git a/releasenotes/notes/fix-inspection-reliability-4d2b6c8e9f01a357.yaml b/releasenotes/notes/fix-inspection-reliability-4d2b6c8e9f01a357.yaml new file mode 100644 index 000000000..4215074b7 --- /dev/null +++ b/releasenotes/notes/fix-inspection-reliability-4d2b6c8e9f01a357.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue with agent-based bare metal inspection, where the collectors + were not being configured by Ironic. This was the case when using the + managed inspection mode in Ironic. From df5617498934dd7903690cc909fe860c87045cef Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 30 Sep 2026 08:57:59 +0000 Subject: [PATCH 2/8] Revert "CI: Disable inspection testing" This reverts commit 372e505a474c8c137796e1f936b3f6431d319bcd. Inspection failures in CI should be resolved by the preceding fix to configure the Ironic inspection kernel options. Change-Id: I69051bfb43567458ffd701010a65a83d8534b17e Signed-off-by: Will Szumski --- playbooks/kayobe-overcloud-base/run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/kayobe-overcloud-base/run.yml b/playbooks/kayobe-overcloud-base/run.yml index 2d1076858..74c16090b 100644 --- a/playbooks/kayobe-overcloud-base/run.yml +++ b/playbooks/kayobe-overcloud-base/run.yml @@ -49,7 +49,7 @@ chdir: "{{ kayobe_src_dir }}" executable: /bin/bash # TODO(priteau): Fix baremetal inspect issues with UEFI - when: test_inspection | default(false) and ironic_boot_mode == 'bios' + when: ironic_boot_mode == 'bios' - name: Perform testing of the baremetal machines shell: From 3cf705b318f69cf9bf0c438009d2dccaf59c91a2 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 2 Oct 2026 09:56:29 +0200 Subject: [PATCH 3/8] CI: Make all experimental jobs voting These jobs are not failing anymore, so there is no reason to keep them as non-voting. There is no impact anyway since this is in the experimental pipeline. Change-Id: Ib0fc67c51dc0974b6031993c5f4a165d9b1c8618 Signed-off-by: Pierre Riteau --- zuul.d/jobs.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 81a4a3cc0..25d65a9c4 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -431,7 +431,6 @@ name: kayobe-seed-vm-ubuntu-noble-cloud-image parent: kayobe-seed-vm-base nodeset: kayobe-ubuntu-noble - voting: false vars: seed_vm_use_cirros: false @@ -490,6 +489,5 @@ name: kayobe-infra-vm-ubuntu-noble-cloud-image parent: kayobe-infra-vm-base nodeset: kayobe-ubuntu-noble - voting: false vars: infra_vm_use_cirros: false From 043352ae97df010afad9fb311541b96c62a01804 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 2 Oct 2026 21:53:35 +0200 Subject: [PATCH 4/8] Revert "CI: Prefer ipv4 over ipv6" This reverts commit 9e55adad9fe28100339ea2a0e31aecde80374017. Reason for revert: IPv6 is fixed in ca-ymq-1 region Change-Id: Ie91a072a05c9345483c03b6038a2547ead8d519c Signed-off-by: Pierre Riteau --- roles/kayobe-ci-prep/tasks/main.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/roles/kayobe-ci-prep/tasks/main.yml b/roles/kayobe-ci-prep/tasks/main.yml index f2cb6735b..cc813b87b 100644 --- a/roles/kayobe-ci-prep/tasks/main.yml +++ b/roles/kayobe-ci-prep/tasks/main.yml @@ -1,13 +1,4 @@ --- -# NOTE: prefer IPv4 over IPv6 in getaddrinfo() to avoid slow/failing AAAA -# lookups against hosts without real IPv6 connectivity -- name: Prefer IPv4 over IPv6 for name resolution - become: true - ansible.builtin.lineinfile: - path: /etc/gai.conf - line: "precedence ::ffff:0:0/96 100" - create: true - - name: Configure secondary storage drive #NOTE(wszumski) Some instances provide a smaller 40G root disk with a #secondary storage drive. This kolla-ansible provided role sets up the From 34210b16fb3e5aba6beade12ce9e70b659c60e46 Mon Sep 17 00:00:00 2001 From: MurrayGroves Date: Tue, 6 Oct 2026 11:03:42 +0100 Subject: [PATCH 5/8] Explicitly convert `bifrost_limit` to boolean `bifrost_limit` is a list containing bifrost hosts. Ansible no longer allows non-booleans to be implicitly converted to booleans in conditionals. Instead, we check if the length is above 0. Change-Id: Icd6f8525e07f696d5f45c7506e50421326726d81 Signed-off-by: MurrayGroves --- ansible/overcloud-provision.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/overcloud-provision.yml b/ansible/overcloud-provision.yml index 424b45756..a25621d63 100644 --- a/ansible/overcloud-provision.yml +++ b/ansible/overcloud-provision.yml @@ -159,7 +159,7 @@ -e @/etc/bifrost/bifrost.yml -e @/etc/bifrost/dib.yml --limit {{ bifrost_limit | join(':') }}' - when: bifrost_limit + when: bifrost_limit | length > 0 delegate_to: "{{ seed_host }}" vars: # NOTE: Without this, the seed's ansible_host variable will not be From b52e11c6f6eec344a1bcb63d0bb2f58b5f1c3605 Mon Sep 17 00:00:00 2001 From: MurrayGroves Date: Tue, 6 Oct 2026 11:23:34 +0100 Subject: [PATCH 6/8] Fix cpu_mode and memory for dev overcloud Leaving `cpu_mode` as none caused IPA to never boot during inspection when following https://docs.openstack.org/kayobe/latest/contributor/automated.html#seed. When left as none, the hypervisor (libvirt) is allowed to choose which cpu model to use. It chooses qemu64 which is a very basic model and not recommended for use: https://qemu-project.gitlab.io/qemu/system/qemu-cpu-models.html#default-x86-cpu-models Instead we now set it to maximum, which is used in the other tenks dev configs, and ensures all CPU features are passed through. The memory allocation was also too small, causing an out of space error when downloading the deployment image during provisioning. Change-Id: I12baa91022d3c444d67f13b817429d5471c91ebe Signed-off-by: MurrayGroves --- dev/tenks-deploy-config-overcloud.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dev/tenks-deploy-config-overcloud.yml b/dev/tenks-deploy-config-overcloud.yml index 521f7d03a..86209e874 100644 --- a/dev/tenks-deploy-config-overcloud.yml +++ b/dev/tenks-deploy-config-overcloud.yml @@ -4,7 +4,7 @@ node_types: type0: - memory_mb: 3072 + memory_mb: 4096 vcpus: 1 volumes: # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent: @@ -14,9 +14,7 @@ node_types: physical_networks: - physnet1 console_log_enabled: true - # We seem to hit issues with missing cpu features in CI as a result of using host-model, e.g: - # https://zuul.opendev.org/t/openstack/build/02c33ab51664419a88a5a54ad22852a9/log/primary/system_logs/libvirt/qemu/tk0.txt.gz#38 - cpu_mode: + cpu_mode: maximum specs: - type: type0 From 2b54fd10a922a481efd7f42af7a5647128214ec4 Mon Sep 17 00:00:00 2001 From: MurrayGroves Date: Wed, 7 Oct 2026 17:41:49 +0100 Subject: [PATCH 7/8] Add more context to automated setup docs Since the automated setup docs are part of the new contributor experience, it is likely the reader lacks context about different ways in which the seed can be provisioned. This adds some details in the seed provisioning explaining the choice between running the seed directly on the host, or using it as a hypervisor. The setup is also unclear that at the end of the seed setup the overcloud has not been setup. Added a brief section with some next steps. Change-Id: Ic4c1c492766886ba4a3abf306d35ff28e8f77395 Signed-off-by: MurrayGroves --- doc/source/contributor/automated.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/source/contributor/automated.rst b/doc/source/contributor/automated.rst index 42eeb8f23..70306f4e6 100644 --- a/doc/source/contributor/automated.rst +++ b/doc/source/contributor/automated.rst @@ -278,6 +278,13 @@ Run the ``dev/seed-deploy.sh`` script to deploy the seed services:: export KAYOBE_SEED_VM_PROVISION=0 ./dev/seed-deploy.sh +``KAYOBE_SEED_VM_PROVISION=0`` tells the deployment script to skip deploying the +seed vm. This is because we set up our host's IP as ``192.168.33.5``, which is +set in ``network-allocation.yml`` as the seed's IP - meaning Kayobe will use our +host as the seed. If you would prefer to run the seed in a VM on your host, +you can set your IP as ``192.168.33.4`` to match with ``seed-hypervisor`` in +``network-allocation.yml``, and ``export KAYOBE_SEED_VM_PROVISION=1``. + Upon successful completion of this script, the seed will be active. Testing @@ -319,6 +326,12 @@ It is now possible to discover, inspect and provision the controller VM:: The controller VM is now accessible via SSH as the bootstrap user (``cloud-user``, ``rocky`` or ``ubuntu``) at ``192.168.33.3``. +At this point you may wish to continue setting up your development environment with:: + + kayobe overcloud host configure + kayobe overcloud container image pull + kayobe overcloud service deploy + The machines and networking created by Tenks can be cleaned up via ``dev/tenks-teardown-overcloud.sh``:: From db8fda0828c509e4ed11f38a1550df0e9dcd56fc Mon Sep 17 00:00:00 2001 From: Michal Nasiadka Date: Fri, 21 Aug 2026 09:40:06 +0200 Subject: [PATCH 8/8] Fix merge_configs/yaml and concat behaviour Closes-Bug: #2164718 Change-Id: I9a7b12c660978f51b6102d58cc64de0d8dc30dd4 Signed-off-by: Michal Nasiadka --- ansible/roles/kolla-openstack/tasks/config.yml | 11 +++++++---- kayobe/plugins/action/merge_configs.py | 6 +++++- kayobe/plugins/action/merge_yaml.py | 6 +++++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/ansible/roles/kolla-openstack/tasks/config.yml b/ansible/roles/kolla-openstack/tasks/config.yml index ce318fbc6..96cda133d 100644 --- a/ansible/roles/kolla-openstack/tasks/config.yml +++ b/ansible/roles/kolla-openstack/tasks/config.yml @@ -117,12 +117,15 @@ - name: "Ensure extra configuration files exist (strategy: concat)" vars: params: - content: | - {%- for path in item.sources -%} - {{ lookup('file', path) }} - {%- endfor -%} + # join() guarantees exactly one newline between sources, regardless + # of whether a source file ends with one - lookup('file') rstrips + # trailing whitespace from each source, so concatenating without an + # explicit separator can glue the last line of one file to the first + # line of the next. + content: "{{ query('file', *item.sources) | join('\n') }}\n" dest: "{{ item.dest }}" mode: 0640 + lstrip_blocks: true template_content: "{{ params | combine(item.params) }}" with_items: "{{ kolla_custom_config_info.concat }}" diff --git a/kayobe/plugins/action/merge_configs.py b/kayobe/plugins/action/merge_configs.py index a75ba18c7..7f2154375 100644 --- a/kayobe/plugins/action/merge_configs.py +++ b/kayobe/plugins/action/merge_configs.py @@ -167,7 +167,11 @@ def read_config(self, source, config): ] templar = self._templar.copy_with_new_env(searchpath=searchpath) - result = templar.template(template_data) + + # lstrip_blocks avoids Jinja2 block tags (e.g. {% if %}) leaving + # behind leading whitespace that glues adjacent lines together. + result = templar.template( + template_data, overrides=dict(lstrip_blocks=True)) fakefile = StringIO(result) config.parse(fakefile) fakefile.close() diff --git a/kayobe/plugins/action/merge_yaml.py b/kayobe/plugins/action/merge_yaml.py index 3d122232f..b34e41181 100644 --- a/kayobe/plugins/action/merge_yaml.py +++ b/kayobe/plugins/action/merge_yaml.py @@ -107,7 +107,11 @@ def read_config(self, source): ] templar = self._templar.copy_with_new_env(searchpath=searchpath) - template_data = templar.template(template_data) + + # lstrip_blocks avoids Jinja2 block tags (e.g. {% if %}) leaving + # behind leading whitespace that glues adjacent lines together. + template_data = templar.template( + template_data, overrides=dict(lstrip_blocks=True)) result = yaml.safe_load(template_data) return result or {}