From 6101e18f05aafc22d891b48a66d4d493f395f9d3 Mon Sep 17 00:00:00 2001 From: Hollie Hutchinson Date: Wed, 14 Oct 2026 13:46:31 +0100 Subject: [PATCH 1/3] VF-LAG host configure --- doc/source/contributor/ofed.rst | 20 ++- .../deployment/vf-lag-host-configure.yml | 138 ++++++++++++++++++ ...vflag-host-configure-4ba351fa23327e16.yaml | 4 + 3 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 etc/kayobe/ansible/deployment/vf-lag-host-configure.yml create mode 100644 releasenotes/notes/vflag-host-configure-4ba351fa23327e16.yaml diff --git a/doc/source/contributor/ofed.rst b/doc/source/contributor/ofed.rst index 693277a392..a11870fb6f 100644 --- a/doc/source/contributor/ofed.rst +++ b/doc/source/contributor/ofed.rst @@ -2,8 +2,6 @@ OFED ==== -Warning: Experimental workflow subject to change - The Nvidia DOCA framework is distributed as part of StackHPC Release Train for OFED driver support, this repository is synced into Ark as part of the Release Train workflows, however to ensure compatibility with Release Train packages, we are required to build OFED modules with support for @@ -104,3 +102,21 @@ playbook: .. code-block:: console kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/tools/install-doca.yml + +Host configuration +------------------ + +Ensure DOCA hosts are a member of the ``vflag`` group, this will be needed to ensure VF-LAG +and DOCA configuration is applied on DOCA hosts. + +.. code-block:: ini + + [vflag:children] + mlnx + +A playbook is provided to ensure VF-LAG and OFED setup is applied on ``vflag`` hosts. To run +the configuration playbook: + +.. code-block:: console + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deployment/vf-lag-host-configure.yml diff --git a/etc/kayobe/ansible/deployment/vf-lag-host-configure.yml b/etc/kayobe/ansible/deployment/vf-lag-host-configure.yml new file mode 100644 index 0000000000..89578c627f --- /dev/null +++ b/etc/kayobe/ansible/deployment/vf-lag-host-configure.yml @@ -0,0 +1,138 @@ +--- +# Setup of VF-LAG systemd configuration. +# This playbook should execute after kayobe overcloud host configure has completed. + +- name: Enable SRIOV + hosts: vflag + tags: + - sriov + handlers: + - name: Reboot + ansible.builtin.set_fact: + kayobe_needs_reboot: true + tasks: + # NOTE: define sriov_devices in group_vars + - name: Include SRIOV role + ansible.builtin.include_role: + name: stackhpc.linux.sriov + vars: + sriov_install_enabled: true + when: + - sriov_devices is defined + - sriov_devices | length >= 0 + +- name: Prepare hosts for VF-LAG support + hosts: vflag + tags: + - vflag + vars: + # This playbook will execute after nodes are deployed + # and before overcloud host configure - we can't assume + # users and venvs exist. + ansible_user: "{{ ansible_user }}" + ansible_python_interpreter: "/usr/bin/python3" + tasks: + # TC offload requires kernel-modules-extra (act_ct module) + - name: Ensure kernel-modules-extra is installed + ansible.builtin.package: + name: kernel-modules-extra + state: present + become: true + when: ansible_facts.os_family == "RedHat" + + # /etc/sysconfig is a RedHat convention - create equivalent here + - name: Create /etc/sysconfig for Ubuntu + become: true + ansible.builtin.file: + path: /etc/sysconfig + owner: "root" + group: "root" + mode: 0755 + state: directory + when: ansible_facts.os_family == "Debian" + + # Write /etc/sysconfig/sriov configuration to match this host + - name: Define parameters in /etc/sysconfig/sriov + ansible.builtin.template: + src: "sysconfig-sriov.j2" + dest: "/etc/sysconfig/sriov" + owner: "root" + group: "root" + mode: 0755 + become: true + notify: Reboot and wait + + - name: Transfer VF-LAG helper scripts to /usr/local/bin + ansible.builtin.copy: + src: "{{ item }}" + dest: "/usr/local/bin/{{ item }}" + owner: "root" + group: "root" + mode: 0755 + become: true + with_items: + - "mlnx-vflag-early" + - "mlnx-vflag-final" + notify: Reboot and wait + + - name: Transfer VF-LAG systemd units to /etc/systemd/system + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + owner: "root" + group: "root" + mode: 0644 + become: true + with_items: + - "mlnx-vflag-early.service" + - "mlnx-vflag-final.service" + notify: Reboot and wait + + - name: Enable VF-LAG systemd units + ansible.builtin.systemd: + daemon_reload: true + name: "{{ item }}" + enabled: true + become: true + with_items: + - "mlnx-vflag-early.service" + - "mlnx-vflag-final.service" + notify: Reboot and wait + + # Purge some stuff that OFED typically installs + # which may race with the setup we perform. + + - name: Disable OFED openibd service + ansible.builtin.systemd: + name: "openibd.service" + enabled: false + state: "stopped" + become: true + + - name: Move the openib config file + ansible.builtin.command: "mv /etc/infiniband/openib.cf /etc/infiniband/openib.cf-" + args: + removes: "/etc/infiniband/openib.cf" + creates: "/etc/infiniband/openib.cf-" + become: true + + # There appears to be an issue where the reciprocal symlink from a + # WantedBy= to a Wants= doesn't get created. Explicitly ensure it exists. + - name: Ensure the .wants symlink exists (Ubuntu) + when: ansible_facts.os_family == "Debian" + become: true + block: + - name: Ensure the .wants directory exists + ansible.builtin.file: + path: "/etc/systemd/system/systemd-networkd.service.wants" + state: "directory" + - name: Ensure the .wants symlink exists + ansible.builtin.file: + path: "/etc/systemd/system/systemd-networkd.service.wants/mlnx-vflag-early.service" + src: "/etc/systemd/system/mlnx-vflag-early.service" + state: "link" + + handlers: + - name: Reboot and wait + ansible.builtin.set_fact: + kayobe_needs_reboot: true diff --git a/releasenotes/notes/vflag-host-configure-4ba351fa23327e16.yaml b/releasenotes/notes/vflag-host-configure-4ba351fa23327e16.yaml new file mode 100644 index 0000000000..b9f790e7a1 --- /dev/null +++ b/releasenotes/notes/vflag-host-configure-4ba351fa23327e16.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Move VF-LAG host configure script from SMSLab Ansible to SKC. From 271336fe118d0cdadd64184ddb118aafbf512dc2 Mon Sep 17 00:00:00 2001 From: Hollie Hutchinson Date: Wed, 14 Oct 2026 15:48:34 +0100 Subject: [PATCH 2/3] Add VFLAG services --- .../templates/mlnx-vflag-early.service.j2 | 23 +++++++++++++++++++ .../templates/mlnx-vflag-final.service.j2 | 13 +++++++++++ 2 files changed, 36 insertions(+) create mode 100644 etc/kayobe/ansible/templates/mlnx-vflag-early.service.j2 create mode 100644 etc/kayobe/ansible/templates/mlnx-vflag-final.service.j2 diff --git a/etc/kayobe/ansible/templates/mlnx-vflag-early.service.j2 b/etc/kayobe/ansible/templates/mlnx-vflag-early.service.j2 new file mode 100644 index 0000000000..6fd7fb8671 --- /dev/null +++ b/etc/kayobe/ansible/templates/mlnx-vflag-early.service.j2 @@ -0,0 +1,23 @@ +[Unit] +Description=Early configuration for Mellanox SR-IOV and VF-LAG +{% for nic in vflag_pfs %} +Requires=sys-subsystem-net-devices-{{ nic }}.device +After=sys-subsystem-net-devices-{{ nic }}.device +{% endfor %} +Requires=network-pre.target +Before=network-pre.target +{% if ansible_facts.os_family == "Debian" %} +Requires=systemd-networkd.service +Before=systemd-networkd.service +{% endif %} + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/mlnx-vflag-early +RemainAfterExit=yes + +[Install] +WantedBy=network-pre.target +{% if ansible_facts.os_family == "Debian" %} +WantedBy=systemd-networkd.service +{% endif %} diff --git a/etc/kayobe/ansible/templates/mlnx-vflag-final.service.j2 b/etc/kayobe/ansible/templates/mlnx-vflag-final.service.j2 new file mode 100644 index 0000000000..331f80af59 --- /dev/null +++ b/etc/kayobe/ansible/templates/mlnx-vflag-final.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Final configuration for Mellanox SR-IOV and VF-LAG +Wants=network-online.target +After=network-online.target +Before=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/mlnx-vflag-final +RemainAfterExit=yes + +[Install] +WantedBy=docker.service From 20ebffe83dc3ca81dd16fe597380341c3b6e5f96 Mon Sep 17 00:00:00 2001 From: Hollie Hutchinson Date: Thu, 15 Oct 2026 13:26:03 +0100 Subject: [PATCH 3/3] Disable SRIOV role on an empty list --- etc/kayobe/ansible/deployment/vf-lag-host-configure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/deployment/vf-lag-host-configure.yml b/etc/kayobe/ansible/deployment/vf-lag-host-configure.yml index 89578c627f..3cf206fc64 100644 --- a/etc/kayobe/ansible/deployment/vf-lag-host-configure.yml +++ b/etc/kayobe/ansible/deployment/vf-lag-host-configure.yml @@ -19,7 +19,7 @@ sriov_install_enabled: true when: - sriov_devices is defined - - sriov_devices | length >= 0 + - sriov_devices | length > 0 - name: Prepare hosts for VF-LAG support hosts: vflag