Configure the Octavia amphora provider driver and related networking in ci-multinode on OVS

.automation.conf/tempest/tempest-ci-multinode-octavia-smoke.overrides.conf

@@ -0,0 +1 @@
+tempest-ci-multinode-octavia.overrides.conf

.automation.conf/tempest/tempest-ci-multinode-octavia.overrides.conf

@@ -0,0 +1,56 @@
+# Copy of tempest-ci-multinode.overrides.conf with the [load_balancer] and
+# [network-feature-enabled] sections appended; keep the shared parts in sync.
+# The filename is dictated by config.sh and cannot be made more descriptive.
+
+[DEFAULT]
+openstack_client_http_timeout = 300
+
+[auth]
+compute_quotas = cores:-1,ram:-1
+
+[identity]
+v3_endpoint_type = publicURL
+
+[compute]
+min_compute_nodes = 2
+min_microversion = 2.1
+max_microversion = 2.96
+
+[service-clients]
+http_timeout = 600
+
+[compute-feature-enabled]
+resize = true
+live_migration = true
+block_migration_for_live_migration = false
+volume_backed_live_migration = true
+console_output = true
+
+[volume]
+storage_protocol = ceph
+build_timeout = 600
+min_microversion = 3.0
+max_microversion = 3.71
+
+[image]
+build_timeout = 600
+
+[dashboard]
+dashboard_url = https://192.168.39.2
+
+[load_balancer]
+# The legacy 'octavia' alias is absent from octavia_provider_drivers.
+provider = amphora
+# This environment has no shared IPv6 subnet pool.
+test_with_ipv6 = False
+# Must outlast Octavia's own amp_active_retries * amp_active_wait_sec of 900s.
+lb_build_timeout = 1200
+# Listener and pool waits.
+build_timeout = 900
+# Member add waits.
+check_timeout = 600
+# With port_security below, makes the plugin attach a security group to backends.
+enable_security_groups = True
+
+[network-feature-enabled]
+port_security = True

.gitignore

@@ -57,6 +57,7 @@ etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.client.glance.keyring
 # Ignore generated Octavia certificates in the AIO environments
 etc/kayobe/environments/aio/kolla/config/octavia/*.pem
 etc/kayobe/environments/ci-aio/kolla/config/octavia/*.pem
+etc/kayobe/environments/ci-multinode/kolla/config/octavia/*.pem
 
 # Tempest logs
 tempest-artifacts

doc/source/operations/octavia.rst

@@ -2,6 +2,30 @@
 Octavia
 =======
 
+Management network requirements
+===============================
+
+The Amphora provider driver runs each load balancer inside its own virtual
+machine, called an amphora. The Octavia services and the amphorae talk to each
+other directly, so they need to share a network:
+
+* Octavia connects to each amphora on TCP port 9443 to configure it.
+* Each amphora sends health reports back to Octavia on UDP port 5555.
+
+Octavia therefore needs an address of its own on the management network, and
+Neutron needs a network on the same segment to attach the amphorae to.
+
+If both of those are carried by the same network interface on the host, that
+interface must be a Linux bridge. When a plain interface has VLAN subinterfaces
+on it, the host keeps the traffic for itself and Open vSwitch never receives it,
+so amphorae start up but Octavia cannot reach them. A Linux bridge passes a copy
+of the traffic to everything attached to it, so the host and Open vSwitch both
+receive it.
+
+The ``ci-multinode`` environment is set up this way: the VXLAN mesh device sits
+inside a Linux bridge, and Open vSwitch attaches to the same bridge through a
+veth pair.
+
 .. _Amphora image:
 
 Updating amphora images

etc/kayobe/ansible/tools/configure-vxlan.yml

@@ -9,5 +9,25 @@
     ansible_python_interpreter: /usr/bin/python3
     # Work around no known_hosts entry on first boot.
     ansible_ssh_common_args: -o StrictHostKeyChecking=no
+  pre_tasks:
+    # stackhpc.vxlan connects VXLAN to a master when an interface
+    # is of the 'bridge' type, but Network Manager cannot attach the device
+    # before the bridge exists. Kayobe creates the bridge later, so we will create it here
+    # and have Kayobe overwrite it later as needed
+    - name: Get existing NetworkManager connections
+      ansible.builtin.command: nmcli -g NAME connection show
+      become: true
+      changed_when: false
+      register: vxlan_nm_connections
+
+    - name: Ensure a bridge exists for each VXLAN interface that needs one
+      ansible.builtin.command:
+        cmd: >-
+          nmcli connection add type bridge
+          con-name {{ item.bridge }} ifname {{ item.bridge }}
+          stp no ipv4.method disabled ipv6.method disabled
+      become: true
+      loop: "{{ vxlan_interfaces | selectattr('bridge', 'defined') | list }}"
+      when: item.bridge not in vxlan_nm_connections.stdout_lines
   roles:
     - role: stackhpc.vxlan

etc/kayobe/environments/ci-multinode/controllers.yml

@@ -5,6 +5,10 @@ controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' el
 controller_lvm_groups:
   - "{{ stackhpc_lvm_group_rootvg }}"
 
+# Octavia management network, for the health manager to receive amphora
+# heartbeats. OVN provider driver needs no management network.
+controller_extra_network_interfaces: "{{ ['lb_mgmt'] if octavia_amphora_driver_enabled | bool else [] }}"
+
 
 ###############################################################################
 # Controller node firewalld configuration.

etc/kayobe/environments/ci-multinode/globals.yml

@@ -60,9 +60,20 @@ os_release: >-
 stackhpc_write_barbican_role_id_to_file: true
 stackhpc_barbican_role_id_file_path: "/tmp/barbican-role-id"
 
+# Trust store for Kayobe's own playbooks. The Octavia amphora image
+# register playbook cannot verify Glance's TLS certificate without this.
+openstack_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution == 'rocky' else '/etc/ssl/certs/ca-certificates.crt' }}"
+
 # Enable rebooting to update SELinux state
 selinux_do_reboot: true
 
+###############################################################################
+# Octavia.
+
+# Use Octavia's amphora provider driver instead of the OVN one. Requires
+# nested virtualisation on the compute hosts, which the CI cloud (Leaf Cloud as of now) does not have.
+octavia_amphora_driver_enabled: false
+
 ###############################################################################
 # Extra vars.
 

etc/kayobe/environments/ci-multinode/hooks/overcloud-host-configure/post.d/10-check-kvm.yml

@@ -0,0 +1,31 @@
+---
+
+- hosts: compute
+  gather_facts: true
+  tasks:
+    - name: Assert that the host architecture supports the amphora driver
+      assert:
+        that: ansible_facts.architecture == 'x86_64'
+        fail_msg: >-
+          The Octavia amphora driver is only supported on x86_64 hosts, because
+          the amphora image is built for x86_64. This host is
+          {{ ansible_facts.architecture }}. Set octavia_amphora_driver_enabled
+          to false to deploy with the OVN provider driver instead.
+      when: octavia_amphora_driver_enabled | bool
+
+    - name: Check whether /dev/kvm exists
+      stat:
+        path: /dev/kvm
+      register: kvm_device
+      when: octavia_amphora_driver_enabled | bool
+
+    - name: Assert that KVM is available
+      assert:
+        that: kvm_device.stat.exists
+        fail_msg: >-
+          /dev/kvm is not present. The ci-multinode environment uses KVM when
+          the Octavia amphora driver is in use, because amphorae are unusable
+          under emulation. Enable nested virtualisation on this host, or set
+          octavia_amphora_driver_enabled to false to deploy with the OVN
+          provider driver.
+      when: octavia_amphora_driver_enabled | bool

etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml

@@ -16,3 +16,6 @@ vxlan_vni:
 vxlan_interfaces:
   - device: "vxlan{{ vxlan_vni }}"
     group: "{{ '239.0.0.0/8' | ansible.utils.next_nth_usable(vxlan_vni) }}"
+    # Enslave the mesh to the bridge here, so the role owns the one connection
+    # profile for this device and Kayobe does not overwrite it as an ethernet.
+    bridge: br-vxlan

etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces

@@ -4,11 +4,15 @@
 
 admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}"
 
-internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}"
+# Linux bridge containing the VXLAN mesh device. Open vSwitch reaches the mesh
+# through a veth pair from this bridge, so the host and Neutron both get frames.
+vxlan_mesh_interface: br-vxlan
 
-storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}"
+internal_interface: "{{ vxlan_mesh_interface }}.{{ internal_vlan }}"
 
-tunnel_interface: "{{ vxlan_interfaces[0].device }}.{{ tunnel_vlan }}"
+storage_interface: "{{ vxlan_mesh_interface }}.{{ storage_vlan }}"
+
+tunnel_interface: "{{ vxlan_mesh_interface }}.{{ tunnel_vlan }}"
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.

etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces

@@ -4,13 +4,22 @@
 
 admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}"
 
-internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}"
+# Linux bridge containing the VXLAN mesh device. Open vSwitch reaches the mesh
+# through a veth pair from this bridge, so the host and Neutron both get frames.
+vxlan_mesh_interface: br-vxlan
 
-storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}"
+internal_interface: "{{ vxlan_mesh_interface }}.{{ internal_vlan }}"
 
-tunnel_interface: "{{ vxlan_interfaces[0].device }}.{{ tunnel_vlan }}"
+storage_interface: "{{ vxlan_mesh_interface }}.{{ storage_vlan }}"
 
-public_interface: "{{ vxlan_interfaces[0].device }}.{{ public_vlan }}"
+tunnel_interface: "{{ vxlan_mesh_interface }}.{{ tunnel_vlan }}"
+
+public_interface: "{{ vxlan_mesh_interface }}.{{ public_vlan }}"
+
+# Octavia management network. Bridge as interface instead of mesh device
+# Open vSwitch will still see frames the host has not claimed due to
+# bridge's broadcast
+lb_mgmt_interface: "{{ vxlan_mesh_interface }}.{{ lb_mgmt_vlan }}"
 
 public_routes:
   - cidr: "{{ external_cidr }}"

etc/kayobe/environments/ci-multinode/kolla/config/octavia.conf

@@ -0,0 +1,5 @@
+# Amphorae boot from Ceph-backed Cinder volumes here, which can be slow.
+# Increase timeout from 5 minutes to 15 minutes for Nova to report them active.
+[controller_worker]
+amp_active_retries = 90
+amp_active_wait_sec = 10

etc/kayobe/environments/ci-multinode/kolla/globals.yml

@@ -17,9 +17,11 @@ rabbitmq_enable_tls: "yes"
 
 ############################################################################
 # Most development environments will use nested virtualisation, and we can't
-# guarantee that nested KVM support is available. Use QEMU as a lowest common
-# denominator.
-nova_compute_virt_type: qemu
+# guarantee that nested KVM support is available, so QEMU is the lowest common
+# denominator. However, amphorae are unusable under emulation in terms of time
+# during tempest testing and Octavia times out waiting for them to spin up
+# we require KVM when the amphora driver is in use
+nova_compute_virt_type: "{{ 'kvm' if octavia_amphora_driver_enabled | bool else 'qemu' }}"
 
 # Reduce the control plane's memory footprint by limiting the number of worker
 # processes to two per-service when running in a VM.
@@ -35,22 +37,36 @@ cinder_backend_ceph: "yes"
 # Nova Ceph configuration
 nova_backend_ceph: "yes"
 
-# Because this is a vxlan interface, Kayobe can't generate the config for it.
-# Therefore we can't use this interface as the external network
-# (external_net_names). Instead, make external_net_names an empty
-# list, and add the config that would be generated from it here.
-neutron_bridge_name: "{{ vxlan_interfaces[0].device }}-ovs"
-neutron_external_interface: "{{ vxlan_interfaces[0].device }}"
-
 neutron_dns_domain: "{{ root_domain }}."
 
 # OpenSearch memory tuning
 opensearch_heap_size: 1g
 
-# Octavia load balancer configuration
-octavia_auto_configure: "no"
-octavia_provider_drivers: "ovn:OVN provider"
-octavia_provider_agents: "ovn"
+# Octavia load balancer configuration.
+
+octavia_provider_drivers: "{{ 'amphora:Amphora provider' if octavia_amphora_driver_enabled | bool else 'ovn:OVN provider' }}"
+octavia_provider_agents: "{{ 'amphora_agent' if octavia_amphora_driver_enabled | bool else 'ovn' }}"
+
+# Host interface the health manager binds to. octavia.conf references this
+# unconditionally, so fall back to the internal network on OVN, which is what
+# kolla's api_interface default resolves to.
+octavia_network_interface: "{{ (octavia_net_name if octavia_amphora_driver_enabled | bool else internal_net_name) | net_interface(groups['controllers'][0]) }}"
+
+# Management network registered by octavia_auto_configure, so only with amphora.
+octavia_amp_network:
+  name: lb-mgmt-net
+  provider_network_type: vlan
+  provider_segmentation_id: "{{ lb_mgmt_vlan }}"
+  provider_physical_network: physnet1
+  external: false
+  shared: false
+  subnet:
+    name: lb-mgmt-subnet
+    cidr: "{{ lb_mgmt_cidr }}"
+    allocation_pool_start: "{{ lb_mgmt_neutron_allocation_pool_start }}"
+    allocation_pool_end: "{{ lb_mgmt_neutron_allocation_pool_end }}"
+    no_gateway_ip: true
+    enable_dhcp: true
 
 # Manila CephFS configuration
 manila_cephfs_filesystem_name: manila-cephfs

etc/kayobe/environments/ci-multinode/networks.yml

@@ -38,7 +38,8 @@ internal_net_name: internal
 # If external_net_name is defined, external_net_names will default to a list
 # containing one item, external_net_name.
 #external_net_names:
-external_net_names: []
+external_net_names:
+  - vxlan_mesh
 
 # Name of the network used to expose the public OpenStack API endpoints.
 public_net_name: public
@@ -68,6 +69,9 @@ storage_mgmt_net_name: storage_mgmt
 # hosts
 #cleaning_net_name:
 
+# Name of the network used to manage octavia loadbalancers
+octavia_net_name: lb_mgmt
+
 ###############################################################################
 # Network definitions.
 
@@ -131,6 +135,26 @@ provision_oc_allocation_pool_end: 192.168.33.254
 provision_oc_vlan: 107
 provision_oc_zone: "provision_oc"
 
+# Octavia load balancer management network. VLAN provider network on physnet1
+# reusing the existing VXLAN mesh
+lb_mgmt_cidr: 192.168.34.0/24
+lb_mgmt_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}"
+lb_mgmt_vlan: 108
+lb_mgmt_allocation_pool_start: 192.168.34.3
+lb_mgmt_allocation_pool_end: 192.168.34.30
+lb_mgmt_neutron_allocation_pool_start: 192.168.34.100
+lb_mgmt_neutron_allocation_pool_end: 192.168.34.200
+lb_mgmt_zone: lb-mgmt
+
+# Linux bridge carrying the VXLAN mesh. Without the bridge the
+# host will claim the packet and drop if not addressed to its listener.
+# A bridge is needed to flood the packet to both the OS and the OVS.
+# Empty ports because stackhpc.vxlan does the enslaving via its bridge option.
+# We dont want MichaelRigart's role to overwrite the Network Manager's VXLAN handling
+vxlan_mesh_bridge_ports: []
+vxlan_mesh_bootproto: manual
+vxlan_mesh_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}"
+
 ###############################################################################
 # Network virtual patch link configuration.
 

etc/kayobe/environments/ci-multinode/stackhpc-ci.yml

@@ -64,5 +64,8 @@ stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?:
 stackhpc_docker_registry_username: "{{ stackhpc_repo_mirror_username }}"
 stackhpc_docker_registry_password: "{{ stackhpc_repo_mirror_password }}"
 
+# Credentials for downloading the Octavia amphora image from Ark
+stackhpc_release_pulp_username: "{{ stackhpc_repo_mirror_username }}"
+stackhpc_release_pulp_password: "{{ stackhpc_repo_mirror_password }}"
 # Path to a CA certificate file to trust
 stackhpc_cloud_tests_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/{{ stackhpc_ca_secret_store }}.crt"