diff --git a/.automation.conf/tempest/tempest-ci-multinode-octavia-smoke.overrides.conf b/.automation.conf/tempest/tempest-ci-multinode-octavia-smoke.overrides.conf new file mode 120000 index 0000000000..4ab71e7a3b --- /dev/null +++ b/.automation.conf/tempest/tempest-ci-multinode-octavia-smoke.overrides.conf @@ -0,0 +1 @@ +tempest-ci-multinode-octavia.overrides.conf \ No newline at end of file diff --git a/.automation.conf/tempest/tempest-ci-multinode-octavia.overrides.conf b/.automation.conf/tempest/tempest-ci-multinode-octavia.overrides.conf new file mode 100644 index 0000000000..99a8d42012 --- /dev/null +++ b/.automation.conf/tempest/tempest-ci-multinode-octavia.overrides.conf @@ -0,0 +1,56 @@ +# Copy of tempest-ci-multinode.overrides.conf with the [load_balancer] and +# [network-feature-enabled] sections appended; keep the shared parts in sync. +# The filename is dictated by config.sh and cannot be made more descriptive. + +[DEFAULT] +openstack_client_http_timeout = 300 + +[auth] +compute_quotas = cores:-1,ram:-1 + +[identity] +v3_endpoint_type = publicURL + +[compute] +min_compute_nodes = 2 +min_microversion = 2.1 +max_microversion = 2.96 + +[service-clients] +http_timeout = 600 + +[compute-feature-enabled] +resize = true +live_migration = true +block_migration_for_live_migration = false +volume_backed_live_migration = true +console_output = true + +[volume] +storage_protocol = ceph +build_timeout = 600 +min_microversion = 3.0 +max_microversion = 3.71 + +[image] +build_timeout = 600 + +[dashboard] +dashboard_url = https://192.168.39.2 + +[load_balancer] +# The legacy 'octavia' alias is absent from octavia_provider_drivers. +provider = amphora +# This environment has no shared IPv6 subnet pool. +test_with_ipv6 = False +# Must outlast Octavia's own amp_active_retries * amp_active_wait_sec of 900s. +lb_build_timeout = 1200 +# Listener and pool waits. +build_timeout = 900 +# Member add waits. +check_timeout = 600 +# With port_security below, makes the plugin attach a security group to backends. +enable_security_groups = True + +[network-feature-enabled] +port_security = True diff --git a/.gitignore b/.gitignore index b2b5221ca0..e797cd3689 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.client.glance.keyring # Ignore generated Octavia certificates in the AIO environments etc/kayobe/environments/aio/kolla/config/octavia/*.pem etc/kayobe/environments/ci-aio/kolla/config/octavia/*.pem +etc/kayobe/environments/ci-multinode/kolla/config/octavia/*.pem # Tempest logs tempest-artifacts diff --git a/doc/source/operations/octavia.rst b/doc/source/operations/octavia.rst index b404801506..d10b7b4f5c 100644 --- a/doc/source/operations/octavia.rst +++ b/doc/source/operations/octavia.rst @@ -2,6 +2,30 @@ Octavia ======= +Management network requirements +=============================== + +The Amphora provider driver runs each load balancer inside its own virtual +machine, called an amphora. The Octavia services and the amphorae talk to each +other directly, so they need to share a network: + +* Octavia connects to each amphora on TCP port 9443 to configure it. +* Each amphora sends health reports back to Octavia on UDP port 5555. + +Octavia therefore needs an address of its own on the management network, and +Neutron needs a network on the same segment to attach the amphorae to. + +If both of those are carried by the same network interface on the host, that +interface must be a Linux bridge. When a plain interface has VLAN subinterfaces +on it, the host keeps the traffic for itself and Open vSwitch never receives it, +so amphorae start up but Octavia cannot reach them. A Linux bridge passes a copy +of the traffic to everything attached to it, so the host and Open vSwitch both +receive it. + +The ``ci-multinode`` environment is set up this way: the VXLAN mesh device sits +inside a Linux bridge, and Open vSwitch attaches to the same bridge through a +veth pair. + .. _Amphora image: Updating amphora images diff --git a/etc/kayobe/ansible/tools/configure-vxlan.yml b/etc/kayobe/ansible/tools/configure-vxlan.yml index 0101b1ce1d..831cce7ccc 100644 --- a/etc/kayobe/ansible/tools/configure-vxlan.yml +++ b/etc/kayobe/ansible/tools/configure-vxlan.yml @@ -9,5 +9,25 @@ ansible_python_interpreter: /usr/bin/python3 # Work around no known_hosts entry on first boot. ansible_ssh_common_args: -o StrictHostKeyChecking=no + pre_tasks: + # stackhpc.vxlan connects VXLAN to a master when an interface + # is of the 'bridge' type, but Network Manager cannot attach the device + # before the bridge exists. Kayobe creates the bridge later, so we will create it here + # and have Kayobe overwrite it later as needed + - name: Get existing NetworkManager connections + ansible.builtin.command: nmcli -g NAME connection show + become: true + changed_when: false + register: vxlan_nm_connections + + - name: Ensure a bridge exists for each VXLAN interface that needs one + ansible.builtin.command: + cmd: >- + nmcli connection add type bridge + con-name {{ item.bridge }} ifname {{ item.bridge }} + stp no ipv4.method disabled ipv6.method disabled + become: true + loop: "{{ vxlan_interfaces | selectattr('bridge', 'defined') | list }}" + when: item.bridge not in vxlan_nm_connections.stdout_lines roles: - role: stackhpc.vxlan diff --git a/etc/kayobe/environments/ci-multinode/controllers.yml b/etc/kayobe/environments/ci-multinode/controllers.yml index 173bcc3714..af763c9eee 100644 --- a/etc/kayobe/environments/ci-multinode/controllers.yml +++ b/etc/kayobe/environments/ci-multinode/controllers.yml @@ -5,6 +5,10 @@ controller_bootstrap_user: "{{ os_distribution if os_distribution == 'ubuntu' el controller_lvm_groups: - "{{ stackhpc_lvm_group_rootvg }}" +# Octavia management network, for the health manager to receive amphora +# heartbeats. OVN provider driver needs no management network. +controller_extra_network_interfaces: "{{ ['lb_mgmt'] if octavia_amphora_driver_enabled | bool else [] }}" + ############################################################################### # Controller node firewalld configuration. diff --git a/etc/kayobe/environments/ci-multinode/globals.yml b/etc/kayobe/environments/ci-multinode/globals.yml index e23f8111ce..c9cf83b713 100644 --- a/etc/kayobe/environments/ci-multinode/globals.yml +++ b/etc/kayobe/environments/ci-multinode/globals.yml @@ -60,9 +60,20 @@ os_release: >- stackhpc_write_barbican_role_id_to_file: true stackhpc_barbican_role_id_file_path: "/tmp/barbican-role-id" +# Trust store for Kayobe's own playbooks. The Octavia amphora image +# register playbook cannot verify Glance's TLS certificate without this. +openstack_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution == 'rocky' else '/etc/ssl/certs/ca-certificates.crt' }}" + # Enable rebooting to update SELinux state selinux_do_reboot: true +############################################################################### +# Octavia. + +# Use Octavia's amphora provider driver instead of the OVN one. Requires +# nested virtualisation on the compute hosts, which the CI cloud (Leaf Cloud as of now) does not have. +octavia_amphora_driver_enabled: false + ############################################################################### # Extra vars. diff --git a/etc/kayobe/environments/ci-multinode/hooks/overcloud-host-configure/post.d/10-check-kvm.yml b/etc/kayobe/environments/ci-multinode/hooks/overcloud-host-configure/post.d/10-check-kvm.yml new file mode 100644 index 0000000000..890c9d60a3 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/hooks/overcloud-host-configure/post.d/10-check-kvm.yml @@ -0,0 +1,31 @@ +--- + +- hosts: compute + gather_facts: true + tasks: + - name: Assert that the host architecture supports the amphora driver + assert: + that: ansible_facts.architecture == 'x86_64' + fail_msg: >- + The Octavia amphora driver is only supported on x86_64 hosts, because + the amphora image is built for x86_64. This host is + {{ ansible_facts.architecture }}. Set octavia_amphora_driver_enabled + to false to deploy with the OVN provider driver instead. + when: octavia_amphora_driver_enabled | bool + + - name: Check whether /dev/kvm exists + stat: + path: /dev/kvm + register: kvm_device + when: octavia_amphora_driver_enabled | bool + + - name: Assert that KVM is available + assert: + that: kvm_device.stat.exists + fail_msg: >- + /dev/kvm is not present. The ci-multinode environment uses KVM when + the Octavia amphora driver is in use, because amphorae are unusable + under emulation. Enable nested virtualisation on this host, or set + octavia_amphora_driver_enabled to false to deploy with the OVN + provider driver. + when: octavia_amphora_driver_enabled | bool diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml index fe92c42a09..ca788fc3e2 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/all/vxlan.yml @@ -16,3 +16,6 @@ vxlan_vni: vxlan_interfaces: - device: "vxlan{{ vxlan_vni }}" group: "{{ '239.0.0.0/8' | ansible.utils.next_nth_usable(vxlan_vni) }}" + # Enslave the mesh to the bridge here, so the role owns the one connection + # profile for this device and Kayobe does not overwrite it as an ethernet. + bridge: br-vxlan diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces index 96c6f15c35..8a62a0d063 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/compute/network-interfaces @@ -4,11 +4,15 @@ admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}" -internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}" +# Linux bridge containing the VXLAN mesh device. Open vSwitch reaches the mesh +# through a veth pair from this bridge, so the host and Neutron both get frames. +vxlan_mesh_interface: br-vxlan -storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}" +internal_interface: "{{ vxlan_mesh_interface }}.{{ internal_vlan }}" -tunnel_interface: "{{ vxlan_interfaces[0].device }}.{{ tunnel_vlan }}" +storage_interface: "{{ vxlan_mesh_interface }}.{{ storage_vlan }}" + +tunnel_interface: "{{ vxlan_mesh_interface }}.{{ tunnel_vlan }}" ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces index 7df8bec280..3e3b83ffe1 100644 --- a/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces +++ b/etc/kayobe/environments/ci-multinode/inventory/group_vars/controllers/network-interfaces @@ -4,13 +4,22 @@ admin_oc_interface: "{{ ansible_facts.default_ipv4.interface }}" -internal_interface: "{{ vxlan_interfaces[0].device }}.{{ internal_vlan }}" +# Linux bridge containing the VXLAN mesh device. Open vSwitch reaches the mesh +# through a veth pair from this bridge, so the host and Neutron both get frames. +vxlan_mesh_interface: br-vxlan -storage_interface: "{{ vxlan_interfaces[0].device }}.{{ storage_vlan }}" +internal_interface: "{{ vxlan_mesh_interface }}.{{ internal_vlan }}" -tunnel_interface: "{{ vxlan_interfaces[0].device }}.{{ tunnel_vlan }}" +storage_interface: "{{ vxlan_mesh_interface }}.{{ storage_vlan }}" -public_interface: "{{ vxlan_interfaces[0].device }}.{{ public_vlan }}" +tunnel_interface: "{{ vxlan_mesh_interface }}.{{ tunnel_vlan }}" + +public_interface: "{{ vxlan_mesh_interface }}.{{ public_vlan }}" + +# Octavia management network. Bridge as interface instead of mesh device +# Open vSwitch will still see frames the host has not claimed due to +# bridge's broadcast +lb_mgmt_interface: "{{ vxlan_mesh_interface }}.{{ lb_mgmt_vlan }}" public_routes: - cidr: "{{ external_cidr }}" diff --git a/etc/kayobe/environments/ci-multinode/kolla/config/octavia.conf b/etc/kayobe/environments/ci-multinode/kolla/config/octavia.conf new file mode 100644 index 0000000000..b3ba8ce094 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/kolla/config/octavia.conf @@ -0,0 +1,5 @@ +# Amphorae boot from Ceph-backed Cinder volumes here, which can be slow. +# Increase timeout from 5 minutes to 15 minutes for Nova to report them active. +[controller_worker] +amp_active_retries = 90 +amp_active_wait_sec = 10 diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index fd57398aed..277d41974f 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -17,9 +17,11 @@ rabbitmq_enable_tls: "yes" ############################################################################ # Most development environments will use nested virtualisation, and we can't -# guarantee that nested KVM support is available. Use QEMU as a lowest common -# denominator. -nova_compute_virt_type: qemu +# guarantee that nested KVM support is available, so QEMU is the lowest common +# denominator. However, amphorae are unusable under emulation in terms of time +# during tempest testing and Octavia times out waiting for them to spin up +# we require KVM when the amphora driver is in use +nova_compute_virt_type: "{{ 'kvm' if octavia_amphora_driver_enabled | bool else 'qemu' }}" # Reduce the control plane's memory footprint by limiting the number of worker # processes to two per-service when running in a VM. @@ -35,22 +37,36 @@ cinder_backend_ceph: "yes" # Nova Ceph configuration nova_backend_ceph: "yes" -# Because this is a vxlan interface, Kayobe can't generate the config for it. -# Therefore we can't use this interface as the external network -# (external_net_names). Instead, make external_net_names an empty -# list, and add the config that would be generated from it here. -neutron_bridge_name: "{{ vxlan_interfaces[0].device }}-ovs" -neutron_external_interface: "{{ vxlan_interfaces[0].device }}" - neutron_dns_domain: "{{ root_domain }}." # OpenSearch memory tuning opensearch_heap_size: 1g -# Octavia load balancer configuration -octavia_auto_configure: "no" -octavia_provider_drivers: "ovn:OVN provider" -octavia_provider_agents: "ovn" +# Octavia load balancer configuration. + +octavia_provider_drivers: "{{ 'amphora:Amphora provider' if octavia_amphora_driver_enabled | bool else 'ovn:OVN provider' }}" +octavia_provider_agents: "{{ 'amphora_agent' if octavia_amphora_driver_enabled | bool else 'ovn' }}" + +# Host interface the health manager binds to. octavia.conf references this +# unconditionally, so fall back to the internal network on OVN, which is what +# kolla's api_interface default resolves to. +octavia_network_interface: "{{ (octavia_net_name if octavia_amphora_driver_enabled | bool else internal_net_name) | net_interface(groups['controllers'][0]) }}" + +# Management network registered by octavia_auto_configure, so only with amphora. +octavia_amp_network: + name: lb-mgmt-net + provider_network_type: vlan + provider_segmentation_id: "{{ lb_mgmt_vlan }}" + provider_physical_network: physnet1 + external: false + shared: false + subnet: + name: lb-mgmt-subnet + cidr: "{{ lb_mgmt_cidr }}" + allocation_pool_start: "{{ lb_mgmt_neutron_allocation_pool_start }}" + allocation_pool_end: "{{ lb_mgmt_neutron_allocation_pool_end }}" + no_gateway_ip: true + enable_dhcp: true # Manila CephFS configuration manila_cephfs_filesystem_name: manila-cephfs diff --git a/etc/kayobe/environments/ci-multinode/networks.yml b/etc/kayobe/environments/ci-multinode/networks.yml index 59d3760b9d..82d7608057 100644 --- a/etc/kayobe/environments/ci-multinode/networks.yml +++ b/etc/kayobe/environments/ci-multinode/networks.yml @@ -38,7 +38,8 @@ internal_net_name: internal # If external_net_name is defined, external_net_names will default to a list # containing one item, external_net_name. #external_net_names: -external_net_names: [] +external_net_names: + - vxlan_mesh # Name of the network used to expose the public OpenStack API endpoints. public_net_name: public @@ -68,6 +69,9 @@ storage_mgmt_net_name: storage_mgmt # hosts #cleaning_net_name: +# Name of the network used to manage octavia loadbalancers +octavia_net_name: lb_mgmt + ############################################################################### # Network definitions. @@ -131,6 +135,26 @@ provision_oc_allocation_pool_end: 192.168.33.254 provision_oc_vlan: 107 provision_oc_zone: "provision_oc" +# Octavia load balancer management network. VLAN provider network on physnet1 +# reusing the existing VXLAN mesh +lb_mgmt_cidr: 192.168.34.0/24 +lb_mgmt_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" +lb_mgmt_vlan: 108 +lb_mgmt_allocation_pool_start: 192.168.34.3 +lb_mgmt_allocation_pool_end: 192.168.34.30 +lb_mgmt_neutron_allocation_pool_start: 192.168.34.100 +lb_mgmt_neutron_allocation_pool_end: 192.168.34.200 +lb_mgmt_zone: lb-mgmt + +# Linux bridge carrying the VXLAN mesh. Without the bridge the +# host will claim the packet and drop if not addressed to its listener. +# A bridge is needed to flood the packet to both the OS and the OVS. +# Empty ports because stackhpc.vxlan does the enslaving via its bridge option. +# We dont want MichaelRigart's role to overwrite the Network Manager's VXLAN handling +vxlan_mesh_bridge_ports: [] +vxlan_mesh_bootproto: manual +vxlan_mesh_mtu: "{{ ansible_facts.default_ipv4.mtu - 50 }}" + ############################################################################### # Network virtual patch link configuration. diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index 7e4d3323b7..57d6eb88f6 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -64,5 +64,8 @@ stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?: stackhpc_docker_registry_username: "{{ stackhpc_repo_mirror_username }}" stackhpc_docker_registry_password: "{{ stackhpc_repo_mirror_password }}" +# Credentials for downloading the Octavia amphora image from Ark +stackhpc_release_pulp_username: "{{ stackhpc_repo_mirror_username }}" +stackhpc_release_pulp_password: "{{ stackhpc_repo_mirror_password }}" # Path to a CA certificate file to trust stackhpc_cloud_tests_openstack_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/{{ stackhpc_ca_secret_store }}.crt"