diff --git a/README.md b/README.md index f27368a..a63ff35 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,17 @@ openstack-nix.nixosModules There are modules to setup the OpenStack controller, an OpenStack compute node and modules simplifying the creation of new NixOS tests. +#### Designate DNS backend + +Enabling `designate.enable` requires a separately configured BIND server. +The Designate module manages the OpenStack DNS services and their pool +configuration; the deployment owns BIND, its RNDC credentials, and DNS access. + +The default pool expects BIND on the controller, serving DNS on port 53 and +accepting RNDC on `127.0.0.1:953`. Provision the matching RNDC key at +`/etc/bind/rndc.key` on the controller. Keep the key outside the Nix store; +Designate invokes RNDC through its privileged rootwrap helper. + ### NixOS tests The NixOS tests are located under the `tests` attribute. To execute a NixOS test diff --git a/flake.nix b/flake.nix index 149e67a..7b18cf3 100644 --- a/flake.nix +++ b/flake.nix @@ -65,6 +65,7 @@ python3Packages writeText lib + libredirect ; }; diff --git a/modules/compute/compute.nix b/modules/compute/compute.nix index a085916..02a9868 100644 --- a/modules/compute/compute.nix +++ b/modules/compute/compute.nix @@ -1,9 +1,19 @@ { neutron, nova }: -{ ... }: +{ lib, ... }: { imports = [ + ../generic/global-options.nix ../generic/controller-host-entry.nix (import ./neutron.nix { inherit neutron; }) (import ./nova.nix { inherit nova; }) ]; + + options.openstack.computeIP = lib.mkOption { + type = lib.types.str; + description = '' + IP address of the compute node, used as Nova's my_ip and as the address + reached by the serial-console and VNC proxies. This address must already + be configured on the host; this option does not configure an interface. + ''; + }; } diff --git a/modules/compute/neutron.nix b/modules/compute/neutron.nix index f9327bc..0e93af0 100644 --- a/modules/compute/neutron.nix +++ b/modules/compute/neutron.nix @@ -3,6 +3,7 @@ config, lib, pkgs, + utils, ... }: @@ -10,6 +11,7 @@ with lib; let cfg = config.neutron; + providerDevice = "${utils.escapeSystemdPath "/sys/subsystem/net/devices/${cfg.providerInterface}"}.device"; neutron_env = pkgs.python3.buildEnv.override { extraLibs = [ neutron ]; }; @@ -25,7 +27,7 @@ let debug = false # File name for the paste.deploy config for api service (string value) api_paste_config = ${neutron}/etc/neutron/api-paste.ini - transport_url = rabbit://openstack:openstack@controller + transport_url = rabbit://openstack:openstack@${config.openstack.controllerHostname} log_dir = /var/log/neutron [agent] @@ -50,6 +52,30 @@ let filterPath = "/etc/neutron/rootwrap.d"; inherit utils_env; }; + + openvswitchPrepareScript = pkgs.writeShellScript "openvswitch-setup.sh" '' + export PATH=${ + lib.makeBinPath [ + pkgs.openvswitch + ] + }:$PATH + + ovs-vsctl br-exists br-provider >/dev/null + status=$? + + if [[ $status -eq 0 ]]; then + echo "br-provider already exists. No further setup tasks." + else + echo "br-provider didn't exist already. Proceed with basic setup." + set -euxo pipefail + ovs-vsctl add-br br-provider + fi + ovs-vsctl --may-exist add-port br-provider ${cfg.providerInterface} + + # enable uplink provider interface + ip link set dev ${cfg.providerInterface} up + ''; + in { options.neutron = { @@ -68,6 +94,13 @@ in The Neutron OpenVSwitch config. ''; }; + openvswitchPrepare = mkOption { + default = openvswitchPrepareScript; + description = '' + Default OpenVswitch prepare script of systemd unit: neutron-openvswitch-agent.service + This prepare script creates the basic openvswitch setup. + ''; + }; providerInterface = mkOption { default = "eth2"; type = types.str; @@ -78,7 +111,7 @@ in }; }; - config = mkIf cfg.enable { + config = { users.extraUsers.neutron = { group = "neutron"; isSystemUser = true; @@ -136,6 +169,14 @@ in after = [ "network.target" "ovsdb.service" + "ovs-vswitchd.service" + providerDevice + "systemd-networkd.service" + ]; + # Bridge setup needs both the switching daemon and the provider NIC. + requires = [ + "ovs-vswitchd.service" + providerDevice ]; wantedBy = [ "multi-user.target" ]; path = with pkgs; [ @@ -150,14 +191,12 @@ in conntrack-tools ]; serviceConfig = { - ExecStartPre = pkgs.writeShellScript "neutron-openvswitch-agent-pre.sh" '' - ${pkgs.openvswitch}/bin/ovs-vsctl add-br br-provider - ${pkgs.openvswitch}/bin/ovs-vsctl add-port br-provider ${cfg.providerInterface} - ''; + ExecStartPre = "${cfg.openvswitchPrepare}"; ExecStart = pkgs.writeShellScript "neutron-openvswitch-agent.sh" '' ${neutron}/bin/neutron-openvswitch-agent --config-file=${cfg.config} --config-file=${cfg.openvswitchConfig} ''; }; + enable = cfg.enable; }; }; } diff --git a/modules/compute/nova.nix b/modules/compute/nova.nix index 9d0b0d1..d43fb43 100644 --- a/modules/compute/nova.nix +++ b/modules/compute/nova.nix @@ -25,8 +25,8 @@ let state_path = /var/lib/nova rootwrap_config = ${rootwrapConf} compute_driver = libvirt.LibvirtDriver - my_ip = 10.0.0.39 - transport_url = rabbit://openstack:openstack@controller + my_ip = ${config.openstack.computeIP} + transport_url = rabbit://openstack:openstack@${config.openstack.controllerHostname} [api] auth_strategy = keystone @@ -38,12 +38,12 @@ let connection = sqlite:////var/lib/nova/nova.sqlite [glance] - api_servers = http://controller:9292 + api_servers = http://${config.openstack.controllerHostname}:9292 [keystone_authtoken] - www_authenticate_uri = http://controller:5000/ - auth_url = http://controller:5000/ - memcached_servers = controller:11211 + www_authenticate_uri = http://${config.openstack.controllerHostname}:5000/v3 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 + memcached_servers = ${config.openstack.controllerHostname}:11211 auth_type = password project_domain_name = Default user_domain_name = Default @@ -53,9 +53,12 @@ let [libvirt] virt_type = kvm + ${optionalString (config.openstack.live_migration_inbound_addr != null) '' + live_migration_inbound_addr = ${config.openstack.live_migration_inbound_addr} + ''} [neutron] - auth_url = http://controller:5000 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 auth_type = password project_domain_name = Default user_domain_name = Default @@ -76,13 +79,13 @@ let project_name = service auth_type = password user_domain_name = Default - auth_url = http://controller:5000/v3 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 username = placement password = placement [service_user] send_service_user_token = true - auth_url = http://controller:5000/ + auth_url = http://${config.openstack.controllerHostname}:5000/v3 auth_strategy = keystone auth_type = password project_domain_name = Default @@ -146,7 +149,7 @@ in }; }; - config = mkIf cfg.enable { + config = { users.extraUsers.nova = { group = "nova"; isSystemUser = true; @@ -162,28 +165,28 @@ in systemd.tmpfiles.settings = { "10-nova" = { "/var/log/nova" = { - D = { + d = { group = "nova"; mode = "0755"; user = "nova"; }; }; "/var/lock/nova" = { - D = { + d = { group = "nova"; mode = "0755"; user = "nova"; }; }; "/var/lib/nova" = { - D = { + d = { group = "nova"; mode = "0755"; user = "nova"; }; }; "/var/lib/nova/instances" = { - D = { + d = { group = "nova"; mode = "0755"; user = "nova"; @@ -194,7 +197,7 @@ in }; services.openiscsi = { - enable = true; + enable = cfg.enable; name = "iqn.iscsi.${config.networking.hostName}"; }; @@ -220,6 +223,7 @@ in lvm2 openiscsi nfs-utils + e2fsprogs ] ++ cfg.extraPkgs; environment.PYTHONPATH = "${nova_env}/${pkgs.python3.sitePackages}"; @@ -228,6 +232,7 @@ in ${cfg.novaPackage}/bin/nova-compute --config-file=${cfg.config} ''; }; + enable = cfg.enable; }; }; } diff --git a/modules/controller/cinder.nix b/modules/controller/cinder.nix index 55b31a7..5bd3c4c 100644 --- a/modules/controller/cinder.nix +++ b/modules/controller/cinder.nix @@ -11,20 +11,20 @@ with lib; let cfg = config.cinder; - cinderConf = pkgs.writeText "cinder-api.conf" '' + cinderConf = pkgs.writeText "cinder.conf" '' [DEFAULT] - transport_url = rabbit://openstack:openstack@controller + transport_url = rabbit://openstack:openstack@${config.openstack.controllerHostname} auth_strategy = keystone - my_ip = controller + my_ip = ${config.openstack.controllerIP} verify_glance_signatures = disabled [database] - connection = mysql+pymysql://cinder:cinder@controller/cinder + connection = mysql+pymysql://cinder:cinder@${config.openstack.controllerHostname}/cinder [keystone_authtoken] - www_authenticate_uri = http://controller:5000 - auth_url = http://controller:5000 - memcached_servers = controller:11211 + www_authenticate_uri = http://${config.openstack.controllerHostname}:5000/v3 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 + memcached_servers = ${config.openstack.controllerHostname}:11211 auth_type = password project_domain_name = default user_domain_name = default @@ -56,8 +56,26 @@ in The OpenStack Cinder package to use. ''; }; + envCinderApi = mkOption { + type = types.listOf types.str; + default = [ + "PYTHONWARNINGS=ignore::DeprecationWarning" + "PATH=$PATH:/run/current-system/sw/bin" + ]; + description = '' + Environment variables passed to the cinder-api uWSGI vassal. + ''; + }; + envCinderScheduler = mkOption { + default = { + PYTHONWARNINGS = "ignore::DeprecationWarning"; + }; + description = '' + Environment variables passed to the cinder-scheduler systemd unit. + ''; + }; }; - config = mkIf cfg.enable { + config = { users.extraUsers.cinder = { group = "cinder"; @@ -71,21 +89,21 @@ in systemd.tmpfiles.settings = { "10-cinder" = { "/var/lib/cinder/" = { - D = { + d = { user = "cinder"; group = "cinder"; mode = "0755"; }; }; "/var/lib/cinder/volumes" = { - D = { + d = { user = "cinder"; group = "cinder"; mode = "0755"; }; }; "/var/log/cinder/" = { - D = { + d = { user = "cinder"; group = "cinder"; mode = "0755"; @@ -98,28 +116,51 @@ in }; "/etc/cinder/cinder.conf" = { L = { - argument = "${cinderConf}"; + argument = "${cfg.config}"; }; }; }; }; - systemd.services.cinder-api = { - description = "OpenStack Cinder API Daemon"; - after = [ - "cinder.service" - "rabbitmq.service" - "mysql.service" - "network.target" - ]; - path = [ cinder ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - User = "cinder"; - Group = "cinder"; - ExecStart = pkgs.writeShellScript "cinder-api.sh" '' - .cinder-wsgi-wrapped --port 8776 - ''; + services.uwsgi = { + instance.vassals = optionalAttrs cfg.enable { + cinder-api = { + type = "normal"; + http-socket = "0.0.0.0:8776"; + wsgi-file = "${cinder}/bin/.cinder-wsgi-wrapped"; + pyargv = "--config-file ${cfg.config}"; + env = cfg.envCinderApi; + + master = true; + processes = 4; + enable-threads = true; + thunder-lock = true; + lazy-apps = true; + die-on-term = true; + vacuum = true; + need-app = true; + buffer-size = 65535; + + immediate-uid = "cinder"; + immediate-gid = "cinder"; + } + // optionalAttrs (!config.openstack.manual_setup) { + # Keystone must be available in the same emperor before automatic + # Cinder setup can finish, so only this vassal waits for the schema. + # As there is no uwsgi mechanism to serialize the vassals, we use a + # marker file created in the systemd service setup of cinder to wait + # for first. + exec-asap = pkgs.writeShellScript "cinder-api-wait-for-setup" '' + for ((attempt = 0; attempt < 300; attempt++)); do + if test -f /run/cinder-setup/ready; then + exit 0 + fi + ${pkgs.coreutils}/bin/sleep 1 + done + echo "Timed out waiting for Cinder database setup" >&2 + exit 1 + ''; + }; }; }; @@ -137,9 +178,11 @@ in User = "cinder"; Group = "cinder"; ExecStart = pkgs.writeShellScript "cinder-scheduler.sh" '' - .cinder-scheduler-wrapped + .cinder-scheduler-wrapped --config-file ${cfg.config} ''; }; + enable = cfg.enable; + environment = cfg.envCinderScheduler; }; }; } diff --git a/modules/controller/designate.nix b/modules/controller/designate.nix new file mode 100644 index 0000000..9638a9b --- /dev/null +++ b/modules/controller/designate.nix @@ -0,0 +1,243 @@ +{ designate }: +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.designate; + + designateConf = pkgs.writeText "designate.conf" '' + [DEFAULT] + transport_url = rabbit://openstack:openstack@${config.openstack.controllerHostname} + auth_strategy = keystone + log_dir = /var/log/designate + root_helper = "/run/wrappers/bin/sudo ${designate}/bin/designate-rootwrap ${rootwrapConf}" + + [storage:sqlalchemy] + connection = mysql+pymysql://designate:designate@${config.openstack.controllerHostname}/designate + + [service:api] + listen = 0.0.0.0:9001 + api_base_uri = http://${config.openstack.controllerHostname}:9001/ + api_paste_config = ${designate}/etc/designate/api-paste.ini + auth_strategy = keystone + enable_api_v2 = true + enable_api_admin = true + enable_host_header = true + enabled_extensions_admin = quotas,reports + + [service:mdns] + listen = 0.0.0.0:5354 + + [service:worker] + threads = 20 + + [keystone_authtoken] + www_authenticate_uri = http://${config.openstack.controllerHostname}:5000/v3 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 + memcached_servers = ${config.openstack.controllerHostname}:11211 + auth_type = password + project_domain_name = Default + user_domain_name = Default + project_name = service + username = designate + password = designate + + [oslo_concurrency] + lock_path = /var/lib/designate/tmp + ''; + + # BIND and its RNDC key must be provisioned by the deployment. This module + # only configures Designate's connection to that independently managed server. + designatePools = pkgs.writeText "pools.yaml" '' + - name: default + description: Designate bind backend pool + attributes: {} + + ns_records: + - hostname: osdns.openstack.test. + priority: 2 + - hostname: controller.openstack.test. + priority: 1 + + # List out the nameservers for this pool. These are the actual BIND servers. + # We use these to verify changes have propagated to all nameservers. + nameservers: + # controller bind + - host: ${config.openstack.controllerIP} + port: 53 + # dedicated dns server knot (replace it later) + - host: ${cfg.knot.address} + port: 53 + + # List out the targets for this pool. For BIND, most often, there will be one + # entry for each BIND server. + targets: + - type: bind9 + description: local bind server on controller + + # List out the designate-mdns servers from which BIND servers should + # request zone transfers (AXFRs) from. + masters: + # IP address of controller + - host: ${config.openstack.controllerIP} + port: 5354 + + # BIND Configuration options + # in our use case localhost == controller + options: + host: 127.0.0.1 + port: 53 + rndc_host: 127.0.0.1 + rndc_port: 953 + rndc_key_file: /etc/bind/rndc.key + # set relative path so rootwrap works + rndc_bin_path: rndc + + also_notifies: + - host: ${cfg.knot.address} + port: 53 + ''; + + service = command: { + after = [ + "rabbitmq.service" + "mysql.service" + "network.target" + ] + ++ lib.optional (!config.openstack.manual_setup) "designate.service"; + requires = lib.optional (!config.openstack.manual_setup) "designate.service"; + wantedBy = [ "multi-user.target" ]; + path = [ designate ]; + restartTriggers = [ cfg.config ]; + serviceConfig = { + User = "designate"; + Group = "designate"; + ExecStart = "${designate}/bin/${command} --config-file=${cfg.config}"; + Restart = "on-failure"; + }; + }; + + designate_env = pkgs.python3.buildEnv.override { + extraLibs = [ designate ]; + }; + utils_env = pkgs.buildEnv { + name = "utils"; + paths = with pkgs; [ + designate_env + bind + ]; + }; + + rootwrapConf = pkgs.callPackage ../../lib/rootwrap-conf.nix { + package = designate_env; + filterPath = "/etc/designate/rootwrap.d"; + inherit utils_env; + }; + +in +{ + options.designate = { + enable = lib.mkEnableOption "OpenStack Designate with an externally configured BIND backend" // { + default = false; + }; + + config = lib.mkOption { + type = lib.types.path; + default = designateConf; + description = "Designate configuration file."; + }; + + pools = lib.mkOption { + type = lib.types.path; + default = designatePools; + description = '' + Designate pools.yaml. The deployment must configure the authoritative + BIND server and provision its RNDC key separately; enabling Designate + does not configure BIND. The default pool expects BIND on the controller, + with DNS on port 53, RNDC on 127.0.0.1:953, and its key available locally + at /etc/bind/rndc.key. It also expects an external Knot secondary at + designate.knot.address. Override this file for a different DNS topology. + ''; + }; + + knot = { + address = lib.mkOption { + type = lib.types.str; + default = "192.168.200.23"; + description = "IP address of the external Knot DNS server."; + }; + }; + }; + + config = lib.mkIf cfg.enable { + users.users.designate = { + group = "designate"; + isSystemUser = true; + }; + users.groups.designate = { }; + + security.sudo.enable = true; + security.sudo.extraConfig = '' + designate ALL = (root) NOPASSWD: ${designate}/bin/designate-rootwrap ${rootwrapConf} * + ''; + + systemd.tmpfiles.settings."10-designate" = { + "/etc/designate/designate.conf"."L+".argument = "${cfg.config}"; + "/etc/designate/pools.yaml"."L+".argument = "${cfg.pools}"; + "/var/lib/designate".d = { + user = "designate"; + group = "designate"; + mode = "0755"; + }; + "/var/lib/designate/.ssh".d = { + user = "designate"; + group = "designate"; + mode = "0755"; + }; + "/var/lib/designate/tmp".D = { + user = "designate"; + group = "designate"; + mode = "0755"; + }; + "/var/log/designate".d = { + user = "designate"; + group = "designate"; + mode = "0755"; + }; + }; + + systemd.services.designate-api = service "designate-api"; + systemd.services.designate-central = service "designate-central"; + systemd.services.designate-mdns = service "designate-mdns"; + systemd.services.designate-producer = service "designate-producer"; + systemd.services.designate-worker = lib.recursiveUpdate (service "designate-worker") { + path = [ + designate + pkgs.openssh + pkgs.sudo + ]; + }; + + networking.firewall.allowedTCPPorts = [ 5354 ]; + networking.firewall.allowedUDPPorts = [ 5354 ]; + + systemd.services.designate-pool-update = { + description = "Update the OpenStack Designate pools"; + after = [ "designate-central.service" ]; + requires = [ "designate-central.service" ]; + wantedBy = [ "multi-user.target" ]; + path = [ designate ]; + restartTriggers = [ cfg.pools ]; + serviceConfig = { + Type = "oneshot"; + User = "designate"; + Group = "designate"; + ExecStart = "${designate}/bin/designate-manage --config-file=${cfg.config} pool update --file=${cfg.pools}"; + }; + }; + }; +} diff --git a/modules/controller/generic.nix b/modules/controller/generic.nix index 615f8ba..e6787c4 100644 --- a/modules/controller/generic.nix +++ b/modules/controller/generic.nix @@ -6,6 +6,11 @@ { services.mysql.enable = true; services.mysql.package = lib.mkDefault pkgs.mariadb; + services.mysql.settings = { + mysqld = { + max_connections = 500; + }; + }; services.rabbitmq = { enable = true; @@ -44,4 +49,9 @@ ]; instance.type = "emperor"; }; + + # Match ExecStop's SIGINT when systemd stops the remaining processes too. + # SIGQUIT can dump core for vassals still starting and their helper commands + # during the uWSGI restart performed by Keystone setup. + systemd.services.uwsgi.serviceConfig.KillSignal = lib.mkForce "SIGINT"; } diff --git a/modules/controller/glance.nix b/modules/controller/glance.nix index ffabe08..3453806 100644 --- a/modules/controller/glance.nix +++ b/modules/controller/glance.nix @@ -17,12 +17,12 @@ let log_dir = /var/log/glance [database] - connection = mysql+pymysql://glance:glance@controller/glance + connection = mysql+pymysql://glance:glance@${config.openstack.controllerHostname}/glance [keystone_authtoken] - www_authenticate_uri = http://controller:5000 - auth_url = http://controller:5000 - memcached_servers = controller:11211 + www_authenticate_uri = http://${config.openstack.controllerHostname}:5000/v3 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 + memcached_servers = ${config.openstack.controllerHostname}:11211 auth_type = password project_domain_name = Default user_domain_name = Default @@ -40,7 +40,7 @@ let filesystem_store_datadir = /var/lib/glance/images/ [oslo_limit] - auth_url = http://controller:5000 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 auth_type = password user_domain_id = default username = glance @@ -61,8 +61,17 @@ in The Glance config. ''; }; + env = mkOption { + type = types.listOf types.str; + default = [ + "PYTHONWARNINGS=ignore::DeprecationWarning" + ]; + description = '' + Environment variables passed to the Keystone uWSGI vassal. + ''; + }; }; - config = mkIf cfg.enable { + config = { users.extraUsers.glance = { group = "glance"; @@ -75,42 +84,42 @@ in systemd.tmpfiles.settings = { "10-glance" = { + "/etc/glance/glance-api.conf" = { + "L+" = { + argument = "${cfg.config}"; + }; + }; + "/etc/glance/glance-api-paste.ini" = { + "L+" = { + argument = "${glance}/etc/glance/glance-api-paste.ini"; + }; + }; + "/etc/glance/schema-image.json" = { + "L+" = { + argument = "${glance}/etc/glance/schema-image.json"; + }; + }; "/var/lib/glance/" = { - D = { + d = { user = "glance"; group = "glance"; mode = "0755"; }; }; "/var/log/glance/" = { - D = { + d = { user = "glance"; group = "glance"; mode = "0755"; }; }; "/var/lib/glance/images" = { - D = { + d = { user = "glance"; group = "glance"; mode = "0755"; }; }; - "/etc/glance/glance-api.conf" = { - L = { - argument = "${cfg.config}"; - }; - }; - "/etc/glance/glance-api-paste.ini" = { - L = { - argument = "${glance}/etc/glance/glance-api-paste.ini"; - }; - }; - "/etc/glance/schema-image.json" = { - L = { - argument = "${glance}/etc/glance/schema-image.json"; - }; - }; }; }; @@ -131,10 +140,11 @@ in glance-api --config-file=${cfg.config} --config-file=/etc/glance/glance-api-paste.ini ''; }; + enable = cfg.enable; }; services.uwsgi = { - instance.vassals.glance = { + instance.vassals.glance = mkIf cfg.enable { socket-timeout = 10; http-auto-chunked = true; http-chunked-input = true; @@ -154,6 +164,7 @@ in immediate-uid = "glance"; immediate-gid = "glance"; wsgi-file = "${glance}/bin/.glance-wsgi-api-wrapped"; + env = cfg.env; }; }; }; diff --git a/modules/controller/horizon.nix b/modules/controller/horizon.nix index 90b0898..9fc5d3e 100644 --- a/modules/controller/horizon.nix +++ b/modules/controller/horizon.nix @@ -30,6 +30,15 @@ in description = "The Horizon Package to use"; type = types.package; }; + env = mkOption { + type = types.listOf types.str; + default = [ + "PYTHONWARNINGS=ignore::DeprecationWarning" + ]; + description = '' + Environment variables passed to the horizon uWSGI vassal. + ''; + }; }; config = { @@ -89,6 +98,7 @@ in thunder-lock = true; lazy-apps = true; chdir = "/var/lib/openstack_dashboard"; + env = cfg.env; }; }; diff --git a/modules/controller/keystone.nix b/modules/controller/keystone.nix index 9315577..2d533d8 100644 --- a/modules/controller/keystone.nix +++ b/modules/controller/keystone.nix @@ -14,43 +14,53 @@ let serviceEndPointTemplateConf = pkgs.writeText "default_catalog.templates" '' # config for templated.Catalog, using camelCase because I don't want to do # translations for keystone compat - catalog.RegionOne.identity.publicURL = http://controller:5000/v3 - catalog.RegionOne.identity.adminURL = http://controller:5000/v3 - catalog.RegionOne.identity.internalURL = http://controller:5000/v3 + catalog.RegionOne.identity.publicURL = http://${config.openstack.controllerHostname}:5000/v3 + catalog.RegionOne.identity.adminURL = http://${config.openstack.controllerHostname}:5000/v3 + catalog.RegionOne.identity.internalURL = http://${config.openstack.controllerHostname}:5000/v3 catalog.RegionOne.identity.name = Identity Service # fake compute service for now to help novaclient tests work - catalog.RegionOne.compute.publicURL = http://controller:8774/v2.1 - catalog.RegionOne.compute.adminURL = http://controller:8774/v2.1 - catalog.RegionOne.compute.internalURL = http://controller:8774/v2.1 + catalog.RegionOne.compute.publicURL = http://${config.openstack.controllerHostname}:8774/v2.1 + catalog.RegionOne.compute.adminURL = http://${config.openstack.controllerHostname}:8774/v2.1 + catalog.RegionOne.compute.internalURL = http://${config.openstack.controllerHostname}:8774/v2.1 catalog.RegionOne.compute.name = Compute Service V2.1 - catalog.RegionOne.image.publicURL = http://controller:9292 - catalog.RegionOne.image.adminURL = http://controller:9292 - catalog.RegionOne.image.internalURL = http://controller:9292 + catalog.RegionOne.image.publicURL = http://${config.openstack.controllerHostname}:9292 + catalog.RegionOne.image.adminURL = http://${config.openstack.controllerHostname}:9292 + catalog.RegionOne.image.internalURL = http://${config.openstack.controllerHostname}:9292 catalog.RegionOne.image.name = Image Service - catalog.RegionOne.network.publicURL = http://controller:9696 - catalog.RegionOne.network.adminURL = http://controller:9696 - catalog.RegionOne.network.internalURL = http://controller:9696 + catalog.RegionOne.network.publicURL = http://${config.openstack.controllerHostname}:9696 + catalog.RegionOne.network.adminURL = http://${config.openstack.controllerHostname}:9696 + catalog.RegionOne.network.internalURL = http://${config.openstack.controllerHostname}:9696 catalog.RegionOne.network.name = Network Service - catalog.RegionOne.placement.publicURL = http://controller:8778 - catalog.RegionOne.placement.adminURL = http://controller:8778 - catalog.RegionOne.placement.internalURL = http://controller:8778 + catalog.RegionOne.placement.publicURL = http://${config.openstack.controllerHostname}:8778 + catalog.RegionOne.placement.adminURL = http://${config.openstack.controllerHostname}:8778 + catalog.RegionOne.placement.internalURL = http://${config.openstack.controllerHostname}:8778 catalog.RegionOne.placement.name = Placement Service - catalog.RegionOne.volumev3.publicURL = http://controller:8776/v3 - catalog.RegionOne.volumev3.adminURL = http://controller:8776/v3 - catalog.RegionOne.volumev3.internalURL = http://controller:8776/v3 + catalog.RegionOne.volumev3.publicURL = http://${config.openstack.controllerHostname}:8776/v3 + catalog.RegionOne.volumev3.adminURL = http://${config.openstack.controllerHostname}:8776/v3 + catalog.RegionOne.volumev3.internalURL = http://${config.openstack.controllerHostname}:8776/v3 catalog.RegionOne.volumev3.name = Cinder Service + + catalog.RegionOne.dns.publicURL = http://${config.openstack.controllerHostname}:9001/ + catalog.RegionOne.dns.adminURL = http://${config.openstack.controllerHostname}:9001/ + catalog.RegionOne.dns.internalURL = http://${config.openstack.controllerHostname}:9001/ + catalog.RegionOne.dns.name = DNS Service ''; keystoneConf = pkgs.writeText "keystone.conf" '' [DEFAULT] log_dir = /var/log/keystone [database] - connection = mysql+pymysql://keystone:keystone@controller/keystone + connection = mysql+pymysql://keystone:keystone@${config.openstack.controllerHostname}/keystone + + [cache] + enabled = true + backend = dogpile.cache.memcached + memcache_servers = 127.0.0.1:11211 [token] provider = fernet @@ -71,9 +81,18 @@ in The Keystone config. ''; }; + env = mkOption { + type = types.listOf types.str; + default = [ + "PYTHONWARNINGS=ignore::DeprecationWarning" + ]; + description = '' + Environment variables passed to the Keystone uWSGI vassal. + ''; + }; }; - config = mkIf cfg.enable { + config = { users.extraUsers.keystone = { group = "keystone"; @@ -87,14 +106,14 @@ in systemd.tmpfiles.settings = { "10-keystone" = { "/var/lib/keystone/" = { - D = { + d = { user = "keystone"; group = "keystone"; mode = "0755"; }; }; "/var/log/keystone/" = { - D = { + d = { user = "keystone"; group = "keystone"; mode = "0755"; @@ -103,7 +122,7 @@ in # Certain executables e.g. keystone-wsgi-public expect the config file # at a default location. "/etc/keystone/keystone.conf" = { - L = { + "L+" = { argument = "${cfg.config}"; }; }; @@ -113,7 +132,7 @@ in services.nginx = { enable = true; virtualHosts = { - controller = { + ${config.openstack.controllerHostname} = { locations."/".proxyPass = "http://127.0.0.1:5001/"; listen = [ { @@ -134,7 +153,7 @@ in ]; instance.type = "emperor"; - instance.vassals.keystone = { + instance.vassals.keystone = mkIf cfg.enable { type = "normal"; http11-socket = "127.0.0.1:5001"; buffer-size = 65535; @@ -148,6 +167,7 @@ in threads = 4; thunder-lock = true; lazy-apps = true; + env = cfg.env; }; }; }; diff --git a/modules/controller/neutron.nix b/modules/controller/neutron.nix index 448d38f..b3c9a5b 100644 --- a/modules/controller/neutron.nix +++ b/modules/controller/neutron.nix @@ -3,6 +3,7 @@ config, lib, pkgs, + utils, ... }: @@ -10,6 +11,7 @@ with lib; let cfg = config.neutron; + providerDevice = "${utils.escapeSystemdPath "/sys/subsystem/net/devices/${cfg.providerInterface}"}.device"; # For live migration to work, neutron-server must allow certain actions. policyConf = pkgs.writeText "policy.yaml" '' @@ -25,24 +27,24 @@ let # neutron.conf is used as configuration file for neutron-metadata-agent as well neutronConf = pkgs.writeText "neutron.conf" '' [database] - connection = mysql+pymysql://neutron:neutron@controller/neutron + connection = mysql+pymysql://neutron:neutron@${config.openstack.controllerHostname}/neutron [DEFAULT] core_plugin = ml2 service_plugins = api_paste_config = ${neutron}/etc/neutron/api-paste.ini - transport_url = rabbit://openstack:openstack@controller + transport_url = rabbit://openstack:openstack@${config.openstack.controllerHostname} auth_strategy = keystone notify_nova_on_port_status_changes = true notify_nova_on_port_data_changes = true log_dir = /var/log/neutron - nova_metadata_host = controller + nova_metadata_host = ${config.openstack.controllerHostname} metadata_proxy_shared_secret = neutron_metadata_secret [keystone_authtoken] - www_authenticate_uri = http://controller:5000 - auth_url = http://controller:5000 - memcached_servers = controller:11211 + www_authenticate_uri = http://${config.openstack.controllerHostname}:5000/v3 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 + memcached_servers = ${config.openstack.controllerHostname}:11211 auth_type = password project_domain_name = Default user_domain_name = Default @@ -57,7 +59,7 @@ let service_token_roles = admin [nova] - auth_url = http://controller:5000 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 auth_type = password project_domain_name = Default user_domain_name = Default @@ -172,7 +174,7 @@ in ''; }; }; - config = mkIf cfg.enable { + config = { users.extraUsers.neutron = { group = "neutron"; @@ -186,53 +188,53 @@ in systemd.tmpfiles.settings = { "10-neutron" = { "/var/log/neutron" = { - D = { + d = { group = "neutron"; mode = "0755"; user = "neutron"; }; }; "/etc/neutron/neutron.conf" = { - L = { + "L+" = { argument = "${cfg.config}"; }; }; "/etc/neutron/plugins/ml2/ml2_conf.ini" = { - L = { + "L+" = { argument = "${cfg.ml2Config}"; }; }; "/etc/neutron/plugins/ml2/openvswitch_agent.ini" = { - L = { + "L+" = { argument = "${cfg.openvswitchConfig}"; }; }; "/etc/neutron/dhcp_agent.ini" = { - L = { + "L+" = { argument = "${cfg.dhcpAgentConfig}"; }; }; "/etc/neutron/api-paste.ini" = { - L = { + "L+" = { argument = "${neutron}/etc/neutron/api-paste.ini"; }; }; "/var/lock/neutron" = { - D = { + d = { group = "neutron"; mode = "0755"; user = "neutron"; }; }; "/var/lib/neutron" = { - D = { + d = { group = "neutron"; mode = "0755"; user = "neutron"; }; }; "/var/lib/neutron/dhcp" = { - D = { + d = { group = "neutron"; mode = "0755"; user = "neutron"; @@ -252,6 +254,7 @@ in serviceConfig = { ExecStart = "${neutron}/bin/neutron-metadata-agent --config-file=${cfg.config}"; }; + enable = cfg.enable; }; virtualisation.vswitch = { @@ -269,8 +272,15 @@ in after = [ "network.target" "ovsdb.service" + "ovs-vswitchd.service" + providerDevice "neutron.service" ]; + # Bridge setup needs both the switching daemon and the provider NIC. + requires = [ + "ovs-vswitchd.service" + providerDevice + ]; wantedBy = [ "multi-user.target" ]; path = with pkgs; [ sudo @@ -286,11 +296,14 @@ in ExecStartPre = pkgs.writeShellScript "pre.sh" '' ${pkgs.openvswitch}/bin/ovs-vsctl add-br br-provider || true ${pkgs.openvswitch}/bin/ovs-vsctl add-port br-provider ${cfg.providerInterface} || true + # enable uplink provider interface + ip link set dev ${cfg.providerInterface} up ''; ExecStart = pkgs.writeShellScript "neutron-openvswitch.sh" '' ${neutron}/bin/neutron-openvswitch-agent --config-file=${cfg.config} --config-file=${cfg.openvswitchConfig} ''; }; + enable = cfg.enable; }; systemd.services.neutron-server = { @@ -321,6 +334,7 @@ in LimitNOFILE = 65535; TimeoutStopSec = 15; }; + enable = cfg.enable; }; systemd.services.neutron-dhcp-agent = { @@ -352,6 +366,7 @@ in LimitNOFILE = 65535; TimeoutStopSec = 15; }; + enable = cfg.enable; }; }; diff --git a/modules/controller/nova.nix b/modules/controller/nova.nix index 1ea834b..81825a1 100644 --- a/modules/controller/nova.nix +++ b/modules/controller/nova.nix @@ -13,14 +13,14 @@ let novaConf = pkgs.writeText "nova.conf" '' [api_database] - connection = mysql+pymysql://nova:nova@controller/nova_api + connection = mysql+pymysql://nova:nova@${config.openstack.controllerHostname}/nova_api [database] - connection = mysql+pymysql://nova:nova@controller/nova + connection = mysql+pymysql://nova:nova@${config.openstack.controllerHostname}/nova [DEFAULT] - transport_url = rabbit://openstack:openstack@controller:5672/ - my_ip = 10.0.0.11 + transport_url = rabbit://openstack:openstack@${config.openstack.controllerHostname}:5672/ + my_ip = ${config.openstack.controllerIP} log_dir = /var/log/nova lock_path = /var/lock/nova state_path = /var/lib/nova @@ -29,9 +29,9 @@ let auth_strategy = keystone [keystone_authtoken] - www_authenticate_uri = http://controller:5000/ - auth_url = http://controller:5000/ - memcached_servers = controller:11211 + www_authenticate_uri = http://${config.openstack.controllerHostname}:5000/v3 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 + memcached_servers = ${config.openstack.controllerHostname}:11211 auth_type = password project_domain_name = Default user_domain_name = Default @@ -41,7 +41,7 @@ let [service_user] send_service_user_token = true - auth_url = http://controller:5000/ + auth_url = http://${config.openstack.controllerHostname}:5000/v3 auth_strategy = keystone auth_type = password project_domain_name = Default @@ -59,7 +59,7 @@ let server_proxyclient_address = $my_ip [glance] - api_servers = http://controller:9292 + api_servers = http://${config.openstack.controllerHostname}:9292 [oslo_concurrency] lock_path = /var/lib/nova/tmp @@ -69,7 +69,7 @@ let project_name = service auth_type = password user_domain_name = Default - auth_url = http://controller:5000/v3 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 username = placement password = placement @@ -77,7 +77,7 @@ let discover_hosts_in_cells_interval = 300 [neutron] - auth_url = http://controller:5000 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 auth_type = password project_domain_name = Default user_domain_name = Default @@ -108,7 +108,7 @@ in ''; }; }; - config = mkIf cfg.enable { + config = { users.extraUsers.nova = { group = "nova"; @@ -122,12 +122,12 @@ in systemd.tmpfiles.settings = { "10-nova" = { "/etc/nova/nova.conf" = { - L = { + "L+" = { argument = "${cfg.config}"; }; }; "/etc/nova/api-paste.ini" = { - L = { + "L+" = { argument = "${cfg.novaPackage}/etc/nova/api-paste.ini"; }; }; @@ -153,7 +153,7 @@ in }; }; "/usr/share/novnc" = { - L = { + "L+" = { argument = "${pkgs.novnc}/share/webapps/novnc"; }; }; @@ -184,6 +184,7 @@ in nova-api --config-file ${cfg.config} ''; }; + enable = cfg.enable; }; systemd.services.nova-conductor = { @@ -192,7 +193,7 @@ in "neutron-server.service" "nova.service" "glance-api.service" - "placement-api.service" + "uwsgi.service" "postgresql.service" "mysql.service" "keystone.service" @@ -216,6 +217,7 @@ in LimitNOFILE = 65535; TimeoutStopSec = 15; }; + enable = cfg.enable; }; systemd.services.nova-scheduler = { @@ -223,7 +225,7 @@ in after = [ "neutron-server.service" "glance-api.service" - "placement-api.service" + "uwsgi.service" "nova.service" "postgresql.service" "mysql.service" @@ -248,6 +250,7 @@ in LimitNOFILE = 65535; TimeoutStopSec = 15; }; + enable = cfg.enable; }; systemd.services.nova-host-discovery = { @@ -269,6 +272,7 @@ in nova-manage cell_v2 discover_hosts --verbose ''; }; + enable = cfg.enable; }; systemd.services.nova-novncproxy = { @@ -276,7 +280,7 @@ in after = [ "neutron-server.service" "glance-api.service" - "placement-api.service" + "uwsgi.service" "nova.service" "postgresql.service" "mysql.service" @@ -301,6 +305,7 @@ in LimitNOFILE = 65535; TimeoutStopSec = 15; }; + enable = cfg.enable; }; systemd.services.nova-serialproxy = { @@ -326,6 +331,7 @@ in LimitNOFILE = 65535; TimeoutStopSec = 15; }; + enable = cfg.enable; }; }; } diff --git a/modules/controller/openstack-controller.nix b/modules/controller/openstack-controller.nix index 3861fe7..0ef2cbd 100644 --- a/modules/controller/openstack-controller.nix +++ b/modules/controller/openstack-controller.nix @@ -6,9 +6,12 @@ placement, horizon, cinder, + python-openstackclient, + designate, }: { config, + lib, pkgs, ... }: @@ -19,12 +22,284 @@ let OS_PROJECT_NAME = "admin"; OS_USER_DOMAIN_NAME = "Default"; OS_PROJECT_DOMAIN_NAME = "Default"; - OS_AUTH_URL = "http://controller:5000/v3"; + OS_AUTH_URL = "http://${config.openstack.controllerHostname}:5000/v3"; OS_IDENTITY_API_VERSION = "3"; }; + + adminEnvScript = pkgs.writeShellScript "openstack-admin-env" ( + lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: value: "export ${name}=${lib.escapeShellArg value}") adminEnv + ) + ); + + databaseCleanupScript = pkgs.writeShellScript "database-cleanup.sh" '' + export PATH=${lib.makeBinPath [ pkgs.mariadb ]}:$PATH + mariadb -N -e "drop database keystone;" || true + mariadb -N -e "drop database glance;" || true + mariadb -N -e "drop database cinder;" || true + mariadb -N -e "drop database placement;" || true + mariadb -N -e "drop database nova_api;" || true + mariadb -N -e "drop database nova;" || true + mariadb -N -e "drop database nova_cell0;" || true + mariadb -N -e "drop database neutron;" || true + mariadb -N -e "drop database designate;" || true + ''; + + # Keep manual provisioning commands and systemd units on the same store scripts. + setupCommand = + name: script: + pkgs.writeShellApplication { + inherit name; + runtimeInputs = [ + pkgs.coreutils + pkgs.systemd + ]; + text = '' + if (( EUID != 0 )); then + echo "Run this command as root." >&2 + exit 1 + fi + # shellcheck source=/dev/null + source ${adminEnvScript} + exec ${script} "$@" + ''; + }; + + databaseSetupScript = pkgs.writeShellScript "database-setup.sh" '' + export PATH=${lib.makeBinPath [ pkgs.mariadb ]}:$PATH + set -euxo pipefail + + # Keystone + mariadb -N -e "CREATE DATABASE IF NOT EXISTS keystone;" + mariadb -N -e "CREATE USER IF NOT EXISTS 'keystone'@'%' IDENTIFIED BY 'keystone';" + mariadb -N -e "ALTER USER 'keystone'@'%' IDENTIFIED BY 'keystone';" + mariadb -N -e "GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'%';" + + # Glance + mariadb -N -e "CREATE DATABASE IF NOT EXISTS glance;" + mariadb -N -e "CREATE USER IF NOT EXISTS 'glance'@'%' IDENTIFIED BY 'glance';" + mariadb -N -e "ALTER USER 'glance'@'%' IDENTIFIED BY 'glance';" + mariadb -N -e "GRANT ALL PRIVILEGES ON glance.* TO 'glance'@'%';" + + # Cinder + mariadb -N -e "CREATE DATABASE IF NOT EXISTS cinder;" + mariadb -N -e "CREATE USER IF NOT EXISTS 'cinder'@'%' IDENTIFIED BY 'cinder';" + mariadb -N -e "ALTER USER 'cinder'@'%' IDENTIFIED BY 'cinder';" + mariadb -N -e "GRANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'%';" + + # Placement + mariadb -N -e "CREATE DATABASE IF NOT EXISTS placement;" + mariadb -N -e "CREATE USER IF NOT EXISTS 'placement'@'%' IDENTIFIED BY 'placement';" + mariadb -N -e "ALTER USER 'placement'@'%' IDENTIFIED BY 'placement';" + mariadb -N -e "GRANT ALL PRIVILEGES ON placement.* TO 'placement'@'%';" + + # Nova + mariadb -N -e "CREATE DATABASE IF NOT EXISTS nova_api;" + mariadb -N -e "CREATE DATABASE IF NOT EXISTS nova;" + mariadb -N -e "CREATE DATABASE IF NOT EXISTS nova_cell0;" + mariadb -N -e "CREATE USER IF NOT EXISTS 'nova'@'%' IDENTIFIED BY 'nova';" + mariadb -N -e "ALTER USER 'nova'@'%' IDENTIFIED BY 'nova';" + mariadb -N -e "GRANT ALL PRIVILEGES ON nova_api.* TO 'nova'@'%';" + mariadb -N -e "GRANT ALL PRIVILEGES ON nova.* TO 'nova'@'%';" + mariadb -N -e "GRANT ALL PRIVILEGES ON nova_cell0.* TO 'nova'@'%';" + + # Neutron + mariadb -N -e "CREATE DATABASE IF NOT EXISTS neutron;" + mariadb -N -e "CREATE USER IF NOT EXISTS 'neutron'@'%' IDENTIFIED BY 'neutron';" + mariadb -N -e "ALTER USER 'neutron'@'%' IDENTIFIED BY 'neutron';" + mariadb -N -e "GRANT ALL PRIVILEGES ON neutron.* TO 'neutron'@'%';" + + # Designate + mariadb -N -e "CREATE DATABASE IF NOT EXISTS designate CHARACTER SET utf8 COLLATE utf8_general_ci;" + mariadb -N -e "CREATE USER IF NOT EXISTS 'designate'@'%' IDENTIFIED BY 'designate';" + mariadb -N -e "ALTER USER 'designate'@'%' IDENTIFIED BY 'designate';" + mariadb -N -e "GRANT ALL PRIVILEGES ON designate.* TO 'designate'@'%';" + + # fix mariadb permissions + mariadb -N -e "delete from mysql.user where user = ''';" + + mariadb -N -e "FLUSH PRIVILEGES;" + ''; + + keystonePreStartScript = pkgs.writeShellScript "keystone-all-pre-start.sh" '' + export PATH=${ + lib.makeBinPath [ + keystone + pkgs.coreutils + ] + }:$PATH + + set -euxo pipefail + + # Initialise the database + keystone-manage --config-file ${config.keystone.config} db_sync + # Set up the keystone's PKI infrastructure + keystone-manage --config-file ${config.keystone.config} fernet_setup --keystone-user keystone --keystone-group keystone + keystone-manage --config-file ${config.keystone.config} credential_setup --keystone-user keystone --keystone-group keystone + chown -R keystone:keystone /etc/keystone + chown -R keystone:keystone /var/log/keystone + + systemctl restart uwsgi.service + ''; + + keystoneStartScript = pkgs.writeShellScript "keystone-all.sh" '' + export PATH=${ + lib.makeBinPath [ + keystone + pkgs.openstackclient + pkgs.util-linux + ] + }:$PATH + + runuser --user keystone --preserve-environment -- ${pkgs.runtimeShell} <<'EOF' + set -euxo pipefail + keystone-manage --config-file ${config.keystone.config} bootstrap \ + --bootstrap-password admin \ + --bootstrap-region-id RegionOne + openstack project create --domain default --description "Service Project" service + EOF + ''; + + glanceStartScript = pkgs.writeShellScript "glance.sh" '' + export PATH=${ + lib.makeBinPath [ + glance + pkgs.openstackclient + pkgs.util-linux + ] + }:$PATH + + mkdir -p /var/lib/glance + mkdir -p /var/lib/glance/images + chown -R glance /var/lib/glance + chgrp -R glance /var/lib/glance + + mkdir -p /var/log/glance + chown glance /var/log/glance + chgrp glance /var/log/glance + + runuser --user glance --preserve-environment -- ${pkgs.runtimeShell} <<'EOF' + set -euxo pipefail + openstack user create --domain default --password glance glance + openstack role add --project service --user glance admin + openstack role add --user glance --user-domain default --system all reader + glance-manage --config-file ${config.glance.config} db_sync + EOF + ''; + + cinderStartScript = pkgs.writeShellScript "cinder.sh" '' + export PATH=${ + lib.makeBinPath [ + cinder + pkgs.openstackclient + pkgs.util-linux + ] + }:$PATH + + runuser --user cinder --preserve-environment -- ${pkgs.runtimeShell} <<'EOF' + set -euxo pipefail + openstack user create --domain default --password cinder cinder || true + openstack role add --project service --user cinder admin || true + openstack role add --user cinder --user-domain default --system all reader || true + cinder-manage --config-file ${config.cinder.config} db sync + EOF + ''; + + placementStartScript = pkgs.writeShellScript "placement.sh" '' + export PATH=${ + lib.makeBinPath [ + placement + pkgs.openstackclient + pkgs.util-linux + ] + }:$PATH + + runuser --user placement --preserve-environment -- ${pkgs.runtimeShell} <<'EOF' + set -euxo pipefail + openstack user create --domain default --password placement placement + openstack role add --project service --user placement admin + placement-manage --config-file ${config.placement.config} db sync + EOF + ''; + + novaStartScript = pkgs.writeShellScript "nova.sh" '' + export PATH=${ + lib.makeBinPath [ + nova + pkgs.openstackclient + pkgs.util-linux + ] + }:$PATH + + runuser --user nova --preserve-environment -- ${pkgs.runtimeShell} <<'EOF' + set -euxo pipefail + openstack user create --domain default --password nova nova + openstack role add --project service --user nova admin + nova-manage --config-file ${config.nova.config} api_db sync + nova-manage --config-file ${config.nova.config} cell_v2 map_cell0 + nova-manage --config-file ${config.nova.config} cell_v2 create_cell --name=cell1 --verbose + nova-manage --config-file ${config.nova.config} db sync + EOF + ''; + + neutronStartScript = pkgs.writeShellScript "neutron.sh" '' + export PATH=${ + lib.makeBinPath [ + neutron + pkgs.openstackclient + pkgs.util-linux + ] + }:$PATH + + runuser --user neutron --preserve-environment -- ${pkgs.runtimeShell} <<'EOF' + set -euxo pipefail + openstack user create --domain default --password neutron neutron + openstack role add --project service --user neutron admin + neutron-db-manage --config-file ${config.neutron.config} --config-file ${config.neutron.ml2Config} upgrade head + EOF + ''; + + designateStartScript = pkgs.writeShellScript "designate.sh" '' + export PATH=${ + lib.makeBinPath [ + designate + pkgs.openstackclient + pkgs.util-linux + ] + }:$PATH + + set -euxo pipefail + + runuser --user designate -- \ + ${designate}/bin/designate-manage --config-file ${config.designate.config} database sync + + openstack user show designate >/dev/null 2>&1 || \ + openstack user create --domain default --password designate designate + openstack role add --project service --user designate admin + ''; + + checkControllerScript = pkgs.writeShellScript "check-controller.sh" '' + export PATH=${ + lib.makeBinPath [ + pkgs.openstackclient + ] + }:$PATH + + systemctl status neutron-server.service + systemctl status glance-api.service + systemctl status uwsgi.service + systemctl status cinder-scheduler.service + systemctl status nova-api.service + systemctl status nova-scheduler.service + systemctl status nova-conductor.service + systemctl status nova-novncproxy.service + systemctl status nova-serialproxy.service + ''; + in { imports = [ + ../generic/global-options.nix ./generic.nix ../generic/controller-host-entry.nix (import ./keystone.nix { inherit keystone; }) @@ -34,11 +309,36 @@ in (import ./neutron.nix { inherit neutron; }) (import ./horizon.nix { inherit horizon; }) (import ./cinder.nix { inherit cinder; }) # only cinder management component + (import ./designate.nix { inherit designate; }) ]; config = { - systemd.services.database-setup = { + environment.variables = adminEnv; + + environment.systemPackages = [ + (setupCommand "openstack-database-cleanup-000" databaseCleanupScript) + (setupCommand "openstack-database-setup-000" databaseSetupScript) + (setupCommand "openstack-keystone-pre-setup-001" keystonePreStartScript) + (setupCommand "openstack-keystone-setup-002" keystoneStartScript) + (setupCommand "openstack-glance-setup-003" glanceStartScript) + (setupCommand "openstack-cinder-setup-004" cinderStartScript) + (setupCommand "openstack-placement-setup-005" placementStartScript) + (setupCommand "openstack-neutron-setup-006" neutronStartScript) + (setupCommand "openstack-nova-setup-007" novaStartScript) + (setupCommand "openstack-designate-setup-008" designateStartScript) + (setupCommand "openstack-check-controller-100" checkControllerScript) + python-openstackclient + cinder + designate + glance + keystone + neutron + nova + placement + ]; + + systemd.services.database-setup = lib.mkIf (!config.openstack.manual_setup) { description = "OpenStack Database setup"; after = [ "mysql.service" @@ -47,58 +347,16 @@ in ]; wantedBy = [ "multi-user.target" ]; path = [ pkgs.mariadb ]; + restartIfChanged = false; serviceConfig = { + RemainAfterExit = true; Type = "oneshot"; - ExecStart = pkgs.writeShellScript "database-setup.sh" '' - # Keystone - mysql -N -e "drop database keystone;" || true - mysql -N -e "create database keystone;" || true - mysql -N -e "GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'localhost' IDENTIFIED BY 'keystone';" - mysql -N -e "GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'%' IDENTIFIED BY 'keystone';" - - # Glance - mysql -N -e "drop database glance;" || true - mysql -N -e "create database glance;" || true - mysql -N -e "GRANT ALL PRIVILEGES ON glance.* TO 'glance'@'localhost' IDENTIFIED BY 'glance';" - mysql -N -e "GRANT ALL PRIVILEGES ON glance.* TO 'glance'@'%' IDENTIFIED BY 'glance';" - - # Cinder - mysql -N -e "drop database cinder;" || true - mysql -N -e "create database cinder;" || true - mysql -N -e "GRANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'localhost' IDENTIFIED BY 'cinder';" - mysql -N -e "GRANT ALL PRIVILEGES ON cinder.* TO 'cinder'@'%' IDENTIFIED BY 'cinder';" - - # Placement - mysql -N -e "drop database placement;" || true - mysql -N -e "create database placement;" || true - mysql -N -e "GRANT ALL PRIVILEGES ON placement.* TO 'placement'@'localhost' IDENTIFIED BY 'placement';" - mysql -N -e "GRANT ALL PRIVILEGES ON placement.* TO 'placement'@'%' IDENTIFIED BY 'placement';" - - # Nova - mysql -N -e "drop database nova_api;" || true - mysql -N -e "drop database nova;" || true - mysql -N -e "drop database nova_cell0;" || true - mysql -N -e "create database nova_api;" || true - mysql -N -e "create database nova;" || true - mysql -N -e "create database nova_cell0;" || true - - mysql -N -e "GRANT ALL PRIVILEGES ON nova_api.* TO 'nova'@'localhost' IDENTIFIED BY 'nova';" - mysql -N -e "GRANT ALL PRIVILEGES ON nova_api.* TO 'nova'@'%' IDENTIFIED BY 'nova';" - mysql -N -e "GRANT ALL PRIVILEGES ON nova.* TO 'nova'@'localhost' IDENTIFIED BY 'nova';" - mysql -N -e "GRANT ALL PRIVILEGES ON nova.* TO 'nova'@'%' IDENTIFIED BY 'nova';" - mysql -N -e "GRANT ALL PRIVILEGES ON nova_cell0.* TO 'nova'@'localhost' IDENTIFIED BY 'nova';" - mysql -N -e "GRANT ALL PRIVILEGES ON nova_cell0.* TO 'nova'@'%' IDENTIFIED BY 'nova';" - - # Neutron - mysql -N -e "drop database neutron;" || true - mysql -N -e "create database neutron;" || true - mysql -N -e "GRANT ALL PRIVILEGES ON neutron.* TO 'neutron'@'localhost' IDENTIFIED BY 'neutron';" - mysql -N -e "GRANT ALL PRIVILEGES ON neutron.* TO 'neutron'@'%' IDENTIFIED BY 'neutron';" - ''; + ExecStartPre = "${databaseCleanupScript}"; + ExecStart = "${databaseSetupScript}"; }; }; - systemd.services.keystone-all = { + systemd.services.keystone-all = lib.mkIf (!config.openstack.manual_setup) { description = "OpenStack Keystone Daemon"; after = [ "database-setup.service" ]; path = [ @@ -107,31 +365,18 @@ in ]; environment = adminEnv; wantedBy = [ "multi-user.target" ]; - preStart = '' - # Initialise the database - keystone-manage --config-file ${config.keystone.config} db_sync - # Set up the keystone's PKI infrastructure - keystone-manage --config-file ${config.keystone.config} fernet_setup --keystone-user keystone --keystone-group keystone - keystone-manage --config-file ${config.keystone.config} credential_setup --keystone-user keystone --keystone-group keystone - chown -R keystone:keystone /etc/keystone - chown -R keystone:keystone /var/log/keystone - ''; + restartIfChanged = false; serviceConfig = { - PermissionsStartOnly = true; - User = "keystone"; - Group = "keystone"; + User = "root"; + Group = "root"; + RemainAfterExit = true; Type = "oneshot"; - ExecStart = pkgs.writeShellScript "keystone-all.sh" '' - set -euxo pipefail - keystone-manage --config-file ${config.keystone.config} bootstrap \ - --bootstrap-password admin\ - --bootstrap-region-id RegionOne - openstack project create --domain default --description "Service Project" service - ''; + ExecStartPre = "+${keystonePreStartScript}"; + ExecStart = "+${keystoneStartScript}"; }; }; - systemd.services.glance = { + systemd.services.glance = lib.mkIf (!config.openstack.manual_setup) { description = "OpenStack Glance setup"; after = [ "keystone-all.service" ]; wantedBy = [ "multi-user.target" ]; @@ -140,21 +385,17 @@ in pkgs.openstackclient glance ]; + restartIfChanged = false; serviceConfig = { + RemainAfterExit = true; Type = "oneshot"; - User = "glance"; - Group = "glance"; - ExecStart = pkgs.writeShellScript "glance.sh" '' - set -euxo pipefail - openstack user create --domain default --password glance glance - openstack role add --project service --user glance admin - openstack role add --user glance --user-domain default --system all reader - glance-manage --config-file ${config.glance.config} db_sync - ''; + User = "root"; + Group = "root"; + ExecStart = "+${glanceStartScript}"; }; }; - systemd.services.cinder = { + systemd.services.cinder = lib.mkIf (!config.openstack.manual_setup) { description = "OpenStack Cinder setup"; after = [ "keystone-all.service" ]; wantedBy = [ "multi-user.target" ]; @@ -163,24 +404,22 @@ in pkgs.openstackclient cinder ]; + restartIfChanged = false; serviceConfig = { + RemainAfterExit = true; Type = "oneshot"; - User = "cinder"; - Group = "cinder"; - ExecStart = pkgs.writeShellScript "cinder.sh" '' - set -euxo pipefail - openstack user create --domain default --password cinder cinder || true - openstack role add --project service --user cinder admin || true - openstack role add --user cinder --user-domain default --system all reader || true - cinder-manage --config-file ${config.cinder.config} db sync - ''; + User = "root"; + Group = "root"; + ExecStart = "+${cinderStartScript}"; + RuntimeDirectory = "cinder-setup"; + ExecStartPost = "${pkgs.coreutils}/bin/touch /run/cinder-setup/ready"; }; }; # Placement service can be tested by executing - # curl http://controller:8778 + # curl http://${config.openstack.controllerHostname}:8778 # and receive some json with version info as result. - systemd.services.placement = { + systemd.services.placement = lib.mkIf (!config.openstack.manual_setup) { description = "OpenStack Placement setup"; after = [ "glance.service" ]; requiredBy = [ "multi-user.target" ]; @@ -189,20 +428,19 @@ in pkgs.openstackclient placement ]; + restartIfChanged = false; serviceConfig = { + RemainAfterExit = true; Type = "oneshot"; - User = "placement"; - Group = "placement"; - ExecStart = pkgs.writeShellScript "placement.sh" '' - set -euxo pipefail - openstack user create --domain default --password placement placement - openstack role add --project service --user placement admin - placement-manage --config-file ${config.placement.config} db sync - ''; + User = "root"; + Group = "root"; + ExecStart = "+${placementStartScript}"; + RuntimeDirectory = "placement-setup"; + ExecStartPost = "${pkgs.coreutils}/bin/touch /run/placement-setup/ready"; }; }; - systemd.services.nova = { + systemd.services.nova = lib.mkIf (!config.openstack.manual_setup) { description = "OpenStack Nova setup"; after = [ "neutron.service" ]; wantedBy = [ "multi-user.target" ]; @@ -211,23 +449,17 @@ in pkgs.openstackclient nova ]; + restartIfChanged = false; serviceConfig = { + RemainAfterExit = true; Type = "oneshot"; - User = "nova"; - Group = "nova"; - ExecStart = pkgs.writeShellScript "nova.sh" '' - set -euxo pipefail - openstack user create --domain default --password nova nova - openstack role add --project service --user nova admin - nova-manage --config-file ${config.nova.config} api_db sync - nova-manage --config-file ${config.nova.config} cell_v2 map_cell0 - nova-manage --config-file ${config.nova.config} cell_v2 create_cell --name=cell1 --verbose - nova-manage --config-file ${config.nova.config} db sync - ''; + User = "root"; + Group = "root"; + ExecStart = "+${novaStartScript}"; }; }; - systemd.services.neutron = { + systemd.services.neutron = lib.mkIf (!config.openstack.manual_setup) { description = "OpenStack Neutron setup"; after = [ "placement.service" ]; wantedBy = [ "multi-user.target" ]; @@ -236,16 +468,34 @@ in pkgs.openstackclient neutron ]; + restartIfChanged = false; + serviceConfig = { + RemainAfterExit = true; + Type = "oneshot"; + User = "root"; + Group = "root"; + ExecStart = "+${neutronStartScript}"; + }; + }; + + systemd.services.designate = lib.mkIf (config.designate.enable && !config.openstack.manual_setup) { + description = "OpenStack Designate setup"; + after = [ "keystone-all.service" ]; + wantedBy = [ "multi-user.target" ]; + environment = adminEnv; + path = [ + pkgs.gnugrep + pkgs.openstackclient + pkgs.util-linux + designate + ]; + restartIfChanged = false; serviceConfig = { + RemainAfterExit = true; Type = "oneshot"; - User = "neutron"; - Group = "neutron"; - ExecStart = pkgs.writeShellScript "neutron.sh" '' - set -euxo pipefail - openstack user create --domain default --password neutron neutron - openstack role add --project service --user neutron admin - neutron-db-manage --config-file ${config.neutron.config} --config-file ${config.neutron.ml2Config} upgrade head - ''; + User = "root"; + Group = "root"; + ExecStart = "+${designateStartScript}"; }; }; }; diff --git a/modules/controller/placement.nix b/modules/controller/placement.nix index 98be12c..3a426ad 100644 --- a/modules/controller/placement.nix +++ b/modules/controller/placement.nix @@ -16,8 +16,8 @@ let auth_strategy = keystone [keystone_authtoken] - www_authenticate_uri = http://controller:5000 - auth_url = http://controller:5000 + www_authenticate_uri = http://${config.openstack.controllerHostname}:5000/v3 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 auth_type = password project_domain_name = Default user_domain_name = Default @@ -26,7 +26,7 @@ let password = placement [placement_database] - connection = mysql+pymysql://placement:placement@controller/placement + connection = mysql+pymysql://placement:placement@${config.openstack.controllerHostname}/placement ''; in { @@ -40,8 +40,17 @@ in The Placement config. ''; }; + env = mkOption { + type = types.listOf types.str; + default = [ + "PYTHONWARNINGS=ignore::DeprecationWarning" + ]; + description = '' + Environment variables passed to the placement uWSGI vassal. + ''; + }; }; - config = mkIf cfg.enable { + config = { users.extraUsers.placement = { group = "placement"; @@ -55,34 +64,49 @@ in systemd.tmpfiles.settings = { "10-placement" = { "/etc/placement/placement.conf" = { - L = { + "L+" = { argument = "${cfg.config}"; }; }; }; }; - systemd.services.placement-api = { - description = "OpenStack Placement API Daemon"; - after = [ - "placement.service" - "rabbitmq.service" - "mysql.service" - "network.target" - ]; - path = [ placement ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - User = "placement"; - Group = "placement"; - ExecStart = pkgs.writeShellScript "placement-api.sh" '' - # Using the placement-api binary directly warns about that should only - # be done in development environments. The "correct" way is to use - # uwsgi, which is kinda tricky in nixos because we would have to start - # the keystone uwsgi service at first, and placement later. That - # cannot be expressed in the nixos config AFAIK. - placement-api --port 8778 - ''; + services.uwsgi = { + instance.vassals = optionalAttrs cfg.enable { + placement-api = { + type = "normal"; + http-socket = "0.0.0.0:8778"; + wsgi-file = "${placement}/bin/.placement-api-wrapped"; + pyargv = "--config-file ${cfg.config}"; + + master = true; + processes = 4; + enable-threads = true; + thunder-lock = true; + lazy-apps = true; + die-on-term = true; + vacuum = true; + need-app = true; + buffer-size = 65535; + + immediate-uid = "placement"; + immediate-gid = "placement"; + env = cfg.env; + } + // optionalAttrs (!config.openstack.manual_setup) { + # Keystone must start in the same emperor before Placement setup can + # finish. Wait only in this vassal, using a hook that works as non-root. + exec-asap = pkgs.writeShellScript "placement-api-wait-for-setup" '' + for ((attempt = 0; attempt < 300; attempt++)); do + if test -f /run/placement-setup/ready; then + exit 0 + fi + ${pkgs.coreutils}/bin/sleep 1 + done + echo "Timed out waiting for Placement database setup" >&2 + exit 1 + ''; + }; }; }; }; diff --git a/modules/default.nix b/modules/default.nix index 179ed5d..6103cb6 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -8,6 +8,8 @@ glance horizon cinder + python-openstackclient + designate ; placement = openstackPkgs.openstack-placement; }; diff --git a/modules/generic/controller-host-entry.nix b/modules/generic/controller-host-entry.nix index 1a80a62..0a996f1 100644 --- a/modules/generic/controller-host-entry.nix +++ b/modules/generic/controller-host-entry.nix @@ -1,19 +1,18 @@ -{ lib, config, ... }: -with lib; +{ config, lib, ... }: { - options.openstack = { - controllerIP = mkOption { - type = types.str; - description = '' - IP address of the controller. Will be used to make a /etc/hosts entry - to make the controller available via "controller". - ''; - }; - }; + imports = [ ./global-options.nix ]; config = { networking.extraHosts = '' - ${config.openstack.controllerIP} controller controller.local + ${config.openstack.controllerIP} ${ + lib.concatStringsSep " " ( + lib.unique [ + config.openstack.controllerHostname + "controller" + "controller.local" + ] + ) + } ''; }; } diff --git a/modules/generic/global-options.nix b/modules/generic/global-options.nix new file mode 100644 index 0000000..ed09dc0 --- /dev/null +++ b/modules/generic/global-options.nix @@ -0,0 +1,43 @@ +{ lib, ... }: +with lib; +{ + options.openstack.controllerHostname = lib.mkOption { + type = types.str; + default = "controller"; + description = '' + Hostname of the OpenStack controller. + ''; + }; + + options.openstack.controllerIP = lib.mkOption { + type = types.str; + description = '' + IP address of the controller, used by controller services and for the + configured controller hostname in /etc/hosts. The controller and + controller.local aliases also resolve to this address. + ''; + }; + + options.openstack.manual_setup = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether to manage OpenStack setup manually. When enabled, automatic + database setup, OpenStack service bootstrapping, and Cinder storage + initialization are disabled. Setup must be performed separately. + + This option is meant to allow persistent setups of openstack-nix as + physical deployments. + ''; + }; + + options.openstack.live_migration_inbound_addr = lib.mkOption { + default = null; + type = types.nullOr types.str; + description = '' + Address of this compute node on the live migration network. When set, + configures Nova's libvirt live_migration_inbound_addr. When null, the + setting is omitted and Nova uses its default. + ''; + }; +} diff --git a/modules/storage/cinder-storage-node.nix b/modules/storage/cinder-storage-node.nix index a6f7ca2..2703c8f 100644 --- a/modules/storage/cinder-storage-node.nix +++ b/modules/storage/cinder-storage-node.nix @@ -35,14 +35,14 @@ let cinderConfLvm = pkgs.writeText "cinder.conf" '' [DEFAULT] - transport_url = rabbit://openstack:openstack@controller + transport_url = rabbit://openstack:openstack@${config.openstack.controllerHostname} auth_strategy = keystone - my_ip = 10.0.0.20 + my_ip = ${config.openstack.storageIP} enabled_backends = lvm volumes_dir = /var/lib/cinder/volumes state_path = /var/lib/cinder - rootwrap_config = ${rootwrapConf} - glance_api_servers = http://controller:9292 + rootwrap_config = /etc/cinder/rootwrap.conf + glance_api_servers = http://${config.openstack.controllerHostname}:9292 verify_glance_signatures = disabled log_dir = /var/log/cinder iscsi_ip_address = $my_ip @@ -50,12 +50,12 @@ let iscsi_target_prefix = iqn.2010-10.org.openstack: [database] - connection = mysql+pymysql://cinder:cinder@controller/cinder + connection = mysql+pymysql://cinder:cinder@${config.openstack.controllerHostname}/cinder [keystone_authtoken] - www_authenticate_uri = http://controller:5000 - auth_url = http://controller:5000 - memcached_servers = controller:11211 + www_authenticate_uri = http://${config.openstack.controllerHostname}:5000/v3 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 + memcached_servers = ${config.openstack.controllerHostname}:11211 auth_type = password project_domain_name = default user_domain_name = default @@ -82,24 +82,24 @@ let cinderConfNfs = pkgs.writeText "cinder.conf" '' [DEFAULT] - transport_url = rabbit://openstack:openstack@controller + transport_url = rabbit://openstack:openstack@${config.openstack.controllerHostname} auth_strategy = keystone - my_ip = 10.0.0.20 + my_ip = ${config.openstack.storageIP} enabled_backends = nfs volumes_dir = /var/lib/cinder/volumes state_path = /var/lib/cinder - rootwrap_config = ${rootwrapConf} - glance_api_servers = http://controller:9292 + rootwrap_config = /etc/cinder/rootwrap.conf + glance_api_servers = http://${config.openstack.controllerHostname}:9292 verify_glance_signatures = disabled log_dir = /var/log/cinder [database] - connection = mysql+pymysql://cinder:cinder@controller/cinder + connection = mysql+pymysql://cinder:cinder@${config.openstack.controllerHostname}/cinder [keystone_authtoken] - www_authenticate_uri = http://controller:5000 - auth_url = http://controller:5000 - memcached_servers = controller:11211 + www_authenticate_uri = http://${config.openstack.controllerHostname}:5000/v3 + auth_url = http://${config.openstack.controllerHostname}:5000/v3 + memcached_servers = ${config.openstack.controllerHostname}:11211 auth_type = password project_domain_name = default user_domain_name = default @@ -121,16 +121,97 @@ let cinderTgtConf = pkgs.writeText "cinder.conf" '' include /var/lib/cinder/volumes/* ''; + + cinderDefaultNFSexports = pkgs.writeText "exports" '' + ${config.openstack.storagePath} 10.0.0.0/24(rw,no_root_squash,insecure) + ''; + + # Run explicitly via openstack-cinder-volume-setup during manual deployment + # to format and mount the NFS storage disk and restart Cinder. + cinderVolumeSetupScript = pkgs.writeShellScript "cinder-volume-setup.sh" '' + export PATH=${ + lib.makeBinPath [ + pkgs.util-linux + pkgs.nfs-utils + ] + }:$PATH + + set -euxo pipefail + + if [ -e ${config.openstack.storagePath}/.cinder-volume-setup-done-dont-delete-me ]; then + echo "cinder volume setup already done. Check content of this script." + exit 0 + fi + + mkdir ${config.openstack.storagePath} + mkfs.ext4 -F -m 0 -L cinder ${lib.escapeShellArg cfg.setupDevice} + mount ${lib.escapeShellArg cfg.setupDevice} ${config.openstack.storagePath} + exportfs -rv + rm -rf ${config.openstack.storagePath}/lost+found + chown cinder ${config.openstack.storagePath} + chgrp cinder ${config.openstack.storagePath} + + systemctl restart cinder-volume.service + touch ${config.openstack.storagePath}/.cinder-volume-setup-done-dont-delete-me + ''; + + cinderVolumeSetupCommand = pkgs.writeShellApplication { + name = "openstack-cinder-volume-setup"; + runtimeInputs = with pkgs; [ + coreutils + systemd + util-linux + nfs-utils + e2fsprogs + ]; + text = '' + if (( EUID != 0 )); then + echo "Run this command as root." >&2 + exit 1 + fi + exec ${cinderVolumeSetupScript} "$@" + ''; + }; + in { imports = [ + ../generic/global-options.nix ../generic/controller-host-entry.nix ]; + options.openstack = { + storageIP = mkOption { + type = types.str; + default = "10.0.0.20"; + description = '' + IP address of the storage node. + ''; + }; + storagePath = mkOption { + type = types.str; + default = "/exports"; + description = '' + Exported filesystem path on the storage node. + ''; + }; + }; + options.cinder-storage-node = { enable = mkEnableOption "Enable OpenStack Cinder storage node." // { default = true; }; + setupDevice = mkOption { + type = types.str; + default = "/dev/vdb"; + example = "/dev/disk/by-id/your-storage-disk"; + description = '' + Block device formatted and mounted by openstack-cinder-volume-setup + during manual deployment of the NFS backend. Running that command + creates an ext4 filesystem on this device, destroying existing data. + This option does not affect automatic test storage initialization. + ''; + }; config = mkOption { default = if (cfg.backend == "lvm") then cinderConfLvm else cinderConfNfs; description = '' @@ -157,9 +238,22 @@ in Possible options: [ lvm | nfs ] ''; }; + exports = mkOption { + default = cinderDefaultNFSexports; + description = '' + The nfs-server /etc/exports file. + ''; + }; + rootwrapConf = mkOption { + default = rootwrapConf; + description = '' + Cinder root wrap configuration file. + ''; + }; }; - config = mkIf cfg.enable { + config = { + users.extraUsers.cinder = { group = "cinder"; isSystemUser = true; @@ -172,48 +266,54 @@ in security.sudo.enable = true; security.sudo.extraConfig = '' cinder ALL = (root) NOPASSWD: ${cinder_env}/bin/cinder-rootwrap ${rootwrapConf} * + cinder ALL = (root) NOPASSWD: ${cinder_env}/bin/cinder-rootwrap /etc/cinder/rootwrap.conf * ''; + # set this attributes only if this storage module is deployed to a different host than the controller + # or: don't set this attributes if this storage module is deployed alongside the controller module systemd.tmpfiles.settings = { - "20-cinder" = { - "/var/lib/cinder/" = { - D = { - user = "cinder"; - group = "cinder"; - mode = "0755"; - }; - }; - "/var/lib/cinder/volumes" = { - D = { - user = "cinder"; - group = "cinder"; - mode = "0755"; - }; - }; - "/var/log/cinder/" = { - D = { - user = "cinder"; - group = "cinder"; - mode = "0755"; - }; - }; - "/etc/cinder/cinder.conf" = { - L = { - argument = "${cfg.config}"; + "20-cinder" = + lib.mkIf + (!(config ? cinder && builtins.isBool config.cinder.enable && config.cinder.enable == true)) + { + "/var/lib/cinder/" = { + d = { + user = "cinder"; + group = "cinder"; + mode = "0755"; + }; + }; + "/var/lib/cinder/volumes" = { + d = { + user = "cinder"; + group = "cinder"; + mode = "0755"; + }; + }; + "/var/log/cinder/" = { + d = { + user = "cinder"; + group = "cinder"; + mode = "0755"; + }; + }; + "/etc/cinder/cinder.conf" = { + "L+" = { + argument = "${cfg.config}"; + }; + }; }; - }; - }; "20-cinder-backend" = if (cfg.backend == "lvm") then # LVM configuration files { "/etc/tgt/conf.d/cinder.conf" = { - L = { + "L+" = { argument = "${cinderTgtConf}"; }; }; "/etc/tgt/targets.conf" = { - L = { + "L+" = { argument = "${pkgs.tgt}/etc/tgt/targets.conf"; }; }; @@ -222,22 +322,29 @@ in # NFS configuration files { "/etc/cinder/nfs_shares" = { - f = { + "f+" = { user = "cinder"; group = "cinder"; mode = "0644"; argument = '' - 10.0.0.20:/exports + ${config.openstack.storageIP}:${config.openstack.storagePath} ''; }; }; }; + "20-cinder-root-wrap" = { + "/etc/cinder/rootwrap.conf" = { + "L+" = { + argument = "${cfg.rootwrapConf}"; + }; + }; + }; }; # start iSCSI target daemon # we expose LVM block storage as iSCSI to compute hosts systemd.services.tgtd = { - enable = if (cfg.backend == "lvm") then true else false; + enable = if (cfg.backend == "lvm" && cfg.enable) then true else false; description = "iSCSI target framework daemon"; wantedBy = [ "multi-user.target" ]; after = [ @@ -269,12 +376,11 @@ in }; }; - services.nfs.server.enable = if (cfg.backend == "lvm") then false else true; - services.nfs.server.exports = '' - /exports 10.0.0.0/24(rw,no_root_squash,insecure) - ''; + services.nfs.server.enable = cfg.enable && cfg.backend != "lvm"; + services.nfs.server.exports = builtins.readFile cfg.exports; - systemd.services.cinder-volume-group-setup = { + # run this service only in CI/CD setups + systemd.services.cinder-volume-group-setup = lib.mkIf (!config.openstack.manual_setup) { description = "OpenStack Cinder volume group setup"; wantedBy = [ "multi-user.target" ]; path = with pkgs; [ @@ -299,12 +405,13 @@ in set -euxo pipefail # create a filesystem and mount and export it - mkdir /exports + mkdir ${lib.escapeShellArg config.openstack.storagePath} mkfs.ext4 -F -m 0 /dev/vdb - mount /dev/vdb /exports + mount /dev/vdb ${lib.escapeShellArg config.openstack.storagePath} exportfs -rv ''; }; + enable = cfg.enable; }; # It seems regardless of what we do, the cinder-volume service does not @@ -312,7 +419,10 @@ in # workaround, add it as a systemPackage. # Update: still does not work -.- - environment.systemPackages = + environment.systemPackages = [ + cinderVolumeSetupCommand + ] + ++ ( if (cfg.backend == "lvm") then with pkgs; [ @@ -325,7 +435,8 @@ in qemu nfs-utils e2fsprogs - ]; + ] + ); systemd.services.cinder-volume = { description = "OpenStack Cinder Volume"; @@ -370,6 +481,7 @@ in Restart = "on-failure"; RestartSec = 20; }; + enable = cfg.enable; }; }; } diff --git a/modules/testing/default.nix b/modules/testing/default.nix index 7eca5c5..b9ef69d 100644 --- a/modules/testing/default.nix +++ b/modules/testing/default.nix @@ -24,7 +24,12 @@ let services.getty.autologinUser = "root"; - openstack.controllerIP = "10.0.0.11"; + openstack.controllerIP = lib.mkDefault "10.0.0.11"; + + # The shared host Nix store can be owned by a non-root UID. Root's SSH + # client then rejects systemd's included proxy config on ownership + # checks. These tests only need TCP SSH, not systemd's socket proxy. + programs.ssh.systemd-ssh-proxy.enable = false; networking = { useDHCP = false; @@ -301,13 +306,15 @@ in }; testCompute = - { ... }: + { lib, ... }: { imports = [ common portForwardingCompute ]; + openstack.computeIP = lib.mkDefault "10.0.0.39"; + # this is an example how to enable / disable all port forwardings or change port numbers # openstack-testing.enable = true; # enable / disable all port forwardings # openstack-testing.sshHostPort = 3022; diff --git a/packages/default.nix b/packages/default.nix index 573dcf2..681add3 100644 --- a/packages/default.nix +++ b/packages/default.nix @@ -3,6 +3,7 @@ python3Packages, writeText, lib, + libredirect, }: let # In the past, the packages was not ready for the latest python interpreter. @@ -69,6 +70,38 @@ let tooz ; }; + designate = callPackage ./designate.nix { + inherit + futurist + libredirect + infoblox-client + keystoneauth1 + keystonemiddleware + openstacksdk + oslo-concurrency + oslo-config + oslo-context + oslo-db + oslo-i18n + oslo-log + oslo-messaging + oslo-middleware + oslo-policy + oslo-reports + oslo-rootwrap + oslo-serialization + oslo-service + oslo-upgradecheck + oslo-utils + oslo-versionedobjects + osprofiler + oslotest + python3Packages + python-designateclient + sqlalchemy + tooz + ; + }; django = callPackage ./django.nix { inherit python3Packages; }; django-appconf = callPackage ./django-appconf.nix { inherit @@ -220,6 +253,13 @@ let xstatic-tv4 ; }; + infoblox-client = callPackage ./infoblox-client.nix { + inherit + python3Packages + oslo-log + oslo-serialization + ; + }; jsonpath-rw-ext = callPackage ./jsonpath-rw-ext.nix { inherit python3Packages; }; keystone = callPackage ./keystone.nix { inherit @@ -793,6 +833,7 @@ let osc-lib oslo-i18n python-cinderclient + python-designateclient python-keystoneclient python3Packages ; diff --git a/packages/designate.nix b/packages/designate.nix new file mode 100644 index 0000000..5a81872 --- /dev/null +++ b/packages/designate.nix @@ -0,0 +1,156 @@ +{ + fetchPypi, + libredirect, + futurist, + infoblox-client, + keystoneauth1, + keystonemiddleware, + openstacksdk, + oslo-concurrency, + oslo-config, + oslo-context, + oslo-db, + oslo-i18n, + oslo-log, + oslo-messaging, + oslo-middleware, + oslo-policy, + oslo-reports, + oslo-rootwrap, + oslo-serialization, + oslo-service, + oslo-upgradecheck, + oslo-utils, + oslo-versionedobjects, + oslotest, + osprofiler, + python-designateclient, + python3Packages, + sqlalchemy, + tooz, + writeScript, + lib, +}: +let + inherit (python3Packages) + alembic + dnspython + eventlet + flask + greenlet + jinja2 + jsonschema + paste + pastedeploy + pbr + pecan + pymysql + python-memcached + requests + requests-mock + stestr + stevedore + tenacity + testresources + testscenarios + webob + webtest + ; + + testExcludes = [ + "designate.tests.unit.backend.test_infoblox.*" + ]; + + excludeListFile = writeScript "test_excludes" (lib.concatStringsSep "\n" testExcludes); + +in +python3Packages.buildPythonPackage (rec { + pname = "designate"; + version = "19.1.0"; + pyproject = true; + build-system = [ + python3Packages.pbr + python3Packages.setuptools + ]; + + nativeBuildInputs = [ + pbr + ]; + + postPatch = '' + substituteInPlace etc/designate/rootwrap.d/bind9.filters \ + --replace-fail "/usr/sbin/rndc" "rndc" + ''; + + propagatedBuildInputs = [ + (alembic.override { inherit sqlalchemy; }) + dnspython + eventlet + flask + futurist + greenlet + infoblox-client + jinja2 + jsonschema + keystoneauth1 + keystonemiddleware + openstacksdk + oslo-concurrency + oslo-config + oslo-context + oslo-db + oslo-i18n + oslo-log + oslo-messaging + oslo-middleware + oslo-policy + oslo-reports + oslo-rootwrap + oslo-serialization + oslo-service + oslo-upgradecheck + oslo-utils + oslo-versionedobjects + oslotest + osprofiler + paste + pastedeploy + pbr + pecan + pymysql + python-designateclient + python-memcached + requests + sqlalchemy + stevedore + tenacity + tooz + webob + ]; + + nativeCheckInputs = [ + libredirect.hook + stestr + ]; + + checkInputs = [ + oslotest + requests-mock + testresources + testscenarios + webtest + ]; + + checkPhase = '' + runHook preCheck + echo "nameserver 127.0.0.1" > resolv.conf + export NIX_REDIRECTS="/etc/resolv.conf=$(realpath resolv.conf)" + stestr run --exclude-list ${excludeListFile} + runHook postCheck + ''; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-kqZJRM9WX8nXxEi+8KuBotwkIk8kVlpvDgp+3czDSeE="; + }; +}) diff --git a/packages/glance.nix b/packages/glance.nix index 198cb00..e3c0965 100644 --- a/packages/glance.nix +++ b/packages/glance.nix @@ -41,6 +41,11 @@ in python3Packages.buildPythonPackage (rec { pname = "glance"; version = "29.0.0"; + # Single-store tests must register their options independently of test order. + patches = [ + ./patches/glance-single-store-test-options.patch + ]; + pyproject = true; build-system = [ python3Packages.pbr diff --git a/packages/infoblox-client.nix b/packages/infoblox-client.nix new file mode 100644 index 0000000..0790c2c --- /dev/null +++ b/packages/infoblox-client.nix @@ -0,0 +1,45 @@ +{ + fetchPypi, + python3Packages, + oslo-log, + oslo-serialization, +}: +let + inherit (python3Packages) + stestr + requests + urllib3 + six + ; +in +python3Packages.buildPythonPackage (rec { + pname = "infoblox-client"; + version = "0.6.2"; + pyproject = true; + build-system = [ + python3Packages.setuptools + ]; + + nativeBuildInputs = [ + ]; + + propagatedBuildInputs = [ + oslo-log + oslo-serialization + requests + urllib3 + six + ]; + + nativeCheckInputs = [ + stestr + ]; + + checkInputs = [ + ]; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-0hkgof6LLAwp2rYOl8rgLTSFYeQauik7Bi4rzhmszVE="; + }; +}) diff --git a/packages/keystone.nix b/packages/keystone.nix index a6f2f93..147d1b2 100644 --- a/packages/keystone.nix +++ b/packages/keystone.nix @@ -26,6 +26,7 @@ let freezegun hacking jsonschema + ldap ldappool lxml oauthlib @@ -35,7 +36,7 @@ let pycodestyle pymysql pysaml2 - ldap + python-memcached requests stestr tempest @@ -82,6 +83,7 @@ python3Packages.buildPythonPackage (rec { pymysql pysaml2 python-keystoneclient + python-memcached sqlalchemy webob ]; diff --git a/packages/oslo-middleware.nix b/packages/oslo-middleware.nix index c4158a3..f98a1a2 100644 --- a/packages/oslo-middleware.nix +++ b/packages/oslo-middleware.nix @@ -27,6 +27,11 @@ python3Packages.buildPythonPackage rec { pname = "oslo.middleware"; version = "6.3.0"; + # The standalone server test otherwise leaks detailed=True to later tests. + patches = [ + ./patches/oslo-middleware-healthcheck-config-fixture.patch + ]; + pyproject = true; build-system = [ python3Packages.pbr diff --git a/packages/patches/glance-single-store-test-options.patch b/packages/patches/glance-single-store-test-options.patch new file mode 100644 index 0000000..ad64cd2 --- /dev/null +++ b/packages/patches/glance-single-store-test-options.patch @@ -0,0 +1,23 @@ +diff --git a/glance/tests/unit/common/test_utils.py b/glance/tests/unit/common/test_utils.py +--- a/glance/tests/unit/common/test_utils.py ++++ b/glance/tests/unit/common/test_utils.py +@@ -796,17 +796,20 @@ class TestUtils(test_utils.BaseTestCase): + utils.get_stores_from_request, req, body) + + def test_single_store_http_enabled_and_http_not_in_url(self): ++ store.register_opts(CONF) + self.config(stores="http,file", group="glance_store") + loc_url = "rbd://aaaaaaaa/images/id" + self.assertFalse(utils.is_http_store_configured(loc_url)) + + def test_single_store_http_disabled_and_http_in_url(self): ++ store.register_opts(CONF) + self.config(stores="rbd,file", group="glance_store") + loc_url = BASE_URI + self.assertFalse(utils.is_http_store_configured(loc_url)) + + def test_single_store_http_enabled_and_http_in_url(self): ++ store.register_opts(CONF) + self.config(stores="http,file", group="glance_store") + loc_url = BASE_URI + self.assertTrue(utils.is_http_store_configured(loc_url)) diff --git a/packages/patches/oslo-middleware-healthcheck-config-fixture.patch b/packages/patches/oslo-middleware-healthcheck-config-fixture.patch new file mode 100644 index 0000000..9414df6 --- /dev/null +++ b/packages/patches/oslo-middleware-healthcheck-config-fixture.patch @@ -0,0 +1,14 @@ +diff --git a/oslo_middleware/tests/test_healthcheck.py b/oslo_middleware/tests/test_healthcheck.py +--- a/oslo_middleware/tests/test_healthcheck.py ++++ b/oslo_middleware/tests/test_healthcheck.py +@@ -31,6 +31,10 @@ from oslo_middleware.healthcheck import __main__ + + class HealthcheckMainTests(test_base.BaseTestCase): + ++ def setUp(self): ++ super().setUp() ++ self.useFixture(config.Config()) ++ + def test_startup_response(self): + server = __main__.create_server(0) + th = threading.Thread(target=server.serve_forever) diff --git a/packages/python-openstackclient.nix b/packages/python-openstackclient.nix index 2997578..90bc4cb 100644 --- a/packages/python-openstackclient.nix +++ b/packages/python-openstackclient.nix @@ -3,6 +3,7 @@ osc-lib, oslo-i18n, python-cinderclient, + python-designateclient, python-keystoneclient, python3Packages, }: @@ -50,6 +51,7 @@ python3Packages.buildPythonPackage rec { oslo-i18n pbr python-cinderclient + python-designateclient python-keystoneclient requests stevedore diff --git a/tests/openstack-default-setup.nix b/tests/openstack-default-setup.nix index c12a288..5b3490d 100644 --- a/tests/openstack-default-setup.nix +++ b/tests/openstack-default-setup.nix @@ -106,7 +106,6 @@ pkgs.testers.nixosTest { start_all() controllerVM.wait_for_unit("glance-api.service") - controllerVM.wait_for_unit("placement-api.service") controllerVM.wait_for_unit("neutron-server.service") controllerVM.wait_for_unit("nova-scheduler.service") controllerVM.wait_for_unit("nova-conductor.service") @@ -137,25 +136,22 @@ pkgs.testers.nixosTest { # namespace dedicated for the VM to ping it. assert retry_until_succeed(controllerVM, f"ip netns exec {net_ns} ping -c 1 {vm_ip}", 120) - # create volume with 4GB - controllerVM.execute("openstack volume create --size 4 test_vol") - # attach volume to VM - controllerVM.execute("openstack server add volume test_vm test_vol") - - _status, output = controllerVM.execute("openstack volume show test_vol") - print(f"openstack volume show test_vol: {output}") - time.sleep(3) - _status, output = controllerVM.execute("openstack volume show test_vol") - print(f"openstack volume show test_vol: {output}") - time.sleep(3) - _status, output = controllerVM.execute("openstack volume show test_vol") - print(f"openstack volume show test_vol: {output}") - time.sleep(3) - _status, output = controllerVM.execute("openstack volume show test_vol") - print(f"openstack volume show test_vol: {output}") - - # wait until volume is attached - assert retry_until_succeed(controllerVM, "openstack volume show test_vol -f value -c status | grep 'in-use'", 120) + # Volume creation is asynchronous; wait until it is ready to attach. + controllerVM.succeed("openstack volume create --size 4 test_vol") + assert retry_until_succeed( + controllerVM, + "openstack volume show test_vol -f value -c status | grep -Fx available", + 120, + ), "Volume test_vol did not become available" + + controllerVM.succeed("openstack server add volume test_vm test_vol") + + # Attachment is asynchronous too. + assert retry_until_succeed( + controllerVM, + "openstack volume show test_vol -f value -c status | grep -Fx in-use", + 120, + ), "Volume test_vol did not become attached" # add ssh host key to known_hosts retry_until_succeed(controllerVM, f"ip netns exec {net_ns} ssh-keyscan {vm_ip} > ~/.ssh/known_hosts", 60) diff --git a/tests/openstack-live-migration.nix b/tests/openstack-live-migration.nix index 0fed9dc..9c7f58b 100644 --- a/tests/openstack-live-migration.nix +++ b/tests/openstack-live-migration.nix @@ -250,7 +250,6 @@ pkgs.testers.nixosTest { start_all() controllerVM.wait_for_unit("glance-api.service") - controllerVM.wait_for_unit("placement-api.service") controllerVM.wait_for_unit("neutron-server.service") controllerVM.wait_for_unit("nova-scheduler.service") controllerVM.wait_for_unit("nova-conductor.service")