From aa451dcde83fea3aeacf359235f8cd8c56b9e479 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Wed, 14 Oct 2026 16:04:41 +0000 Subject: [PATCH] chore: Adds typing annotations to Coriolis Adds argument and return type annotations to Coriolis, inferred from runtime types, by using RightTyper and running the integration tests. Methodology: - collection: run the integration tests under RightTyper. - emission: `righttyper process --python 3.10 --type-depth-limit=1` - normalisation: `self`, `cls`, `*args`, `**kwargs` left without annotations. Container types are kept shallow. Removed `-> None` from `__init__`. --- coriolis/api/__init__.py | 10 +- coriolis/api/common.py | 8 +- coriolis/api/middleware/fault.py | 6 +- coriolis/api/v1/deployment_actions.py | 8 +- coriolis/api/v1/deployments.py | 18 +- coriolis/api/v1/diagnostics.py | 4 +- coriolis/api/v1/endpoint_actions.py | 6 +- ...ndpoint_destination_minion_pool_options.py | 4 +- .../api/v1/endpoint_destination_options.py | 4 +- coriolis/api/v1/endpoint_instances.py | 6 +- coriolis/api/v1/endpoint_inventory.py | 4 +- coriolis/api/v1/endpoint_networks.py | 4 +- .../v1/endpoint_source_minion_pool_options.py | 4 +- coriolis/api/v1/endpoint_source_options.py | 4 +- coriolis/api/v1/endpoint_storage.py | 4 +- coriolis/api/v1/endpoints.py | 22 +- coriolis/api/v1/minion_pool_actions.py | 14 +- coriolis/api/v1/minion_pools.py | 38 +- coriolis/api/v1/provider_schemas.py | 6 +- coriolis/api/v1/providers.py | 4 +- coriolis/api/v1/regions.py | 20 +- coriolis/api/v1/router.py | 8 +- coriolis/api/v1/services.py | 20 +- coriolis/api/v1/transfer_actions.py | 6 +- coriolis/api/v1/transfer_schedules.py | 24 +- .../v1/transfer_tasks_execution_actions.py | 8 +- coriolis/api/v1/transfer_tasks_executions.py | 16 +- coriolis/api/v1/transfers.py | 36 +- coriolis/api/v1/utils.py | 15 +- coriolis/api/v1/views/deployment_view.py | 6 +- coriolis/api/v1/views/diagnostic_view.py | 2 +- .../api/v1/views/endpoint_options_view.py | 14 +- .../api/v1/views/endpoint_resources_view.py | 8 +- coriolis/api/v1/views/endpoint_view.py | 8 +- coriolis/api/v1/views/minion_pool_view.py | 9 +- coriolis/api/v1/views/region_view.py | 6 +- coriolis/api/v1/views/service_view.py | 6 +- .../api/v1/views/transfer_schedule_view.py | 4 +- .../v1/views/transfer_tasks_execution_view.py | 8 +- coriolis/api/v1/views/transfer_view.py | 6 +- coriolis/api/v1/views/utils.py | 8 +- coriolis/api/wsgi.py | 125 ++- coriolis/conductor/rpc/client.py | 446 +++++++--- coriolis/conductor/rpc/server.py | 814 ++++++++++++------ coriolis/conductor/rpc/utils.py | 14 +- coriolis/conf.py | 2 +- coriolis/context.py | 28 +- coriolis/cron/cron.py | 39 +- coriolis/data_transfer.py | 10 +- coriolis/db/api.py | 362 +++++--- coriolis/db/sqlalchemy/migration.py | 6 +- coriolis/db/sqlalchemy/models.py | 27 +- coriolis/db/sqlalchemy/types.py | 40 +- coriolis/deployer_manager/rpc/client.py | 8 +- coriolis/deployer_manager/rpc/server.py | 12 +- coriolis/deployments/api.py | 37 +- coriolis/diagnostics/api.py | 2 +- coriolis/endpoint_options/api.py | 33 +- coriolis/endpoint_resources/api.py | 41 +- coriolis/endpoints/api.py | 53 +- coriolis/events.py | 29 +- coriolis/exception.py | 8 +- coriolis/i18n.py | 2 +- coriolis/keystone.py | 3 +- coriolis/licensing/client.py | 2 +- coriolis/minion_manager/rpc/client.py | 102 ++- coriolis/minion_manager/rpc/server.py | 259 ++++-- coriolis/minion_manager/rpc/tasks.py | 324 +++++-- coriolis/minion_manager/rpc/utils.py | 7 +- coriolis/minion_pools/api.py | 46 +- coriolis/osmorphing/base.py | 89 +- coriolis/osmorphing/centos.py | 4 +- coriolis/osmorphing/debian.py | 25 +- coriolis/osmorphing/manager.py | 33 +- coriolis/osmorphing/osdetect/base.py | 18 +- coriolis/osmorphing/osdetect/coreos.py | 4 +- coriolis/osmorphing/osdetect/debian.py | 4 +- coriolis/osmorphing/osdetect/manager.py | 10 +- coriolis/osmorphing/osdetect/openwrt.py | 4 +- coriolis/osmorphing/osdetect/suse.py | 3 +- coriolis/osmorphing/osmount/base.py | 78 +- coriolis/osmorphing/osmount/factory.py | 10 +- coriolis/osmorphing/osmount/luks_mixin.py | 65 +- coriolis/osmorphing/osmount/ubuntu.py | 4 +- coriolis/osmorphing/redhat.py | 31 +- coriolis/osmorphing/rocky.py | 6 +- coriolis/osmorphing/ubuntu.py | 2 +- coriolis/policies/base.py | 2 +- coriolis/policies/deployments.py | 4 +- coriolis/policies/diagnostics.py | 4 +- coriolis/policies/endpoints.py | 4 +- coriolis/policies/general.py | 4 +- coriolis/policies/minion_pools.py | 4 +- coriolis/policies/regions.py | 4 +- coriolis/policies/services.py | 4 +- coriolis/policies/transfer_schedules.py | 4 +- .../policies/transfer_tasks_executions.py | 4 +- coriolis/policies/transfers.py | 4 +- coriolis/policy.py | 14 +- coriolis/providers/api.py | 9 +- coriolis/providers/backup_writers.py | 172 ++-- coriolis/providers/factory.py | 9 +- coriolis/providers/provider_utils.py | 11 +- coriolis/providers/replicator.py | 146 ++-- coriolis/regions/api.py | 19 +- coriolis/rpc.py | 32 +- coriolis/scheduler/filters/base.py | 4 +- coriolis/scheduler/filters/trivial_filters.py | 22 +- coriolis/scheduler/rpc/client.py | 33 +- coriolis/scheduler/rpc/server.py | 24 +- coriolis/schemas.py | 10 +- coriolis/service.py | 11 +- coriolis/services/api.py | 21 +- coriolis/taskflow/base.py | 63 +- coriolis/taskflow/runner.py | 16 +- coriolis/taskflow/utils.py | 4 +- coriolis/tasks/base.py | 30 +- coriolis/tasks/factory.py | 4 +- coriolis/tasks/minion_pool_tasks.py | 316 ++++--- coriolis/tasks/osmorphing_tasks.py | 56 +- coriolis/tasks/replica_tasks.py | 381 +++++--- coriolis/transfer_cron/api.py | 41 +- coriolis/transfer_cron/rpc/client.py | 16 +- coriolis/transfer_cron/rpc/server.py | 32 +- coriolis/transfer_tasks_executions/api.py | 46 +- coriolis/transfers/api.py | 66 +- coriolis/utils.py | 194 +++-- coriolis/worker/rpc/client.py | 174 +++- coriolis/worker/rpc/server.py | 154 +++- 129 files changed, 3883 insertions(+), 1976 deletions(-) diff --git a/coriolis/api/__init__.py b/coriolis/api/__init__.py index 6351c1bd6..456e454de 100644 --- a/coriolis/api/__init__.py +++ b/coriolis/api/__init__.py @@ -35,13 +35,13 @@ def root_app_factory(loader, global_conf, **local_conf): class APIMapper(routes.Mapper): - def routematch(self, url=None, environ=None): + def routematch(self, url: str | None = None, environ=None): if url is "": result = self._match("", environ) return result[0], result[1] return routes.Mapper.routematch(self, url, environ) - def connect(self, *args, **kwargs): + def connect(self, *args, **kwargs) -> None: # NOTE(inhye): Default the format part of a route to only accept json # and xml so it doesn't eat all characters after a '.' # in the url. @@ -88,7 +88,7 @@ def __init__(self, ext_mgr=None): self._setup_extensions(ext_mgr) super(APIRouter, self).__init__(mapper) - def _setup_ext_routes(self, mapper, ext_mgr): + def _setup_ext_routes(self, mapper: APIMapper, ext_mgr) -> None: for resource in ext_mgr.get_resources(): LOG.debug('Extended resource: %s', resource.collection) @@ -108,7 +108,7 @@ def _setup_ext_routes(self, mapper, ext_mgr): if resource.custom_routes_fn: resource.custom_routes_fn(mapper, wsgi_resource) - def _setup_extensions(self, ext_mgr): + def _setup_extensions(self, ext_mgr) -> None: for extension in ext_mgr.get_controller_extensions(): collection = extension.collection controller = extension.controller @@ -132,5 +132,5 @@ def _setup_extensions(self, ext_mgr): resource.register_actions(controller) resource.register_extensions(controller) - def _setup_routes(self, mapper, ext_mgr): + def _setup_routes(self, mapper: APIMapper, ext_mgr): raise NotImplementedError diff --git a/coriolis/api/common.py b/coriolis/api/common.py index 864224650..c37a35e3a 100644 --- a/coriolis/api/common.py +++ b/coriolis/api/common.py @@ -4,7 +4,7 @@ from coriolis import utils -def get_paging_params(req): +def get_paging_params(req) -> tuple: marker = req.GET.get("marker") limit = req.GET.get("limit") if limit is not None: @@ -13,8 +13,10 @@ def get_paging_params(req): def get_sort_params( - req, default_keys=('created_at', 'id'), default_dirs=('desc', 'desc') -): + req, + default_keys: tuple[str, str] = ('created_at', 'id'), + default_dirs: tuple[str, str] = ('desc', 'desc'), +) -> tuple[list, list]: """Retrieves sort keys/directions parameters. Processes the parameters to create a list of sort keys and sort directions diff --git a/coriolis/api/middleware/fault.py b/coriolis/api/middleware/fault.py index 1cadcc6ac..d1e4aa791 100644 --- a/coriolis/api/middleware/fault.py +++ b/coriolis/api/middleware/fault.py @@ -32,7 +32,7 @@ class FaultWrapper(wsgi.Middleware): _status_to_type = {} @staticmethod - def status_to_type(status): + def status_to_type(status: int) -> webob.exc.HTTPNotFound: if not FaultWrapper._status_to_type: for clazz in utils.walk_class_hierarchy(webob.exc.HTTPError): FaultWrapper._status_to_type[clazz.code] = clazz @@ -40,7 +40,7 @@ def status_to_type(status): status, webob.exc.HTTPInternalServerError )() - def _error(self, inner, req): + def _error(self, inner: exception.NotFound, req: wsgi.Request) -> wsgi.Fault: LOG.exception( _LE("Caught error: %(type)s %(error)s"), {'type': type(inner), 'error': inner}, @@ -67,7 +67,7 @@ def _error(self, inner, req): return wsgi.Fault(outer) @webob.dec.wsgify(RequestClass=wsgi.Request) - def __call__(self, req): + def __call__(self, req: wsgi.Request) -> wsgi.Fault | webob.Response: try: return req.get_response(self.application) except Exception as ex: diff --git a/coriolis/api/v1/deployment_actions.py b/coriolis/api/v1/deployment_actions.py index 7e88c2d3a..084b0fa3c 100644 --- a/coriolis/api/v1/deployment_actions.py +++ b/coriolis/api/v1/deployment_actions.py @@ -1,6 +1,8 @@ # Copyright 2024 Cloudbase Solutions Srl # All Rights Reserved. +from typing import NoReturn + from webob import exc from coriolis import exception @@ -15,7 +17,9 @@ def __init__(self): super(DeploymentActionsController, self).__init__() @api_wsgi.action('cancel') - def _cancel(self, req, id, body): + def _cancel( + self, req: api_wsgi.Request, id: str, body: dict[str, dict] + ) -> NoReturn: context = req.environ['coriolis.context'] context.can(deployment_policies.get_deployments_policy_label("cancel")) try: @@ -29,5 +33,5 @@ def _cancel(self, req, id, body): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(DeploymentActionsController()) diff --git a/coriolis/api/v1/deployments.py b/coriolis/api/v1/deployments.py index 20809e667..6e707e380 100644 --- a/coriolis/api/v1/deployments.py +++ b/coriolis/api/v1/deployments.py @@ -1,6 +1,8 @@ # Copyright 2024 Cloudbase Solutions Srl # All Rights Reserved. +from typing import NoReturn + from oslo_log import log as logging from webob import exc @@ -22,7 +24,7 @@ def __init__(self): self._endpoints_api = endpoints_api.API() super(DeploymentsController, self).__init__() - def show(self, req, id): + def show(self, req: api_wsgi.Request, id: str) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(deployment_policies.get_deployments_policy_label("show")) include_task_info = api_utils.get_bool_url_arg( @@ -36,7 +38,7 @@ def show(self, req, id): return deployment_view.single(deployment) - def _get_filters(self, req) -> dict: + def _get_filters(self, req: api_wsgi.Request) -> dict: filters = {} # For simplicity and consistency, we'll use "status" to search for a # given "last_execution_status". @@ -49,7 +51,7 @@ def _get_filters(self, req) -> dict: filters["status"] = status return filters - def _list(self, req): + def _list(self, req: api_wsgi.Request) -> dict[str, list]: show_deleted = api_utils.get_bool_url_arg(req, "show_deleted", default=False) context = req.environ["coriolis.context"] context.show_deleted = show_deleted @@ -75,14 +77,14 @@ def _list(self, req): ) ) - def index(self, req): + def index(self, req: api_wsgi.Request) -> dict[str, list]: return self._list(req) def detail(self, req): return self._list(req) @api_utils.format_keyerror_message(resource='deployment', method='create') - def _validate_deployment_input(self, context, body): + def _validate_deployment_input(self, context, body) -> tuple: deployment = body["deployment"] transfer_id = deployment.get("transfer_id", "") @@ -111,7 +113,7 @@ def _validate_deployment_input(self, context, body): user_scripts, ) - def create(self, req, body): + def create(self, req: api_wsgi.Request, body: dict[str, dict]) -> dict[str, dict]: context = req.environ['coriolis.context'] context.can(deployment_policies.get_deployments_policy_label("create")) @@ -138,7 +140,7 @@ def create(self, req, body): return deployment_view.single(deployment) - def delete(self, req, id): + def delete(self, req: api_wsgi.Request, id: str) -> NoReturn: context = req.environ['coriolis.context'] context.can(deployment_policies.get_deployments_policy_label("delete")) try: @@ -148,5 +150,5 @@ def delete(self, req, id): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(DeploymentsController()) diff --git a/coriolis/api/v1/diagnostics.py b/coriolis/api/v1/diagnostics.py index a0a78a95e..d1ea434bd 100644 --- a/coriolis/api/v1/diagnostics.py +++ b/coriolis/api/v1/diagnostics.py @@ -16,12 +16,12 @@ def __init__(self): self._diag_api = api.API() super(DiagnosticsController, self).__init__() - def index(self, req): + def index(self, req: api_wsgi.Request) -> dict[str, list]: context = req.environ['coriolis.context'] context.can(diagnostics.get_diagnostics_policy_label("get")) return diagnostic_view.collection(self._diag_api.get(context)) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(DiagnosticsController()) diff --git a/coriolis/api/v1/endpoint_actions.py b/coriolis/api/v1/endpoint_actions.py index f04c6b8b3..f9dca05a0 100644 --- a/coriolis/api/v1/endpoint_actions.py +++ b/coriolis/api/v1/endpoint_actions.py @@ -15,7 +15,9 @@ def __init__(self): super(EndpointActionsController, self).__init__() @api_wsgi.action('validate-connection') - def _validate_connection(self, req, id, body): + def _validate_connection( + self, req: api_wsgi.Request, id: str, body: dict[str, None] + ) -> dict[str, dict]: context = req.environ['coriolis.context'] context.can( "%s:validate_connection" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX) @@ -29,5 +31,5 @@ def _validate_connection(self, req, id, body): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(EndpointActionsController()) diff --git a/coriolis/api/v1/endpoint_destination_minion_pool_options.py b/coriolis/api/v1/endpoint_destination_minion_pool_options.py index 9f3812060..52c29e43c 100644 --- a/coriolis/api/v1/endpoint_destination_minion_pool_options.py +++ b/coriolis/api/v1/endpoint_destination_minion_pool_options.py @@ -17,7 +17,7 @@ def __init__(self): self._minion_pool_options_api = api.API() super(EndpointDestinationMinionPoolOptionsController, self).__init__() - def index(self, req, endpoint_id): + def index(self, req: api_wsgi.Request, endpoint_id: str) -> dict[str, list]: context = req.environ['coriolis.context'] context.can( "%s:list_destination_minion_pool_options" @@ -43,5 +43,5 @@ def index(self, req, endpoint_id): ) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(EndpointDestinationMinionPoolOptionsController()) diff --git a/coriolis/api/v1/endpoint_destination_options.py b/coriolis/api/v1/endpoint_destination_options.py index 664787f1e..16d31bea4 100644 --- a/coriolis/api/v1/endpoint_destination_options.py +++ b/coriolis/api/v1/endpoint_destination_options.py @@ -17,7 +17,7 @@ def __init__(self): self._destination_options_api = api.API() super(EndpointDestinationOptionsController, self).__init__() - def index(self, req, endpoint_id): + def index(self, req: api_wsgi.Request, endpoint_id: str) -> dict[str, list]: context = req.environ['coriolis.context'] context.can( "%s:list_destination_options" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX) @@ -42,5 +42,5 @@ def index(self, req, endpoint_id): ) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(EndpointDestinationOptionsController()) diff --git a/coriolis/api/v1/endpoint_instances.py b/coriolis/api/v1/endpoint_instances.py index 79470f313..9f7537d62 100644 --- a/coriolis/api/v1/endpoint_instances.py +++ b/coriolis/api/v1/endpoint_instances.py @@ -19,7 +19,7 @@ def __init__(self): self._instance_api = api.API() super(EndpointInstanceController, self).__init__() - def index(self, req, endpoint_id): + def index(self, req: api_wsgi.Request, endpoint_id: str) -> dict[str, list]: context = req.environ['coriolis.context'] context.can("%s:list_instances" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX)) marker, limit = common.get_paging_params(req) @@ -44,7 +44,7 @@ def index(self, req, endpoint_id): ) ) - def show(self, req, endpoint_id, id): + def show(self, req: api_wsgi.Request, endpoint_id: str, id: str) -> dict[str, dict]: context = req.environ['coriolis.context'] context.can("%s:get_instance" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX)) @@ -65,5 +65,5 @@ def show(self, req, endpoint_id, id): ) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(EndpointInstanceController()) diff --git a/coriolis/api/v1/endpoint_inventory.py b/coriolis/api/v1/endpoint_inventory.py index 9bdaf7462..3dbc8ffca 100644 --- a/coriolis/api/v1/endpoint_inventory.py +++ b/coriolis/api/v1/endpoint_inventory.py @@ -18,7 +18,7 @@ def __init__(self): self._endpoint_resources_api = api.API() super(EndpointInventoryController, self).__init__() - def index(self, req, endpoint_id): + def index(self, req: api_wsgi.Request, endpoint_id: str) -> api_wsgi.ResponseObject: context = req.environ['coriolis.context'] context.can("%s:export_inventory" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX)) @@ -35,5 +35,5 @@ def index(self, req, endpoint_id): return api_wsgi.ResponseObject(csv_content) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(EndpointInventoryController()) diff --git a/coriolis/api/v1/endpoint_networks.py b/coriolis/api/v1/endpoint_networks.py index 908a0fe4b..c0ee141ff 100644 --- a/coriolis/api/v1/endpoint_networks.py +++ b/coriolis/api/v1/endpoint_networks.py @@ -17,7 +17,7 @@ def __init__(self): self._network_api = api.API() super(EndpointNetworkController, self).__init__() - def index(self, req, endpoint_id): + def index(self, req: api_wsgi.Request, endpoint_id: str) -> dict[str, list]: context = req.environ['coriolis.context'] context.can("%s:list_networks" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX)) env = req.GET.get("env") @@ -31,5 +31,5 @@ def index(self, req, endpoint_id): ) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(EndpointNetworkController()) diff --git a/coriolis/api/v1/endpoint_source_minion_pool_options.py b/coriolis/api/v1/endpoint_source_minion_pool_options.py index e85c61e3d..2877afa4a 100644 --- a/coriolis/api/v1/endpoint_source_minion_pool_options.py +++ b/coriolis/api/v1/endpoint_source_minion_pool_options.py @@ -17,7 +17,7 @@ def __init__(self): self._minion_pool_options_api = api.API() super(EndpointSourceMinionPoolOptionsController, self).__init__() - def index(self, req, endpoint_id): + def index(self, req: api_wsgi.Request, endpoint_id: str) -> dict[str, list]: context = req.environ['coriolis.context'] context.can( "%s:list_source_minion_pool_options" @@ -43,5 +43,5 @@ def index(self, req, endpoint_id): ) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(EndpointSourceMinionPoolOptionsController()) diff --git a/coriolis/api/v1/endpoint_source_options.py b/coriolis/api/v1/endpoint_source_options.py index 1b0850984..6a503a3f2 100644 --- a/coriolis/api/v1/endpoint_source_options.py +++ b/coriolis/api/v1/endpoint_source_options.py @@ -17,7 +17,7 @@ def __init__(self): self._source_options_api = api.API() super(EndpointSourceOptionsController, self).__init__() - def index(self, req, endpoint_id): + def index(self, req: api_wsgi.Request, endpoint_id: str) -> dict[str, list]: context = req.environ['coriolis.context'] context.can( "%s:list_source_options" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX) @@ -42,5 +42,5 @@ def index(self, req, endpoint_id): ) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(EndpointSourceOptionsController()) diff --git a/coriolis/api/v1/endpoint_storage.py b/coriolis/api/v1/endpoint_storage.py index d6a7f85da..1e5480b4c 100644 --- a/coriolis/api/v1/endpoint_storage.py +++ b/coriolis/api/v1/endpoint_storage.py @@ -17,7 +17,7 @@ def __init__(self): self._storage_api = api.API() super(EndpointStorageController, self).__init__() - def index(self, req, endpoint_id): + def index(self, req: api_wsgi.Request, endpoint_id: str) -> dict[str, dict]: context = req.environ['coriolis.context'] context.can("%s:list_storage" % (endpoint_policies.ENDPOINTS_POLICY_PREFIX)) env = req.GET.get("env") @@ -31,5 +31,5 @@ def index(self, req, endpoint_id): ) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(EndpointStorageController()) diff --git a/coriolis/api/v1/endpoints.py b/coriolis/api/v1/endpoints.py index a2a1a97be..eafa7ad26 100644 --- a/coriolis/api/v1/endpoints.py +++ b/coriolis/api/v1/endpoints.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import NoReturn + from oslo_log import log as logging from webob import exc @@ -19,7 +21,7 @@ def __init__(self): self._endpoint_api = api.API() super(EndpointController, self).__init__() - def show(self, req, id): + def show(self, req: api_wsgi.Request, id: str) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(endpoint_policies.get_endpoints_policy_label("show")) endpoint = self._endpoint_api.get_endpoint(context, id) @@ -28,13 +30,15 @@ def show(self, req, id): return endpoint_view.single(endpoint) - def index(self, req): + def index(self, req: api_wsgi.Request) -> dict[str, list]: context = req.environ["coriolis.context"] context.can(endpoint_policies.get_endpoints_policy_label("list")) return endpoint_view.collection(self._endpoint_api.get_endpoints(context)) @api_utils.format_keyerror_message(resource='endpoint', method='create') - def _validate_create_body(self, body): + def _validate_create_body( + self, body: dict[str, dict] + ) -> tuple[str, str, str, dict, list]: endpoint = body["endpoint"] name = endpoint["name"] description = endpoint.get("description") @@ -43,7 +47,7 @@ def _validate_create_body(self, body): mapped_regions = endpoint.get("mapped_regions", []) return (name, endpoint_type, description, connection_info, mapped_regions) - def create(self, req, body): + def create(self, req: api_wsgi.Request, body: dict[str, dict]) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(endpoint_policies.get_endpoints_policy_label("create")) (name, endpoint_type, description, connection_info, mapped_regions) = ( @@ -61,7 +65,7 @@ def create(self, req, body): ) @api_utils.format_keyerror_message(resource='endpoint', method='update') - def _validate_update_body(self, body): + def _validate_update_body(self, body: dict[str, dict]) -> dict | tuple: endpoint = body["endpoint"] return { k: endpoint[k] @@ -69,7 +73,9 @@ def _validate_update_body(self, body): & {"name", "description", "connection_info", "mapped_regions"} } - def update(self, req, id, body): + def update( + self, req: api_wsgi.Request, id: str, body: dict[str, dict] + ) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(endpoint_policies.get_endpoints_policy_label("update")) updated_values = self._validate_update_body(body) @@ -79,7 +85,7 @@ def update(self, req, id, body): ) ) - def delete(self, req, id): + def delete(self, req: api_wsgi.Request, id: str) -> NoReturn: context = req.environ["coriolis.context"] context.can(endpoint_policies.get_endpoints_policy_label("delete")) try: @@ -89,5 +95,5 @@ def delete(self, req, id): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(EndpointController()) diff --git a/coriolis/api/v1/minion_pool_actions.py b/coriolis/api/v1/minion_pool_actions.py index b377b1eb9..dbe73018f 100644 --- a/coriolis/api/v1/minion_pool_actions.py +++ b/coriolis/api/v1/minion_pool_actions.py @@ -16,7 +16,9 @@ def __init__(self): super(MinionPoolActionsController, self).__init__() @api_wsgi.action('allocate') - def _allocate_pool(self, req, id, body): + def _allocate_pool( + self, req: api_wsgi.Request, id: str, body: dict[str, None] + ) -> dict[str, dict]: context = req.environ['coriolis.context'] context.can(minion_pool_policies.get_minion_pools_policy_label("allocate")) try: @@ -29,7 +31,9 @@ def _allocate_pool(self, req, id, body): raise exc.HTTPNotFound(explanation=ex.msg) @api_wsgi.action('refresh') - def _refresh_pool(self, req, id, body): + def _refresh_pool( + self, req: api_wsgi.Request, id: str, body: dict[str, None] + ) -> dict[str, dict]: context = req.environ['coriolis.context'] context.can(minion_pool_policies.get_minion_pools_policy_label("refresh")) try: @@ -42,7 +46,9 @@ def _refresh_pool(self, req, id, body): raise exc.HTTPNotFound(explanation=ex.msg) @api_wsgi.action('deallocate') - def _deallocate_pool(self, req, id, body): + def _deallocate_pool( + self, req: api_wsgi.Request, id: str, body: dict[str, dict] + ) -> dict[str, dict]: context = req.environ['coriolis.context'] context.can(minion_pool_policies.get_minion_pools_policy_label("deallocate")) force = (body["deallocate"] or {}).get("force", False) @@ -56,5 +62,5 @@ def _deallocate_pool(self, req, id, body): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(MinionPoolActionsController()) diff --git a/coriolis/api/v1/minion_pools.py b/coriolis/api/v1/minion_pools.py index ac7df5ce7..ddceed268 100644 --- a/coriolis/api/v1/minion_pools.py +++ b/coriolis/api/v1/minion_pools.py @@ -1,6 +1,8 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any, NoReturn + from oslo_log import log as logging from webob import exc @@ -21,7 +23,7 @@ def __init__(self): self._endpoints_api = endpoints_api.API() super(MinionPoolController, self).__init__() - def show(self, req, id): + def show(self, req: api_wsgi.Request, id: str) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(pools_policies.get_minion_pools_policy_label("show")) minion_pool = self._minion_pool_api.get_minion_pool(context, id) @@ -30,14 +32,14 @@ def show(self, req, id): return minion_pool_view.single(minion_pool) - def index(self, req): + def index(self, req: api_wsgi.Request) -> dict[str, list]: context = req.environ["coriolis.context"] context.can(pools_policies.get_minion_pools_policy_label("list")) return minion_pool_view.collection( self._minion_pool_api.get_minion_pools(context) ) - def _check_pool_retention_strategy(self, pool_retention_strategy): + def _check_pool_retention_strategy(self, pool_retention_strategy: str) -> None: if not pool_retention_strategy: LOG.debug( "Ignoring void minion pool retention strategy '%s'", @@ -54,8 +56,11 @@ def _check_pool_retention_strategy(self, pool_retention_strategy): ) def _check_pool_numeric_values( - self, minimum_minions, maximum_minions, minion_max_idle_time - ): + self, + minimum_minions: int, + maximum_minions: int, + minion_max_idle_time: int | None, + ) -> None: if minimum_minions is not None: if minimum_minions <= 0: raise Exception( @@ -82,7 +87,9 @@ def _check_pool_numeric_values( ) @api_utils.format_keyerror_message(resource='minion_pool', method='create') - def _validate_create_body(self, ctxt, body): + def _validate_create_body( + self, ctxt, body: dict[str, dict] + ) -> tuple[str, str, str, str, dict, int, int, int, str, None, bool]: minion_pool = body["minion_pool"] name = minion_pool["name"] endpoint_id = minion_pool["endpoint_id"] @@ -147,7 +154,7 @@ def _validate_create_body(self, ctxt, body): skip_allocation, ) - def create(self, req, body): + def create(self, req: api_wsgi.Request, body: dict[str, dict]) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(pools_policies.get_minion_pools_policy_label("create")) ( @@ -181,8 +188,11 @@ def create(self, req, body): ) def _validate_updated_environment_options( - self, context, minion_pool, environment_options - ): + self, + context, + minion_pool, + environment_options: dict[Any, Any], + ) -> None: if minion_pool['platform'] == (constants.PROVIDER_PLATFORM_SOURCE): (self._endpoints_api.validate_endpoint_source_minion_pool_options)( context, minion_pool['endpoint_id'], environment_options @@ -195,7 +205,7 @@ def _validate_updated_environment_options( raise Exception("Unknown pool platform: %s" % minion_pool['platform']) @api_utils.format_keyerror_message(resource='minion_pool', method='update') - def _validate_update_body(self, id, context, body): + def _validate_update_body(self, id, context, body) -> dict | tuple: minion_pool = body["minion_pool"] if 'endpoint_id' in minion_pool: raise Exception("The 'endpoint_id' of a minion pool cannot be updated.") @@ -241,7 +251,9 @@ def _validate_update_body(self, id, context, body): ) return vals - def update(self, req, id, body): + def update( + self, req: api_wsgi.Request, id: str, body: dict[str, dict] + ) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(pools_policies.get_minion_pools_policy_label("update")) updated_values = self._validate_update_body(id, context, body) @@ -251,7 +263,7 @@ def update(self, req, id, body): ) ) - def delete(self, req, id): + def delete(self, req: api_wsgi.Request, id: str) -> NoReturn: context = req.environ["coriolis.context"] context.can(pools_policies.get_minion_pools_policy_label("delete")) try: @@ -261,5 +273,5 @@ def delete(self, req, id): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(MinionPoolController()) diff --git a/coriolis/api/v1/provider_schemas.py b/coriolis/api/v1/provider_schemas.py index 92c02973c..9f7db9e13 100644 --- a/coriolis/api/v1/provider_schemas.py +++ b/coriolis/api/v1/provider_schemas.py @@ -14,7 +14,9 @@ def __init__(self): self._provider_api = api.API() super(ProviderSchemasController, self).__init__() - def index(self, req, platform_name, provider_type): + def index( + self, req: api_wsgi.Request, platform_name: str, provider_type: str + ) -> dict[str, dict]: return { "schemas": self._provider_api.get_provider_schemas( req.environ["coriolis.context"], platform_name, provider_type @@ -22,5 +24,5 @@ def index(self, req, platform_name, provider_type): } -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(ProviderSchemasController()) diff --git a/coriolis/api/v1/providers.py b/coriolis/api/v1/providers.py index b54623582..981be78ac 100644 --- a/coriolis/api/v1/providers.py +++ b/coriolis/api/v1/providers.py @@ -15,11 +15,11 @@ def __init__(self): self._provider_api = api.API() super(ProviderController, self).__init__() - def index(self, req): + def index(self, req: api_wsgi.Request) -> dict[str, dict]: context = req.environ['coriolis.context'] context.can(general_policies.get_providers_policy_label('list')) return {"providers": self._provider_api.get_available_providers(context)} -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(ProviderController()) diff --git a/coriolis/api/v1/regions.py b/coriolis/api/v1/regions.py index 770b3398d..41561ef91 100644 --- a/coriolis/api/v1/regions.py +++ b/coriolis/api/v1/regions.py @@ -1,6 +1,8 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from typing import NoReturn + from oslo_log import log as logging from webob import exc @@ -19,7 +21,7 @@ def __init__(self): self._region_api = api.API() super(RegionController, self).__init__() - def show(self, req, id): + def show(self, req: api_wsgi.Request, id: str) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(region_policies.get_regions_policy_label("show")) region = self._region_api.get_region(context, id) @@ -28,20 +30,20 @@ def show(self, req, id): return region_view.single(region) - def index(self, req): + def index(self, req: api_wsgi.Request) -> dict[str, list]: context = req.environ["coriolis.context"] context.can(region_policies.get_regions_policy_label("list")) return region_view.collection(self._region_api.get_regions(context)) @api_utils.format_keyerror_message(resource='region', method='create') - def _validate_create_body(self, body): + def _validate_create_body(self, body: dict[str, dict]) -> tuple[str, str, bool]: region = body["region"] name = region["name"] description = region.get("description", "") enabled = region.get("enabled", True) return name, description, enabled - def create(self, req, body): + def create(self, req: api_wsgi.Request, body: dict[str, dict]) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(region_policies.get_regions_policy_label("create")) (name, description, enabled) = self._validate_create_body(body) @@ -52,13 +54,15 @@ def create(self, req, body): ) @api_utils.format_keyerror_message(resource='region', method='update') - def _validate_update_body(self, body): + def _validate_update_body(self, body: dict[str, dict]) -> dict[str, str]: region = body["region"] return { k: region[k] for k in region.keys() & {"name", "description", "enabled"} } - def update(self, req, id, body): + def update( + self, req: api_wsgi.Request, id: str, body: dict[str, dict] + ) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(region_policies.get_regions_policy_label("update")) updated_values = self._validate_update_body(body) @@ -66,7 +70,7 @@ def update(self, req, id, body): self._region_api.update(req.environ['coriolis.context'], id, updated_values) ) - def delete(self, req, id): + def delete(self, req: api_wsgi.Request, id: str) -> NoReturn: context = req.environ["coriolis.context"] context.can(region_policies.get_regions_policy_label("delete")) try: @@ -76,5 +80,5 @@ def delete(self, req, id): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(RegionController()) diff --git a/coriolis/api/v1/router.py b/coriolis/api/v1/router.py index 53497a536..b494d11fd 100644 --- a/coriolis/api/v1/router.py +++ b/coriolis/api/v1/router.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from oslo_log import log as logging from coriolis import api @@ -35,17 +37,17 @@ class ExtensionManager(object): - def get_resources(self): + def get_resources(self) -> list[Any]: return [] - def get_controller_extensions(self): + def get_controller_extensions(self) -> list[Any]: return [] class APIRouter(api.APIRouter): ExtensionManager = ExtensionManager - def _setup_routes(self, mapper, ext_mgr): + def _setup_routes(self, mapper: api.APIMapper, ext_mgr: ExtensionManager) -> None: mapper.redirect("", "/") self.resources['providers'] = providers.create_resource() diff --git a/coriolis/api/v1/services.py b/coriolis/api/v1/services.py index 92b993419..690354b7a 100644 --- a/coriolis/api/v1/services.py +++ b/coriolis/api/v1/services.py @@ -1,6 +1,8 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from typing import NoReturn + from oslo_log import log as logging from webob import exc @@ -19,7 +21,7 @@ def __init__(self): self._service_api = api.API() super(ServiceController, self).__init__() - def show(self, req, id): + def show(self, req: api_wsgi.Request, id: str) -> dict: context = req.environ["coriolis.context"] context.can(service_policies.get_services_policy_label("show")) service = self._service_api.get_service(context, id) @@ -28,13 +30,13 @@ def show(self, req, id): return service_view.single(service) - def index(self, req): + def index(self, req: api_wsgi.Request) -> dict[str, list]: context = req.environ["coriolis.context"] context.can(service_policies.get_services_policy_label("list")) return service_view.collection(self._service_api.get_services(context)) @api_utils.format_keyerror_message(resource='service', method='create') - def _validate_create_body(self, body): + def _validate_create_body(self, body: dict[str, dict]) -> dict | tuple: service = body["service"] host = service["host"] binary = service["binary"] @@ -43,7 +45,7 @@ def _validate_create_body(self, body): enabled = service.get("enabled", True) return host, binary, topic, mapped_regions, enabled - def create(self, req, body): + def create(self, req: api_wsgi.Request, body: dict[str, dict]) -> dict: context = req.environ["coriolis.context"] context.can(service_policies.get_services_policy_label("create")) (host, binary, topic, mapped_regions, enabled) = self._validate_create_body( @@ -61,11 +63,13 @@ def create(self, req, body): ) @api_utils.format_keyerror_message(resource='service', method='update') - def _validate_update_body(self, body): + def _validate_update_body(self, body: dict[str, dict]) -> dict: service = body["service"] return {k: service[k] for k in service.keys() & {"enabled", "mapped_regions"}} - def update(self, req, id, body): + def update( + self, req: api_wsgi.Request, id: str, body: dict[str, dict] + ) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(service_policies.get_services_policy_label("update")) updated_values = self._validate_update_body(body) @@ -75,7 +79,7 @@ def update(self, req, id, body): ) ) - def delete(self, req, id): + def delete(self, req: api_wsgi.Request, id: str) -> NoReturn: context = req.environ["coriolis.context"] context.can(service_policies.get_services_policy_label("delete")) try: @@ -85,5 +89,5 @@ def delete(self, req, id): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(ServiceController()) diff --git a/coriolis/api/v1/transfer_actions.py b/coriolis/api/v1/transfer_actions.py index c5f2b230d..07c6a6546 100644 --- a/coriolis/api/v1/transfer_actions.py +++ b/coriolis/api/v1/transfer_actions.py @@ -16,7 +16,9 @@ def __init__(self): super(TransferActionsController, self).__init__() @api_wsgi.action('delete-disks') - def _delete_disks(self, req, id, body): + def _delete_disks( + self, req: api_wsgi.Request, id: str, body: dict[str, None] + ) -> dict[str, dict]: context = req.environ['coriolis.context'] context.can(transfer_policies.get_transfers_policy_label("delete_disks")) try: @@ -29,5 +31,5 @@ def _delete_disks(self, req, id, body): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(TransferActionsController()) diff --git a/coriolis/api/v1/transfer_schedules.py b/coriolis/api/v1/transfer_schedules.py index 2e0964440..a0fdc1278 100644 --- a/coriolis/api/v1/transfer_schedules.py +++ b/coriolis/api/v1/transfer_schedules.py @@ -1,6 +1,8 @@ # Copyright 2017 Cloudbase Solutions Srl # All Rights Reserved. +from typing import NoReturn + import jsonschema from oslo_log import log as logging from oslo_utils import strutils, timeutils @@ -20,7 +22,7 @@ def __init__(self): self._schedule_api = api.API() super(TransferScheduleController, self).__init__() - def show(self, req, transfer_id, id): + def show(self, req: api_wsgi.Request, transfer_id: str, id: str) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(schedules_policies.get_transfer_schedules_policy_label("show")) schedule = self._schedule_api.get_schedule(context, transfer_id, id) @@ -29,7 +31,7 @@ def show(self, req, transfer_id, id): return transfer_schedule_view.single(schedule) - def index(self, req, transfer_id): + def index(self, req: api_wsgi.Request, transfer_id: str) -> dict[str, list]: context = req.environ["coriolis.context"] context.can(schedules_policies.get_transfer_schedules_policy_label("list")) @@ -40,7 +42,7 @@ def index(self, req, transfer_id): self._schedule_api.get_schedules(context, transfer_id, expired=show_expired) ) - def _validate_schedule(self, schedule): + def _validate_schedule(self, schedule: dict[str, int]) -> dict[str, int]: schema = schemas.SCHEDULE_API_BODY_SCHEMA["properties"]["schedule"] schemas.validate_value(schedule, schema) return schedule @@ -54,7 +56,7 @@ def _validate_expiration_date(self, expiration_date): raise exception.InvalidInput("expiration_date is in the past") return exp - def _validate_create_body(self, body): + def _validate_create_body(self, body: dict) -> tuple[dict, bool, None, bool, bool]: schedule = body.get("schedule") if schedule is None: raise exception.InvalidInput("schedule is required") @@ -73,7 +75,7 @@ def _validate_create_body(self, body): auto_deploy = body.get('auto_deploy', False) return schedule, enabled, exp, shutdown, auto_deploy - def _validate_update_body(self, update_body): + def _validate_update_body(self, update_body: dict) -> dict: body = {} schedule = update_body.get("schedule") if schedule is not None: @@ -100,7 +102,9 @@ def _validate_update_body(self, update_body): body["expiration_date"] = exp return body - def create(self, req, transfer_id, body): + def create( + self, req: api_wsgi.Request, transfer_id: str, body: dict + ) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(schedules_policies.get_transfer_schedules_policy_label("create")) @@ -118,7 +122,9 @@ def create(self, req, transfer_id, body): ) ) - def update(self, req, transfer_id, id, body): + def update( + self, req: api_wsgi.Request, transfer_id: str, id: str, body: dict + ) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(schedules_policies.get_transfer_schedules_policy_label("update")) @@ -133,7 +139,7 @@ def update(self, req, transfer_id, id, body): self._schedule_api.update(context, transfer_id, id, update_values) ) - def delete(self, req, transfer_id, id): + def delete(self, req: api_wsgi.Request, transfer_id: str, id: str) -> NoReturn: context = req.environ["coriolis.context"] context.can(schedules_policies.get_transfer_schedules_policy_label("delete")) @@ -141,5 +147,5 @@ def delete(self, req, transfer_id, id): raise exc.HTTPNoContent() -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(TransferScheduleController()) diff --git a/coriolis/api/v1/transfer_tasks_execution_actions.py b/coriolis/api/v1/transfer_tasks_execution_actions.py index 1f81049ae..88bad55b6 100644 --- a/coriolis/api/v1/transfer_tasks_execution_actions.py +++ b/coriolis/api/v1/transfer_tasks_execution_actions.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import NoReturn + from webob import exc from coriolis import exception @@ -15,7 +17,9 @@ def __init__(self): super(TransferTasksExecutionActionsController, self).__init__() @api_wsgi.action('cancel') - def _cancel(self, req, transfer_id, id, body): + def _cancel( + self, req: api_wsgi.Request, transfer_id: str, id: str, body: dict[str, dict] + ) -> NoReturn: context = req.environ['coriolis.context'] context.can(execution_policies.get_transfer_executions_policy_label('cancel')) try: @@ -29,5 +33,5 @@ def _cancel(self, req, transfer_id, id, body): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(TransferTasksExecutionActionsController()) diff --git a/coriolis/api/v1/transfer_tasks_executions.py b/coriolis/api/v1/transfer_tasks_executions.py index 567943ddf..88443f35b 100644 --- a/coriolis/api/v1/transfer_tasks_executions.py +++ b/coriolis/api/v1/transfer_tasks_executions.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import NoReturn + from webob import exc from coriolis import constants, exception @@ -16,7 +18,7 @@ def __init__(self): self._transfer_tasks_execution_api = api.API() super(TransferTasksExecutionController, self).__init__() - def show(self, req, transfer_id, id): + def show(self, req: api_wsgi.Request, transfer_id: str, id: str) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(executions_policies.get_transfer_executions_policy_label("show")) execution = self._transfer_tasks_execution_api.get_execution( @@ -27,7 +29,7 @@ def show(self, req, transfer_id, id): return transfer_tasks_execution_view.single(execution) - def _get_filters(self, req) -> dict: + def _get_filters(self, req: api_wsgi.Request) -> dict: filters = {} status = req.GET.get("status") if status is not None: @@ -38,7 +40,7 @@ def _get_filters(self, req) -> dict: filters["status"] = status return filters - def index(self, req, transfer_id): + def index(self, req: api_wsgi.Request, transfer_id: str) -> dict: context = req.environ["coriolis.context"] context.can(executions_policies.get_transfer_executions_policy_label("list")) @@ -69,7 +71,9 @@ def detail(self, req, transfer_id): ) ) - def create(self, req, transfer_id, body): + def create( + self, req: api_wsgi.Request, transfer_id: str, body: dict[str, dict] + ) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(executions_policies.get_transfer_executions_policy_label("create")) @@ -85,7 +89,7 @@ def create(self, req, transfer_id, body): ) ) - def delete(self, req, transfer_id, id): + def delete(self, req: api_wsgi.Request, transfer_id: str, id: str) -> NoReturn: context = req.environ["coriolis.context"] context.can(executions_policies.get_transfer_executions_policy_label("delete")) @@ -96,5 +100,5 @@ def delete(self, req, transfer_id, id): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(TransferTasksExecutionController()) diff --git a/coriolis/api/v1/transfers.py b/coriolis/api/v1/transfers.py index 352a8e624..a445cf22c 100644 --- a/coriolis/api/v1/transfers.py +++ b/coriolis/api/v1/transfers.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any, NoReturn + from oslo_log import log as logging from webob import exc @@ -27,7 +29,7 @@ def __init__(self): self._endpoints_api = endpoints_api.API() super(TransferController, self).__init__() - def show(self, req, id): + def show(self, req: api_wsgi.Request, id: str) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(transfer_policies.get_transfers_policy_label("show")) include_task_info = api_utils.get_bool_url_arg( @@ -41,7 +43,7 @@ def show(self, req, id): return transfer_view.single(transfer) - def _get_filters(self, req) -> dict: + def _get_filters(self, req: api_wsgi.Request) -> dict: filters = {} status = req.GET.get("status") if status is not None: @@ -50,7 +52,7 @@ def _get_filters(self, req) -> dict: filters["status"] = status return filters - def _list(self, req): + def _list(self, req: api_wsgi.Request) -> dict: show_deleted = api_utils.get_bool_url_arg(req, "show_deleted", default=False) context = req.environ["coriolis.context"] context.show_deleted = show_deleted @@ -74,14 +76,14 @@ def _list(self, req): ) ) - def index(self, req): + def index(self, req: api_wsgi.Request) -> dict[str, list]: return self._list(req) - def detail(self, req): + def detail(self, req: api_wsgi.Request) -> dict[str, list]: return self._list(req) @api_utils.format_keyerror_message(resource='transfer', method='create') - def _validate_create_body(self, context, body): + def _validate_create_body(self, context, body: dict) -> tuple: transfer = body["transfer"] scenario = transfer.get("scenario", "") @@ -173,7 +175,7 @@ def _validate_create_body(self, context, body): skip_os_morphing, ) - def create(self, req, body): + def create(self, req: api_wsgi.Request, body: dict) -> dict: context = req.environ["coriolis.context"] context.can(transfer_policies.get_transfers_policy_label("create")) @@ -216,7 +218,7 @@ def create(self, req, body): ) ) - def delete(self, req, id): + def delete(self, req: api_wsgi.Request, id: str) -> NoReturn: context = req.environ["coriolis.context"] context.can(transfer_policies.get_transfers_policy_label("delete")) try: @@ -226,7 +228,9 @@ def delete(self, req, id): raise exc.HTTPNotFound(explanation=ex.msg) @staticmethod - def _update_storage_mappings(original_storage_mappings, new_storage_mappings): + def _update_storage_mappings( + original_storage_mappings: dict[Any, Any], new_storage_mappings: dict[Any, Any] + ) -> dict[str, list]: backend_mappings = original_storage_mappings.get('backend_mappings', []) new_backend_mappings = new_storage_mappings.get('backend_mappings', []) @@ -270,7 +274,9 @@ def _update_storage_mappings(original_storage_mappings, new_storage_mappings): return storage_mappings @staticmethod - def _get_updated_user_scripts(original_user_scripts, new_user_scripts): + def _get_updated_user_scripts( + original_user_scripts: dict[Any, Any], new_user_scripts: dict[Any, Any] + ) -> dict[str, dict]: global_scripts = original_user_scripts.get('global', {}) new_global_scripts = new_user_scripts.get('global', {}) if new_global_scripts: @@ -288,7 +294,7 @@ def _get_updated_user_scripts(original_user_scripts, new_user_scripts): return user_scripts - def _get_merged_transfer_values(self, transfer, updated_values): + def _get_merged_transfer_values(self, transfer: dict, updated_values: dict) -> dict: """Looks for the following keys in the original transfer body and updated values (preferring the updated values where needed, but using `.update()` on dicts): @@ -373,7 +379,7 @@ def _get_merged_transfer_values(self, transfer, updated_values): return final_values @api_utils.format_keyerror_message(resource='transfer', method='update') - def _validate_update_body(self, id, context, body): + def _validate_update_body(self, id, context, body) -> dict | tuple: transfer = self._transfer_api.get_transfer(context, id) scenario = body.get("scenario", "") @@ -424,7 +430,9 @@ def _validate_update_body(self, id, context, body): return merged_body - def update(self, req, id, body): + def update( + self, req: api_wsgi.Request, id: str, body: dict[str, dict] + ) -> dict[str, dict]: context = req.environ["coriolis.context"] context.can(transfer_policies.get_transfers_policy_label("update")) @@ -441,5 +449,5 @@ def update(self, req, id, body): raise exc.HTTPNotFound(explanation=ex.msg) -def create_resource(): +def create_resource() -> api_wsgi.Resource: return api_wsgi.Resource(TransferController()) diff --git a/coriolis/api/v1/utils.py b/coriolis/api/v1/utils.py index d660c207f..02eef0505 100644 --- a/coriolis/api/v1/utils.py +++ b/coriolis/api/v1/utils.py @@ -2,6 +2,7 @@ # All Rights Reserved. import functools +from typing import Any, Callable from oslo_log import log as logging from oslo_utils import strutils @@ -12,7 +13,7 @@ LOG = logging.getLogger(__name__) -def get_bool_url_arg(req, arg_name, default=False): +def get_bool_url_arg(req, arg_name: str, default: bool = False) -> bool: val = req.GET.get(arg_name, default) try: return strutils.bool_from_string(str(val), strict=True) @@ -21,7 +22,7 @@ def get_bool_url_arg(req, arg_name, default=False): return default -def validate_network_map(network_map): +def validate_network_map(network_map: dict[Any, Any]) -> None: """Validates the JSON schema for the network_map.""" try: schemas.validate_value(network_map, schemas.CORIOLIS_NETWORK_MAP_SCHEMA) @@ -29,7 +30,7 @@ def validate_network_map(network_map): raise exc.HTTPBadRequest(explanation="Invalid network_map: %s" % str(ex)) -def validate_storage_mappings(storage_mappings): +def validate_storage_mappings(storage_mappings: dict[str, list]) -> None: """Validates the JSON schema for the storage_mappings.""" try: schemas.validate_value( @@ -39,8 +40,8 @@ def validate_storage_mappings(storage_mappings): raise exc.HTTPBadRequest(explanation="Invalid storage_mappings: %s" % str(ex)) -def format_keyerror_message(resource='', method=''): - def _format_keyerror_message(func): +def format_keyerror_message(resource: str = '', method: str = '') -> Callable: + def _format_keyerror_message(func: Callable) -> Callable: @functools.wraps(func) def _wrapper(*args, **kwargs): try: @@ -165,7 +166,7 @@ def _process_user_scripts( ) -def validate_user_scripts(user_scripts): +def validate_user_scripts(user_scripts: dict | None) -> dict: # Validate the top level user scripts dict. # Example: # { @@ -242,7 +243,7 @@ def validate_user_scripts(user_scripts): return user_scripts -def validate_instances_list_for_transfer(instances): +def validate_instances_list_for_transfer(instances: list[str]) -> list[str]: if not instances: raise exception.InvalidInput( "No instance identifiers provided for transfer action." diff --git a/coriolis/api/v1/views/deployment_view.py b/coriolis/api/v1/views/deployment_view.py index 382ef99bd..21cb54e1d 100644 --- a/coriolis/api/v1/views/deployment_view.py +++ b/coriolis/api/v1/views/deployment_view.py @@ -5,7 +5,7 @@ from coriolis.api.v1.views import utils as view_utils -def _format_deployment(deployment, keys=None): +def _format_deployment(deployment: dict, keys=None) -> dict: deployment_dict = view_utils.format_opt(deployment, keys) if len(deployment_dict.get("executions", [])): @@ -23,10 +23,10 @@ def _format_deployment(deployment, keys=None): return deployment_dict -def single(deployment, keys=None): +def single(deployment: dict, keys=None) -> dict[str, dict]: return {"deployment": _format_deployment(deployment, keys)} -def collection(deployments, keys=None): +def collection(deployments: list, keys=None) -> dict[str, list]: formatted_deployments = [_format_deployment(m, keys) for m in deployments] return {'deployments': formatted_deployments} diff --git a/coriolis/api/v1/views/diagnostic_view.py b/coriolis/api/v1/views/diagnostic_view.py index 60f7c0240..246689990 100644 --- a/coriolis/api/v1/views/diagnostic_view.py +++ b/coriolis/api/v1/views/diagnostic_view.py @@ -6,5 +6,5 @@ def single(diag): return {"diagnostic": diag} -def collection(diag): +def collection(diag: list[dict]) -> dict[str, list]: return {'diagnostics': diag} diff --git a/coriolis/api/v1/views/endpoint_options_view.py b/coriolis/api/v1/views/endpoint_options_view.py index c3391a99f..c3238cbef 100644 --- a/coriolis/api/v1/views/endpoint_options_view.py +++ b/coriolis/api/v1/views/endpoint_options_view.py @@ -4,23 +4,29 @@ from coriolis.api.v1.views import utils as view_utils -def destination_minion_pool_options_collection(destination_pool_options, keys=None): +def destination_minion_pool_options_collection( + destination_pool_options: list[dict], keys=None +) -> dict[str, list]: formatted_opts = [ view_utils.format_opt(opt, keys) for opt in destination_pool_options ] return {'destination_minion_pool_options': formatted_opts} -def destination_options_collection(destination_options, keys=None): +def destination_options_collection( + destination_options: list[dict], keys=None +) -> dict[str, list]: formatted_opts = [view_utils.format_opt(opt, keys) for opt in destination_options] return {'destination_options': formatted_opts} -def source_minion_pool_options_collection(source_pool_options, keys=None): +def source_minion_pool_options_collection( + source_pool_options: list[dict], keys=None +) -> dict[str, list]: formatted_opts = [view_utils.format_opt(opt, keys) for opt in source_pool_options] return {'source_minion_pool_options': formatted_opts} -def source_options_collection(source_options, keys=None): +def source_options_collection(source_options: list[dict], keys=None) -> dict[str, list]: formatted_opts = [view_utils.format_opt(opt, keys) for opt in source_options] return {'source_options': formatted_opts} diff --git a/coriolis/api/v1/views/endpoint_resources_view.py b/coriolis/api/v1/views/endpoint_resources_view.py index 8fb7a3310..02524e907 100644 --- a/coriolis/api/v1/views/endpoint_resources_view.py +++ b/coriolis/api/v1/views/endpoint_resources_view.py @@ -4,11 +4,11 @@ from coriolis.api.v1.views import utils as view_utils -def instance_single(instance, keys=None): +def instance_single(instance: dict, keys=None) -> dict[str, dict]: return {"instance": view_utils.format_opt(instance, keys)} -def instances_collection(instances, keys=None): +def instances_collection(instances: list[dict], keys=None) -> dict[str, list]: formatted_instances = [view_utils.format_opt(m, keys) for m in instances] return {'instances': formatted_instances} @@ -17,11 +17,11 @@ def network_single(network, keys=None): return {"network": view_utils.format_opt(network, keys)} -def networks_collection(networks, keys=None): +def networks_collection(networks: list[dict], keys=None) -> dict[str, list]: formatted_networks = [view_utils.format_opt(m, keys) for m in networks] return {'networks': formatted_networks} -def storage_collection(storage, keys=None): +def storage_collection(storage: dict[str, list], keys=None) -> dict[str, dict]: formatted_storages = view_utils.format_opt(storage, keys) return {'storage': formatted_storages} diff --git a/coriolis/api/v1/views/endpoint_view.py b/coriolis/api/v1/views/endpoint_view.py index 1b5d312e9..e178c0ae9 100644 --- a/coriolis/api/v1/views/endpoint_view.py +++ b/coriolis/api/v1/views/endpoint_view.py @@ -1,10 +1,12 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from coriolis.api.v1.views import utils as view_utils -def _format_endpoint(endpoint, keys=None): +def _format_endpoint(endpoint, keys=None) -> dict: endpoint_dict = view_utils.format_opt(endpoint, keys) mapped_regions = endpoint_dict.get('mapped_regions', []) endpoint_dict['mapped_regions'] = [reg['id'] for reg in mapped_regions] @@ -12,10 +14,10 @@ def _format_endpoint(endpoint, keys=None): return endpoint_dict -def single(endpoint, keys=None): +def single(endpoint, keys=None) -> dict[str, dict]: return {"endpoint": _format_endpoint(endpoint, keys)} -def collection(endpoints, keys=None): +def collection(endpoints: list[Any], keys=None) -> dict[str, list]: formatted_endpoints = [_format_endpoint(m, keys) for m in endpoints] return {'endpoints': formatted_endpoints} diff --git a/coriolis/api/v1/views/minion_pool_view.py b/coriolis/api/v1/views/minion_pool_view.py index 0db80a546..18dd9b3b8 100644 --- a/coriolis/api/v1/views/minion_pool_view.py +++ b/coriolis/api/v1/views/minion_pool_view.py @@ -1,13 +1,14 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. + from coriolis.api.v1.views import utils as view_utils -def _format_minion_pool(minion_pool, keys=None): +def _format_minion_pool(minion_pool, keys=None) -> dict: minion_pool_dict = view_utils.format_opt(minion_pool, keys) - def _hide_minion_creds(minion_conn): + def _hide_minion_creds(minion_conn: dict) -> None: if not minion_conn: return minion_conn if minion_conn.get('pkey'): @@ -33,10 +34,10 @@ def _hide_minion_creds(minion_conn): return minion_pool_dict -def single(minion_pool, keys=None): +def single(minion_pool, keys=None) -> dict[str, dict]: return {"minion_pool": _format_minion_pool(minion_pool, keys)} -def collection(minion_pools, keys=None): +def collection(minion_pools: list, keys=None) -> dict[str, list]: formatted_minion_pools = [_format_minion_pool(r, keys) for r in minion_pools] return {'minion_pools': formatted_minion_pools} diff --git a/coriolis/api/v1/views/region_view.py b/coriolis/api/v1/views/region_view.py index d66339f14..75df71b78 100644 --- a/coriolis/api/v1/views/region_view.py +++ b/coriolis/api/v1/views/region_view.py @@ -4,7 +4,7 @@ from coriolis.api.v1.views import utils as view_utils -def _format_region(region, keys=None): +def _format_region(region, keys=None) -> dict: region_dict = view_utils.format_opt(region, keys) mapped_endpoints = region_dict.get('mapped_endpoints', []) @@ -16,10 +16,10 @@ def _format_region(region, keys=None): return region_dict -def single(region, keys=None): +def single(region, keys=None) -> dict[str, dict]: return {"region": _format_region(region, keys)} -def collection(regions, keys=None): +def collection(regions, keys=None) -> dict[str, list]: formatted_regions = [_format_region(r, keys) for r in regions] return {'regions': formatted_regions} diff --git a/coriolis/api/v1/views/service_view.py b/coriolis/api/v1/views/service_view.py index 7dc1d1c9b..936d5e9a9 100644 --- a/coriolis/api/v1/views/service_view.py +++ b/coriolis/api/v1/views/service_view.py @@ -4,7 +4,7 @@ from coriolis.api.v1.views import utils as view_utils -def _format_service(service, keys=None): +def _format_service(service, keys=None) -> dict: service_dict = view_utils.format_opt(service, keys) mapped_regions = service_dict.get('mapped_regions', []) @@ -13,10 +13,10 @@ def _format_service(service, keys=None): return service_dict -def single(service, keys=None): +def single(service, keys=None) -> dict[str, dict]: return {"service": _format_service(service, keys)} -def collection(services, keys=None): +def collection(services, keys=None) -> dict[str, list]: formatted_services = [_format_service(r, keys) for r in services] return {'services': formatted_services} diff --git a/coriolis/api/v1/views/transfer_schedule_view.py b/coriolis/api/v1/views/transfer_schedule_view.py index 6965bda66..ba13a2b91 100644 --- a/coriolis/api/v1/views/transfer_schedule_view.py +++ b/coriolis/api/v1/views/transfer_schedule_view.py @@ -4,10 +4,10 @@ from coriolis.api.v1.views import utils as view_utils -def single(schedule, keys=None): +def single(schedule, keys=None) -> dict[str, dict]: return {"schedule": view_utils.format_opt(schedule, keys)} -def collection(schedules, keys=None): +def collection(schedules, keys=None) -> dict[str, list]: formatted_schedules = [view_utils.format_opt(m, keys) for m in schedules] return {'schedules': formatted_schedules} diff --git a/coriolis/api/v1/views/transfer_tasks_execution_view.py b/coriolis/api/v1/views/transfer_tasks_execution_view.py index 9b30029d9..505fe7837 100644 --- a/coriolis/api/v1/views/transfer_tasks_execution_view.py +++ b/coriolis/api/v1/views/transfer_tasks_execution_view.py @@ -10,7 +10,7 @@ LOG = logging.getLogger(__name__) -def _sort_tasks(tasks, filter_error_only_tasks=True): +def _sort_tasks(tasks: list, filter_error_only_tasks: bool = True) -> list: """ Sorts the given list of dicts representing tasks. Tasks are sorted primarily based on their index. @@ -22,7 +22,7 @@ def _sort_tasks(tasks, filter_error_only_tasks=True): return sorted(tasks, key=lambda t: t.get('index', 0)) -def format_transfer_tasks_execution(execution, keys=None): +def format_transfer_tasks_execution(execution: dict, keys=None) -> dict: if "tasks" in execution: execution["tasks"] = _sort_tasks(execution["tasks"]) @@ -31,11 +31,11 @@ def format_transfer_tasks_execution(execution, keys=None): return execution_dict -def single(execution, keys=None): +def single(execution: dict, keys=None) -> dict[str, dict]: return {"execution": format_transfer_tasks_execution(execution, keys)} -def collection(executions, keys=None): +def collection(executions: list, keys=None) -> dict[str, list]: formatted_executions = [ format_transfer_tasks_execution(m, keys) for m in executions ] diff --git a/coriolis/api/v1/views/transfer_view.py b/coriolis/api/v1/views/transfer_view.py index 7cc93cb5e..6b86ceb20 100644 --- a/coriolis/api/v1/views/transfer_view.py +++ b/coriolis/api/v1/views/transfer_view.py @@ -5,7 +5,7 @@ from coriolis.api.v1.views import utils as view_utils -def _format_transfer(transfer, keys=None): +def _format_transfer(transfer: dict, keys=None) -> dict: transfer_dict = view_utils.format_opt(transfer, keys) executions = transfer_dict.get('executions', []) @@ -16,10 +16,10 @@ def _format_transfer(transfer, keys=None): return transfer_dict -def single(transfer, keys=None): +def single(transfer: dict, keys=None) -> dict[str, dict]: return {"transfer": _format_transfer(transfer, keys)} -def collection(transfers, keys=None): +def collection(transfers: list[dict], keys=None) -> dict[str, list]: formatted_transfers = [_format_transfer(t, keys) for t in transfers] return {'transfers': formatted_transfers} diff --git a/coriolis/api/v1/views/utils.py b/coriolis/api/v1/views/utils.py index 7c0a889b3..801828d2c 100644 --- a/coriolis/api/v1/views/utils.py +++ b/coriolis/api/v1/views/utils.py @@ -2,10 +2,14 @@ # All Rights Reserved. import itertools +from typing import Iterator -def format_opt(option, keys=None): - def transform(key, value): +def format_opt( + option, + keys=None, +) -> dict: + def transform(key: str, value) -> Iterator[tuple]: if keys and key not in keys: return yield (key, value) diff --git a/coriolis/api/wsgi.py b/coriolis/api/wsgi.py index 52dd1e95a..4cdafd2c6 100644 --- a/coriolis/api/wsgi.py +++ b/coriolis/api/wsgi.py @@ -17,6 +17,7 @@ import inspect import math import time +from typing import Any, Callable, Iterator import six import webob @@ -269,7 +270,7 @@ def get_db_backups(self): def get_db_backup(self, backup_id): return self.get_db_item('backups', backup_id) - def best_match_content_type(self): + def best_match_content_type(self) -> str: """Determine the requested response content-type.""" if 'coriolis.best_content_type' not in self.environ: # Calculate the best MIME type @@ -291,7 +292,7 @@ def best_match_content_type(self): return self.environ['coriolis.best_content_type'] - def get_content_type(self): + def get_content_type(self) -> str: """Determine content type of the request body. Does not do any body introspection, only checks header @@ -307,7 +308,7 @@ def get_content_type(self): return content_type - def best_match_language(self): + def best_match_language(self) -> None: """Determines best available locale from the Accept-Language header. :returns: the best language match or None if the 'Accept-Language' @@ -379,59 +380,61 @@ def __call__(self, req): class ActionDispatcher(object): """Maps method name to local methods through action name.""" - def dispatch(self, *args, **kwargs): + def dispatch(self, *args, **kwargs) -> dict | str: """Find and call local method.""" action = kwargs.pop('action', 'default') action_method = getattr(self, str(action), self.default) return action_method(*args, **kwargs) - def default(self, data): + def default(self, data: bytes | dict): raise NotImplementedError() class TextDeserializer(ActionDispatcher): """Default request body deserialization.""" - def deserialize(self, datastring, action='default'): + def deserialize( + self, datastring: bytes, action: str = 'default' + ) -> dict[str, dict]: return self.dispatch(datastring, action=action) - def default(self, datastring): + def default(self, datastring: bytes): return {} class JSONDeserializer(TextDeserializer): - def _from_json(self, datastring): + def _from_json(self, datastring: bytes) -> dict: try: return jsonutils.loads(datastring) except ValueError: msg = _("cannot understand JSON") raise exception.MalformedRequestBody(reason=msg) - def default(self, datastring): + def default(self, datastring: bytes | dict) -> dict[str, dict]: return {'body': self._from_json(datastring)} class DictSerializer(ActionDispatcher): """Default request body serialization.""" - def serialize(self, data, action='default'): + def serialize(self, data: dict | str, action: str = 'default') -> str: return self.dispatch(data, action=action) - def default(self, data): + def default(self, data: dict | str): return "" class JSONDictSerializer(DictSerializer): """Default JSON request body serialization.""" - def default(self, data): + def default(self, data: bytes | dict | str) -> str: return jsonutils.dumps(data) class CSVSerializer(DictSerializer): """Serializer for CSV responses. Expects pre-formatted CSV string.""" - def default(self, data): + def default(self, data: dict | str) -> str: return str(data) @@ -492,7 +495,7 @@ class ResponseObject(object): optional. """ - def __init__(self, obj, code=None, **serializers): + def __init__(self, obj: dict | str, code=None, **serializers): """Binds serializers with an object. Takes keyword arguments akin to the @serializer() decorator @@ -514,7 +517,7 @@ def __getitem__(self, key): return self._headers[key.lower()] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: str) -> None: """Sets a header with the given name to the given value.""" self._headers[key.lower()] = value @@ -524,7 +527,7 @@ def __delitem__(self, key): del self._headers[key.lower()] - def _bind_method_serializers(self, meth_serializers): + def _bind_method_serializers(self, meth_serializers: dict[Any, Any]) -> None: """Binds method serializers with the response object. Binds the method serializers with the response object. @@ -541,7 +544,9 @@ def _bind_method_serializers(self, meth_serializers): for mtype, serializer in meth_serializers.items(): self.serializers.setdefault(mtype, serializer) - def get_serializer(self, content_type, default_serializers=None): + def get_serializer( + self, content_type: str, default_serializers: dict | None = None + ) -> tuple: """Returns the serializer for the wrapped object. Returns the serializer for the wrapped object subject to the @@ -562,7 +567,9 @@ def get_serializer(self, content_type, default_serializers=None): except (KeyError, TypeError): raise exception.InvalidContentType(content_type=content_type) - def preserialize(self, content_type, default_serializers=None): + def preserialize( + self, content_type: str, default_serializers: dict | None = None + ) -> None: """Prepares the serializer that will be used to serialize. Determines the serializer that will be used and prepares an @@ -580,7 +587,12 @@ def attach(self, **kwargs): if self.media_type in kwargs: self.serializer.attach(kwargs[self.media_type]) - def serialize(self, request, content_type, default_serializers=None): + def serialize( + self, + request: Request, + content_type: str, + default_serializers: dict | None = None, + ) -> webob.Response: """Serializes the wrapped object. Utility method for serializing the wrapped object. Returns a @@ -607,7 +619,7 @@ def serialize(self, request, content_type, default_serializers=None): return response @property - def code(self): + def code(self) -> int: """Retrieve the response status.""" return self._code or self._default_code @@ -619,7 +631,7 @@ def headers(self): return self._headers.copy() -def action_peek_json(body): +def action_peek_json(body: bytes) -> str: """Determine action to invoke.""" try: @@ -645,10 +657,15 @@ class ResourceExceptionHandler(object): exceptions, with the appropriate logging. """ - def __enter__(self): + def __enter__(self) -> None: return None - def __exit__(self, ex_type, ex_value, ex_traceback): + def __exit__( + self, + ex_type, + ex_value, + ex_traceback, + ) -> bool: if not ex_value: return True @@ -701,7 +718,12 @@ class Resource(Application): wrapped in Fault() to provide API friendly error responses. """ - def __init__(self, controller, action_peek=None, **deserializers): + def __init__( + self, + controller, + action_peek=None, + **deserializers, + ): """Initialize Resource. :param controller: object that implement methods created by routes lib @@ -729,7 +751,10 @@ def __init__(self, controller, action_peek=None, **deserializers): self.wsgi_extensions = {} self.wsgi_action_extensions = {} - def register_actions(self, controller): + def register_actions( + self, + controller, + ) -> None: """Registers controller actions with this resource.""" actions = getattr(controller, 'wsgi_actions', {}) @@ -755,7 +780,7 @@ def register_extensions(self, controller): self.wsgi_extensions[method_name] = [] self.wsgi_extensions[method_name].append(extension) - def get_action_args(self, request_environment): + def get_action_args(self, request_environment: dict) -> dict[str, str]: """Parse dictionary created by routes library.""" # NOTE(Vek): Check for get_action_args() override in the @@ -780,7 +805,7 @@ def get_action_args(self, request_environment): return args - def get_body(self, request): + def get_body(self, request: Request) -> tuple: if len(request.body) == 0: LOG.debug("Empty body provided in request") @@ -798,7 +823,9 @@ def get_body(self, request): return content_type, request.body - def deserialize(self, meth, content_type, body): + def deserialize( + self, meth: Callable, content_type: str, body: bytes + ) -> dict[str, dict]: meth_deserializers = getattr(meth, 'wsgi_deserializers', {}) try: mtype = _MEDIA_TYPE_MAP.get(content_type, content_type) @@ -811,7 +838,9 @@ def deserialize(self, meth, content_type, body): return deserializer().deserialize(body) - def pre_process_extensions(self, extensions, request, action_args): + def pre_process_extensions( + self, extensions: list[Any], request: Request, action_args: dict + ) -> tuple: # List of callables for post-processing extensions post = [] @@ -841,7 +870,13 @@ def pre_process_extensions(self, extensions, request, action_args): # Run post-processing in the reverse order return None, reversed(post) - def post_process_extensions(self, extensions, resp_obj, request, action_args): + def post_process_extensions( + self, + extensions: Iterator, + resp_obj: ResponseObject, + request: Request, + action_args: dict, + ) -> None: for ext in extensions: response = None if inspect.isgenerator(ext): @@ -870,7 +905,7 @@ def post_process_extensions(self, extensions, resp_obj, request, action_args): return None @webob.dec.wsgify(RequestClass=Request) - def __call__(self, request): + def __call__(self, request: Request) -> "Fault|webob.Response": """WSGI method that controls (de)serialization and method dispatch.""" LOG.info( @@ -893,7 +928,15 @@ def __call__(self, request): request, action, action_args, content_type, body, accept ) - def _process_stack(self, request, action, action_args, content_type, body, accept): + def _process_stack( + self, + request: Request, + action: str, + action_args: dict, + content_type: str | None, + body: bytes | str, + accept: str, + ) -> "Fault|webob.Response": """Implement the processing stack.""" # Get the implementing method @@ -980,7 +1023,9 @@ def _process_stack(self, request, action, action_args, content_type, body, accep return response - def get_method(self, request, action, content_type, body): + def get_method( + self, request: Request, action: str, content_type: str | None, body: bytes | str + ) -> tuple[Callable, list]: """Look up the action-specific method and its extensions.""" # Look up the method @@ -1017,13 +1062,13 @@ def get_method(self, request, action, content_type, body): self.wsgi_action_extensions.get(action_name, []), ) - def dispatch(self, method, request, action_args): + def dispatch(self, method: Callable, request: Request, action_args: dict) -> dict: """Dispatch a call to the action-specific method.""" return method(req=request, **action_args) -def action(name): +def action(name: str) -> Callable[[Callable], Callable]: """Mark a function as an action. The given name will be taken as the action key in the body. @@ -1032,7 +1077,7 @@ def action(name): non-extending definitions of create and delete operations. """ - def decorator(func): + def decorator(func: Callable) -> Callable: func.wsgi_action = name return func @@ -1075,7 +1120,9 @@ class ControllerMetaclass(type): mapping action keys to method names. """ - def __new__(mcs, name, bases, cls_dict): + def __new__( + mcs: "type[ControllerMetaclass]", name: str, bases: tuple, cls_dict: dict + ): """Adds the wsgi_actions dictionary to the class.""" # Find all actions @@ -1244,7 +1291,7 @@ def __init__(self, exception): self.status_int = exception.status_int @webob.dec.wsgify(RequestClass=Request) - def __call__(self, req): + def __call__(self, req: Request): """Generate a WSGI response based on the exception passed to ctor.""" # Replace the body with fault details. locale = req.best_match_language() @@ -1281,7 +1328,7 @@ def __str__(self): return self.wrapped_exc.__str__() -def _set_request_id_header(req, headers): +def _set_request_id_header(req: Request, headers) -> None: context = req.environ.get('coriolis.context') if context: headers['x-compute-request-id'] = context.request_id diff --git a/coriolis/conductor/rpc/client.py b/coriolis/conductor/rpc/client.py index 57f33704f..5dce09af7 100644 --- a/coriolis/conductor/rpc/client.py +++ b/coriolis/conductor/rpc/client.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + import oslo_messaging as messaging from oslo_config import cfg from oslo_log import log as logging @@ -22,14 +24,22 @@ class ConductorClient(rpc.BaseRPCClient): - def __init__(self, timeout=None, topic=constants.CONDUCTOR_MAIN_MESSAGING_TOPIC): + def __init__( + self, timeout=None, topic: str = constants.CONDUCTOR_MAIN_MESSAGING_TOPIC + ): target = messaging.Target(topic=topic, version=VERSION) if timeout is None: timeout = CONF.conductor.conductor_rpc_timeout super(ConductorClient, self).__init__(target, timeout=timeout) def create_endpoint( - self, ctxt, name, endpoint_type, description, connection_info, mapped_regions + self, + ctxt, + name: str, + endpoint_type: str, + description: str, + connection_info: dict[str, str], + mapped_regions: list[str], ): return self._call( ctxt, @@ -41,7 +51,12 @@ def create_endpoint( mapped_regions=mapped_regions, ) - def update_endpoint(self, ctxt, endpoint_id, updated_values): + def update_endpoint( + self, + ctxt, + endpoint_id: str, + updated_values: dict, + ): return self._call( ctxt, 'update_endpoint', @@ -49,25 +64,25 @@ def update_endpoint(self, ctxt, endpoint_id, updated_values): updated_values=updated_values, ) - def get_endpoints(self, ctxt): + def get_endpoints(self, ctxt) -> list[Any]: return self._call(ctxt, 'get_endpoints') - def get_endpoint(self, ctxt, endpoint_id): + def get_endpoint(self, ctxt, endpoint_id: str): return self._call(ctxt, 'get_endpoint', endpoint_id=endpoint_id) - def delete_endpoint(self, ctxt, endpoint_id): + def delete_endpoint(self, ctxt, endpoint_id: str) -> None: return self._call(ctxt, 'delete_endpoint', endpoint_id=endpoint_id) def get_endpoint_instances( self, ctxt, - endpoint_id, - source_environment, + endpoint_id: str, + source_environment: dict[Any, Any], marker=None, limit=None, - instance_name_pattern=None, - refresh=False, - ): + instance_name_pattern: str | None = None, + refresh: bool = False, + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_instances', @@ -80,8 +95,12 @@ def get_endpoint_instances( ) def get_endpoint_instance( - self, ctxt, endpoint_id, source_environment, instance_name - ): + self, + ctxt, + endpoint_id: str, + source_environment: dict[Any, Any], + instance_name: str, + ) -> dict: return self._call( ctxt, 'get_endpoint_instance', @@ -90,7 +109,13 @@ def get_endpoint_instance( instance_name=instance_name, ) - def get_endpoint_source_options(self, ctxt, endpoint_id, env, option_names): + def get_endpoint_source_options( + self, + ctxt, + endpoint_id: str, + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_source_options', @@ -99,7 +124,13 @@ def get_endpoint_source_options(self, ctxt, endpoint_id, env, option_names): option_names=option_names, ) - def get_endpoint_destination_options(self, ctxt, endpoint_id, env, option_names): + def get_endpoint_destination_options( + self, + ctxt, + endpoint_id: str, + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_destination_options', @@ -108,17 +139,32 @@ def get_endpoint_destination_options(self, ctxt, endpoint_id, env, option_names) option_names=option_names, ) - def get_endpoint_networks(self, ctxt, endpoint_id, env): + def get_endpoint_networks( + self, + ctxt, + endpoint_id: str, + env: dict[Any, Any], + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_networks', endpoint_id=endpoint_id, env=env ) - def get_endpoint_storage(self, ctxt, endpoint_id, env): + def get_endpoint_storage( + self, + ctxt, + endpoint_id: str, + env: dict[Any, Any], + ) -> dict[str, list]: return self._call( ctxt, 'get_endpoint_storage', endpoint_id=endpoint_id, env=env ) - def get_endpoint_inventory_csv(self, ctxt, endpoint_id, source_environment): + def get_endpoint_inventory_csv( + self, + ctxt, + endpoint_id: str, + source_environment: dict[Any, Any], + ) -> str: return self._call( ctxt, 'get_endpoint_inventory_csv', @@ -126,10 +172,15 @@ def get_endpoint_inventory_csv(self, ctxt, endpoint_id, source_environment): source_environment=source_environment, ) - def validate_endpoint_connection(self, ctxt, endpoint_id): + def validate_endpoint_connection(self, ctxt, endpoint_id: str) -> tuple: return self._call(ctxt, 'validate_endpoint_connection', endpoint_id=endpoint_id) - def validate_endpoint_target_environment(self, ctxt, endpoint_id, target_env): + def validate_endpoint_target_environment( + self, + ctxt, + endpoint_id: str, + target_env: dict, + ) -> tuple[bool, None]: return self._call( ctxt, 'validate_endpoint_target_environment', @@ -137,7 +188,12 @@ def validate_endpoint_target_environment(self, ctxt, endpoint_id, target_env): target_env=target_env, ) - def validate_endpoint_source_environment(self, ctxt, endpoint_id, source_env): + def validate_endpoint_source_environment( + self, + ctxt, + endpoint_id: str, + source_env: dict, + ) -> tuple[bool, None]: return self._call( ctxt, 'validate_endpoint_source_environment', @@ -145,10 +201,15 @@ def validate_endpoint_source_environment(self, ctxt, endpoint_id, source_env): source_env=source_env, ) - def get_available_providers(self, ctxt): + def get_available_providers(self, ctxt) -> dict[str, dict]: return self._call(ctxt, 'get_available_providers') - def get_provider_schemas(self, ctxt, platform_name, provider_type): + def get_provider_schemas( + self, + ctxt, + platform_name: str, + provider_type: int, + ) -> dict[str, dict]: return self._call( ctxt, 'get_provider_schemas', @@ -157,8 +218,12 @@ def get_provider_schemas(self, ctxt, platform_name, provider_type): ) def execute_transfer_tasks( - self, ctxt, transfer_id, shutdown_instances=False, auto_deploy=False - ): + self, + ctxt, + transfer_id: str, + shutdown_instances: bool = False, + auto_deploy: bool = False, + ) -> dict: return self._call( ctxt, 'execute_transfer_tasks', @@ -170,14 +235,14 @@ def execute_transfer_tasks( def get_transfer_tasks_executions( self, ctxt, - transfer_id, - include_tasks=False, - marker=None, - limit=None, - sort_keys=None, - sort_dirs=None, - filters=None, - ): + transfer_id: str, + include_tasks: bool = False, + marker: str | None = None, + limit: int | None = None, + sort_keys: list | None = None, + sort_dirs: list | None = None, + filters: dict | None = None, + ) -> list: return self._call( ctxt, 'get_transfer_tasks_executions', @@ -191,8 +256,12 @@ def get_transfer_tasks_executions( ) def get_transfer_tasks_execution( - self, ctxt, transfer_id, execution_id, include_task_info=False - ): + self, + ctxt, + transfer_id: str, + execution_id: str, + include_task_info: bool = False, + ) -> dict: return self._call( ctxt, 'get_transfer_tasks_execution', @@ -201,7 +270,12 @@ def get_transfer_tasks_execution( include_task_info=include_task_info, ) - def delete_transfer_tasks_execution(self, ctxt, transfer_id, execution_id): + def delete_transfer_tasks_execution( + self, + ctxt, + transfer_id: str, + execution_id: str, + ) -> None: return self._call( ctxt, 'delete_transfer_tasks_execution', @@ -209,7 +283,13 @@ def delete_transfer_tasks_execution(self, ctxt, transfer_id, execution_id): execution_id=execution_id, ) - def cancel_transfer_tasks_execution(self, ctxt, transfer_id, execution_id, force): + def cancel_transfer_tasks_execution( + self, + ctxt, + transfer_id: str, + execution_id: str, + force: bool, + ) -> None: return self._call( ctxt, 'cancel_transfer_tasks_execution', @@ -221,22 +301,22 @@ def cancel_transfer_tasks_execution(self, ctxt, transfer_id, execution_id, force def create_instances_transfer( self, ctxt, - transfer_scenario, - origin_endpoint_id, - destination_endpoint_id, - origin_minion_pool_id, - destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, - source_environment, - destination_environment, - instances, - network_map, - storage_mappings, - notes=None, + transfer_scenario: str, + origin_endpoint_id: str, + destination_endpoint_id: str, + origin_minion_pool_id: str | None, + destination_minion_pool_id: str | None, + instance_osmorphing_minion_pool_mappings: dict[Any, Any], + source_environment: dict, + destination_environment: dict[str, dict], + instances: list[str], + network_map: dict[Any, Any], + storage_mappings: dict[Any, Any], + notes: str | None = None, user_scripts=None, - clone_disks=True, - skip_os_morphing=False, - ): + clone_disks: bool = True, + skip_os_morphing: bool = False, + ) -> dict: return self._call( ctxt, 'create_instances_transfer', @@ -262,14 +342,14 @@ def create_instances_transfer( def get_transfers( self, ctxt, - include_tasks_executions=False, - include_task_info=False, - marker=None, - limit=None, - sort_keys=None, - sort_dirs=None, - filters=None, - ): + include_tasks_executions: bool = False, + include_task_info: bool = False, + marker: str | None = None, + limit: int | None = None, + sort_keys: list | None = None, + sort_dirs: list | None = None, + filters: dict | None = None, + ) -> list[dict]: return self._call( ctxt, 'get_transfers', @@ -282,7 +362,12 @@ def get_transfers( filters=filters, ) - def get_transfer(self, ctxt, transfer_id, include_task_info=False): + def get_transfer( + self, + ctxt, + transfer_id: str, + include_task_info: bool = False, + ) -> dict: return self._call( ctxt, 'get_transfer', @@ -290,23 +375,23 @@ def get_transfer(self, ctxt, transfer_id, include_task_info=False): include_task_info=include_task_info, ) - def delete_transfer(self, ctxt, transfer_id): + def delete_transfer(self, ctxt, transfer_id: str) -> None: self._call(ctxt, 'delete_transfer', transfer_id=transfer_id) - def delete_transfer_disks(self, ctxt, transfer_id): + def delete_transfer_disks(self, ctxt, transfer_id: str) -> dict: return self._call(ctxt, 'delete_transfer_disks', transfer_id=transfer_id) def get_deployments( self, ctxt, - include_tasks=False, - include_task_info=False, + include_tasks: bool = False, + include_task_info: bool = False, marker=None, limit=None, - sort_keys=None, - sort_dirs=None, - filters=None, - ): + sort_keys: list | None = None, + sort_dirs: list | None = None, + filters: dict | None = None, + ) -> list: return self._call( ctxt, 'get_deployments', @@ -319,7 +404,12 @@ def get_deployments( filters=filters, ) - def get_deployment(self, ctxt, deployment_id, include_task_info=False): + def get_deployment( + self, + ctxt, + deployment_id: str, + include_task_info: bool = False, + ) -> dict: return self._call( ctxt, 'get_deployment', @@ -327,12 +417,22 @@ def get_deployment(self, ctxt, deployment_id, include_task_info=False): include_task_info=include_task_info, ) - def confirm_deployer_completed(self, ctxt, deployment_id, force=False): + def confirm_deployer_completed( + self, + ctxt, + deployment_id: str, + force: bool = False, + ) -> None: return self._cast( ctxt, 'confirm_deployer_completed', deployment_id=deployment_id, force=force ) - def report_deployer_failure(self, ctxt, deployemnt_id, deployer_error_details): + def report_deployer_failure( + self, + ctxt, + deployemnt_id: str, + deployer_error_details: str, + ) -> None: return self._cast( ctxt, 'report_deployer_failure', @@ -343,15 +443,15 @@ def report_deployer_failure(self, ctxt, deployemnt_id, deployer_error_details): def deploy_transfer_instances( self, ctxt, - transfer_id, - force, - wait_for_execution=None, - instance_osmorphing_minion_pool_mappings=None, - clone_disks=None, - skip_os_morphing=None, - user_scripts=None, - trust_id=None, - ): + transfer_id: str, + force: bool, + wait_for_execution: str | None = None, + instance_osmorphing_minion_pool_mappings: dict | None = None, + clone_disks: bool | None = None, + skip_os_morphing: bool | None = None, + user_scripts: dict | None = None, + trust_id: str | None = None, + ) -> dict: return self._call( ctxt, 'deploy_transfer_instances', @@ -367,19 +467,19 @@ def deploy_transfer_instances( trust_id=trust_id, ) - def delete_deployment(self, ctxt, deployment_id): + def delete_deployment(self, ctxt, deployment_id: str) -> None: self._call(ctxt, 'delete_deployment', deployment_id=deployment_id) - def cancel_deployment(self, ctxt, deployment_id, force): + def cancel_deployment(self, ctxt, deployment_id: str, force: bool) -> None: self._call(ctxt, 'cancel_deployment', deployment_id=deployment_id, force=force) - def set_task_host(self, ctxt, task_id, host): + def set_task_host(self, ctxt, task_id: str, host: str) -> None: self._call(ctxt, 'set_task_host', task_id=task_id, host=host) - def set_task_process(self, ctxt, task_id, process_id): + def set_task_process(self, ctxt, task_id: str, process_id: int) -> None: self._call(ctxt, 'set_task_process', task_id=task_id, process_id=process_id) - def task_completed(self, ctxt, task_id, task_result): + def task_completed(self, ctxt, task_id: str, task_result: dict) -> None: self._call(ctxt, 'task_completed', task_id=task_id, task_result=task_result) def confirm_task_cancellation(self, ctxt, task_id, cancellation_details): @@ -390,7 +490,12 @@ def confirm_task_cancellation(self, ctxt, task_id, cancellation_details): cancellation_details=cancellation_details, ) - def set_task_error(self, ctxt, task_id, exception_details): + def set_task_error( + self, + ctxt, + task_id: str, + exception_details: str, + ) -> None: self._call( ctxt, 'set_task_error', task_id=task_id, exception_details=exception_details ) @@ -401,7 +506,13 @@ def add_task_event(self, ctxt, task_id, level, message): ) def add_task_progress_update( - self, ctxt, task_id, message, initial_step=0, total_steps=0, return_event=False + self, + ctxt, + task_id: str, + message: str, + initial_step: int = 0, + total_steps: int = 0, + return_event: bool = False, ): operation = self._cast if return_event: @@ -418,13 +529,13 @@ def add_task_progress_update( def update_task_progress_update( self, ctxt, - task_id, - progress_update_index, - new_current_step, + task_id: str, + progress_update_index: int, + new_current_step: int, new_total_steps=None, new_message=None, - sync=False, - ): + sync: bool = False, + ) -> None: # Use _call on the last step to make sure the DB is updated # and the UI is showing the correct progress. # Intermediate steps use cast to avoid blocking on every tick. @@ -451,12 +562,12 @@ def update_task_progress_update( def create_transfer_schedule( self, ctxt, - transfer_id, - schedule, - enabled, + transfer_id: str, + schedule: dict[str, int], + enabled: bool, exp_date, - shutdown_instance, - auto_deploy, + shutdown_instance: bool, + auto_deploy: bool, ): return self._call( ctxt, @@ -469,7 +580,13 @@ def create_transfer_schedule( auto_deploy=auto_deploy, ) - def update_transfer_schedule(self, ctxt, transfer_id, schedule_id, updated_values): + def update_transfer_schedule( + self, + ctxt, + transfer_id: str, + schedule_id: str, + updated_values: dict, + ): return self._call( ctxt, 'update_transfer_schedule', @@ -478,7 +595,12 @@ def update_transfer_schedule(self, ctxt, transfer_id, schedule_id, updated_value updated_values=updated_values, ) - def delete_transfer_schedule(self, ctxt, transfer_id, schedule_id): + def delete_transfer_schedule( + self, + ctxt, + transfer_id: str, + schedule_id: str, + ) -> None: return self._call( ctxt, 'delete_transfer_schedule', @@ -486,12 +608,23 @@ def delete_transfer_schedule(self, ctxt, transfer_id, schedule_id): schedule_id=schedule_id, ) - def get_transfer_schedules(self, ctxt, transfer_id=None, expired=True): + def get_transfer_schedules( + self, + ctxt, + transfer_id: str | None = None, + expired: bool = True, + ): return self._call( ctxt, 'get_transfer_schedules', transfer_id=transfer_id, expired=expired ) - def get_transfer_schedule(self, ctxt, transfer_id, schedule_id, expired=True): + def get_transfer_schedule( + self, + ctxt, + transfer_id: str, + schedule_id: str, + expired: bool = True, + ): return self._call( ctxt, 'get_transfer_schedule', @@ -500,7 +633,12 @@ def get_transfer_schedule(self, ctxt, transfer_id, schedule_id, expired=True): expired=expired, ) - def update_transfer(self, ctxt, transfer_id, updated_properties): + def update_transfer( + self, + ctxt, + transfer_id: str, + updated_properties: dict, + ) -> dict: return self._call( ctxt, 'update_transfer', @@ -511,10 +649,16 @@ def update_transfer(self, ctxt, transfer_id, updated_properties): def get_diagnostics(self, ctxt): return self._call(ctxt, 'get_diagnostics') - def get_all_diagnostics(self, ctxt): + def get_all_diagnostics(self, ctxt) -> list[dict]: return self._call(ctxt, 'get_all_diagnostics') - def create_region(self, ctxt, region_name, description="", enabled=True): + def create_region( + self, + ctxt, + region_name: str, + description: str = "", + enabled: bool = True, + ): return self._call( ctxt, 'create_region', @@ -526,27 +670,32 @@ def create_region(self, ctxt, region_name, description="", enabled=True): def get_regions(self, ctxt): return self._call(ctxt, 'get_regions') - def get_region(self, ctxt, region_id): + def get_region(self, ctxt, region_id: str): return self._call(ctxt, 'get_region', region_id=region_id) - def update_region(self, ctxt, region_id, updated_values): + def update_region( + self, + ctxt, + region_id: str, + updated_values: dict[str, str], + ): return self._call( ctxt, 'update_region', region_id=region_id, updated_values=updated_values ) - def delete_region(self, ctxt, region_id): + def delete_region(self, ctxt, region_id: str) -> None: return self._call(ctxt, 'delete_region', region_id=region_id) def register_service( self, ctxt, - host, - binary, - topic, - enabled, - mapped_regions, - providers=None, - specs=None, + host: str, + binary: str, + topic: str, + enabled: bool, + mapped_regions: list | None, + providers: dict | None = None, + specs: dict | None = None, ): return self._call( ctxt, @@ -560,7 +709,13 @@ def register_service( specs=specs, ) - def check_service_registered(self, ctxt, host, binary, topic): + def check_service_registered( + self, + ctxt, + host: str, + binary: str, + topic: str, + ) -> None: return self._call( ctxt, 'check_service_registered', host=host, binary=binary, topic=topic ) @@ -568,20 +723,28 @@ def check_service_registered(self, ctxt, host, binary, topic): def get_services(self, ctxt): return self._call(ctxt, 'get_services') - def get_service(self, ctxt, service_id): + def get_service(self, ctxt, service_id: str): return self._call(ctxt, 'get_service', service_id=service_id) - def update_service(self, ctxt, service_id, updated_values): + def update_service( + self, + ctxt, + service_id: str, + updated_values: dict, + ): return self._call( ctxt, 'update_service', service_id=service_id, updated_values=updated_values ) - def delete_service(self, ctxt, service_id): + def delete_service(self, ctxt, service_id: str) -> None: return self._call(ctxt, 'delete_service', service_id=service_id) def confirm_transfer_minions_allocation( - self, ctxt, transfer_id, minion_machine_allocations - ): + self, + ctxt, + transfer_id: str, + minion_machine_allocations: dict[str, dict], + ) -> None: self._call( ctxt, 'confirm_transfer_minions_allocation', @@ -590,8 +753,11 @@ def confirm_transfer_minions_allocation( ) def report_transfer_minions_allocation_error( - self, ctxt, transfer_id, minion_allocation_error_details - ): + self, + ctxt, + transfer_id: str, + minion_allocation_error_details: str, + ) -> None: self._call( ctxt, 'report_transfer_minions_allocation_error', @@ -600,8 +766,11 @@ def report_transfer_minions_allocation_error( ) def confirm_deployment_minions_allocation( - self, ctxt, deployment_id, minion_machine_allocations - ): + self, + ctxt, + deployment_id: str, + minion_machine_allocations: dict[str, dict], + ) -> None: self._call( ctxt, 'confirm_deployment_minions_allocation', @@ -610,8 +779,11 @@ def confirm_deployment_minions_allocation( ) def report_deployment_minions_allocation_error( - self, ctxt, deployment_id, minion_allocation_error_details - ): + self, + ctxt, + deployment_id: str, + minion_allocation_error_details: str, + ) -> None: self._call( ctxt, 'report_deployment_minions_allocation_error', @@ -621,13 +793,13 @@ def report_deployment_minions_allocation_error( class ConductorTaskRpcEventHandler(events.BaseEventHandler): - def __init__(self, ctxt, task_id): + def __init__(self, ctxt, task_id: str): self._ctxt = ctxt self._task_id = task_id self._rpc_conductor_client_instance = None @property - def _rpc_conductor_client(self): + def _rpc_conductor_client(self) -> ConductorClient: # NOTE(aznashwan): it is unsafe to fork processes with pre-instantiated # oslo_messaging clients as the underlying thread queues will # be invalidated. @@ -636,11 +808,15 @@ def _rpc_conductor_client(self): return self._rpc_conductor_client_instance @classmethod - def get_progress_update_identifier(cls, progress_update): + def get_progress_update_identifier(cls, progress_update) -> int: return progress_update['index'] def add_progress_update( - self, message, initial_step=0, total_steps=0, return_event=False + self, + message: str, + initial_step: int = 0, + total_steps: int = 0, + return_event: bool = False, ): LOG.info( "Sending progress update for task '%s' to conductor: %s", @@ -658,12 +834,12 @@ def add_progress_update( def update_progress_update( self, - update_identifier, - new_current_step, + update_identifier: int, + new_current_step: int, new_total_steps=None, new_message=None, - sync=False, - ): + sync: bool = False, + ) -> None: LOG.info( "Updating progress update '%s' for task '%s' with new step %s", update_identifier, diff --git a/coriolis/conductor/rpc/server.py b/coriolis/conductor/rpc/server.py index 5655ad608..8a5826899 100644 --- a/coriolis/conductor/rpc/server.py +++ b/coriolis/conductor/rpc/server.py @@ -5,6 +5,7 @@ import functools import itertools import uuid +from typing import Any, Callable from oslo_concurrency import lockutils from oslo_config import cfg @@ -58,7 +59,7 @@ } -def _get_reservation_type(destination_endpoint, scenario): +def _get_reservation_type(destination_endpoint: models.Endpoint, scenario: str) -> str: """Returns the licensing reservation type for the given transfer scenario, taking into account any provider-specific overrides for the given destination endpoint's type (e.g. SAP transfers require @@ -71,13 +72,13 @@ def _get_reservation_type(destination_endpoint, scenario): ) -def endpoint_synchronized(func): +def endpoint_synchronized(func: Callable) -> Callable: @functools.wraps(func) def wrapper(self, ctxt, endpoint_id, *args, **kwargs): @lockutils.synchronized( constants.ENDPOINT_LOCK_NAME_FORMAT % endpoint_id, external=True ) - def inner(): + def inner() -> models.Endpoint | None: return func(self, ctxt, endpoint_id, *args, **kwargs) return inner() @@ -85,13 +86,13 @@ def inner(): return wrapper -def transfer_synchronized(func): +def transfer_synchronized(func: Callable) -> Callable: @functools.wraps(func) def wrapper(self, ctxt, transfer_id, *args, **kwargs): @lockutils.synchronized( constants.TRANSFER_LOCK_NAME_FORMAT % transfer_id, external=True ) - def inner(): + def inner() -> dict | list | None: return func(self, ctxt, transfer_id, *args, **kwargs) return inner() @@ -99,13 +100,13 @@ def inner(): return wrapper -def schedule_synchronized(func): +def schedule_synchronized(func: Callable) -> Callable: @functools.wraps(func) def wrapper(self, ctxt, transfer_id, schedule_id, *args, **kwargs): @lockutils.synchronized( constants.SCHEDULE_LOCK_NAME_FORMAT % schedule_id, external=True ) - def inner(): + def inner() -> models.TransferSchedule | None: return func(self, ctxt, transfer_id, schedule_id, *args, **kwargs) return inner() @@ -113,13 +114,15 @@ def inner(): return wrapper -def task_synchronized(func): +def task_synchronized(func: Callable) -> Callable: @functools.wraps(func) - def wrapper(self, ctxt, task_id, *args, **kwargs): + def wrapper( + self, ctxt: context.RequestContext, task_id: str, *args, **kwargs + ) -> models.TaskProgressUpdate | None: @lockutils.synchronized( constants.TASK_LOCK_NAME_FORMAT % task_id, external=True ) - def inner(): + def inner() -> models.TaskProgressUpdate | None: return func(self, ctxt, task_id, *args, **kwargs) return inner() @@ -127,9 +130,11 @@ def inner(): return wrapper -def parent_tasks_execution_synchronized(func): +def parent_tasks_execution_synchronized(func: Callable) -> Callable: @functools.wraps(func) - def wrapper(self, ctxt, task_id, *args, **kwargs): + def wrapper( + self, ctxt: context.RequestContext, task_id: str, *args, **kwargs + ) -> None: task = db_api.get_task(ctxt, task_id) @lockutils.synchronized( @@ -138,7 +143,7 @@ def wrapper(self, ctxt, task_id, *args, **kwargs): @lockutils.synchronized( constants.TASK_LOCK_NAME_FORMAT % task_id, external=True ) - def inner(): + def inner() -> None: return func(self, ctxt, task_id, *args, **kwargs) return inner() @@ -146,13 +151,13 @@ def inner(): return wrapper -def deployment_synchronized(func): +def deployment_synchronized(func: Callable) -> Callable: @functools.wraps(func) def wrapper(self, ctxt, deployment_id, *args, **kwargs): @lockutils.synchronized( constants.DEPLOYMENT_LOCK_NAME_FORMAT % deployment_id, external=True ) - def inner(): + def inner() -> dict | None: return func(self, ctxt, deployment_id, *args, **kwargs) return inner() @@ -160,13 +165,13 @@ def inner(): return wrapper -def tasks_execution_synchronized(func): +def tasks_execution_synchronized(func: Callable) -> Callable: @functools.wraps(func) def wrapper(self, ctxt, transfer_id, execution_id, *args, **kwargs): @lockutils.synchronized( constants.EXECUTION_LOCK_NAME_FORMAT % execution_id, external=True ) - def inner(): + def inner() -> dict | None: return func(self, ctxt, transfer_id, execution_id, *args, **kwargs) return inner() @@ -174,13 +179,13 @@ def inner(): return wrapper -def region_synchronized(func): +def region_synchronized(func: Callable) -> Callable: @functools.wraps(func) def wrapper(self, ctxt, region_id, *args, **kwargs): @lockutils.synchronized( constants.REGION_LOCK_NAME_FORMAT % region_id, external=True ) - def inner(): + def inner() -> models.Region | None: return func(self, ctxt, region_id, *args, **kwargs) return inner() @@ -188,13 +193,13 @@ def inner(): return wrapper -def service_synchronized(func): +def service_synchronized(func: Callable) -> Callable: @functools.wraps(func) def wrapper(self, ctxt, service_id, *args, **kwargs): @lockutils.synchronized( constants.SERVICE_LOCK_NAME_FORMAT % service_id, external=True ) - def inner(): + def inner() -> models.Service: return func(self, ctxt, service_id, *args, **kwargs) return inner() @@ -223,19 +228,19 @@ def _worker_client(self): return self._worker_client_instance @property - def _scheduler_client(self): + def _scheduler_client(self) -> rpc_scheduler_client.SchedulerClient: if not self._scheduler_client_instance: self._scheduler_client_instance = rpc_scheduler_client.SchedulerClient() return self._scheduler_client_instance @property - def _transfer_cron_client(self): + def _transfer_cron_client(self) -> rpc_cron_client.TransferCronClient: if not self._transfer_cron_client_instance: self._transfer_cron_client_instance = rpc_cron_client.TransferCronClient() return self._transfer_cron_client_instance @property - def _minion_manager_client(self): + def _minion_manager_client(self) -> rpc_minion_manager_client.MinionManagerClient: if not self._minion_manager_client_instance: self._minion_manager_client_instance = ( rpc_minion_manager_client.MinionManagerClient() @@ -243,14 +248,16 @@ def _minion_manager_client(self): return self._minion_manager_client_instance @property - def _deployer_manager_client(self): + def _deployer_manager_client( + self, + ) -> rpc_deployer_manager_client.DeployerManagerClient: if not self._deployer_manager_client_instance: self._deployer_manager_client_instance = ( rpc_deployer_manager_client.DeployerManagerClient() ) return self._deployer_manager_client_instance - def get_all_diagnostics(self, ctxt): + def get_all_diagnostics(self, ctxt: context.RequestContext) -> list[dict]: client_objects = { "conductor": self, "transfer_cron": self._transfer_cron_client, @@ -309,13 +316,13 @@ def get_all_diagnostics(self, ctxt): def _get_worker_service_rpc_for_specs( self, - ctxt, - provider_requirements=None, - region_sets=None, - enabled=True, - random_choice=False, - raise_on_no_matches=True, - ): + ctxt: context.RequestContext, + provider_requirements: dict | None = None, + region_sets: list | None = None, + enabled: bool = True, + random_choice: bool = False, + raise_on_no_matches: bool = True, + ) -> rpc_worker_client.WorkerClient: selected_service = self._scheduler_client.get_worker_service_for_specs( ctxt, provider_requirements=provider_requirements, @@ -327,7 +334,9 @@ def _get_worker_service_rpc_for_specs( service = db_api.get_service(ctxt, selected_service["id"]) return rpc_worker_client.WorkerClient.from_service_definition(service) - def _check_delete_reservation_for_transfer(self, transfer_action): + def _check_delete_reservation_for_transfer( + self, transfer_action: models.Transfer + ) -> None: action_id = transfer_action.base_id if not self._licensing_client: LOG.warn( @@ -350,7 +359,9 @@ def _check_delete_reservation_for_transfer(self, transfer_action): utils.get_exception_details(), ) - def _create_reservation_for_transfer(self, transfer, destination_endpoint): + def _create_reservation_for_transfer( + self, transfer: models.Transfer, destination_endpoint: models.Endpoint + ) -> None: action_id = transfer.base_id scenario = transfer.scenario reservation_type = _get_reservation_type(destination_endpoint, scenario) @@ -387,7 +398,9 @@ def _create_reservation_for_transfer(self, transfer, destination_endpoint): return reservation - def _get_licensing_reservation_for_action(self, transfer_action): + def _get_licensing_reservation_for_action( + self, transfer_action: models.Transfer + ) -> None: action_id = transfer_action.base_id if not self._licensing_client: LOG.warn( @@ -404,8 +417,10 @@ def _get_licensing_reservation_for_action(self, transfer_action): return self._licensing_client.get_reservation(reservation_id) def _check_mark_reservation_fulfilled( - self, transfer_action, must_unfulfilled=False - ): + self, + transfer_action: models.Transfer, + must_unfulfilled: bool = False, + ) -> None: action_id = transfer_action.id reservation = self._get_licensing_reservation_for_action(transfer_action) if not reservation: @@ -438,7 +453,9 @@ def _check_mark_reservation_fulfilled( f"for transfer action '{action_id}' as fulfilled" ) - def _check_reservation_for_transfer(self, ctxt, transfer): + def _check_reservation_for_transfer( + self, ctxt: context.RequestContext, transfer: models.Transfer + ) -> None: scenario = transfer.scenario destination_endpoint = self.get_endpoint(ctxt, transfer.destination_endpoint_id) reservation_type = _get_reservation_type(destination_endpoint, scenario) @@ -524,13 +541,13 @@ def _check_reservation_for_transfer(self, ctxt, transfer): def create_endpoint( self, - ctxt, - name, - endpoint_type, - description, - connection_info, - mapped_regions=None, - ): + ctxt: context.RequestContext, + name: str, + endpoint_type: str, + description: str, + connection_info: dict[str, str], + mapped_regions: list[str] = None, + ) -> models.Endpoint: endpoint = models.Endpoint() endpoint.id = str(uuid.uuid4()) endpoint.name = name @@ -561,7 +578,9 @@ def create_endpoint( return self.get_endpoint(ctxt, endpoint.id) @endpoint_synchronized - def update_endpoint(self, ctxt, endpoint_id, updated_values): + def update_endpoint( + self, ctxt: context.RequestContext, endpoint_id: str, updated_values: dict + ) -> models.Endpoint: LOG.info( "Attempting to update endpoint '%s' with payload: %s", endpoint_id, @@ -571,18 +590,20 @@ def update_endpoint(self, ctxt, endpoint_id, updated_values): LOG.info("Endpoint updated: %s", endpoint_id) return db_api.get_endpoint(ctxt, endpoint_id) - def get_endpoints(self, ctxt): + def get_endpoints(self, ctxt: context.RequestContext) -> list[Any]: return db_api.get_endpoints(ctxt) @endpoint_synchronized - def get_endpoint(self, ctxt, endpoint_id): + def get_endpoint( + self, ctxt: context.RequestContext, endpoint_id: str + ) -> models.Endpoint | None: endpoint = db_api.get_endpoint(ctxt, endpoint_id) if not endpoint: raise exception.NotFound("Endpoint with ID '%s' not found." % endpoint_id) return endpoint @endpoint_synchronized - def delete_endpoint(self, ctxt, endpoint_id): + def delete_endpoint(self, ctxt: context.RequestContext, endpoint_id: str) -> None: q_transfers_count = db_api.get_endpoint_transfers_count(ctxt, endpoint_id) if q_transfers_count != 0: raise exception.NotAuthorized( @@ -592,14 +613,14 @@ def delete_endpoint(self, ctxt, endpoint_id): def get_endpoint_instances( self, - ctxt, - endpoint_id, - source_environment, + ctxt: context.RequestContext, + endpoint_id: str, + source_environment: dict[Any, Any], marker, limit, - instance_name_pattern, - refresh=False, - ): + instance_name_pattern: str | None, + refresh: bool = False, + ) -> list[dict]: endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( @@ -622,8 +643,12 @@ def get_endpoint_instances( ) def get_endpoint_instance( - self, ctxt, endpoint_id, source_environment, instance_name - ): + self, + ctxt: context.RequestContext, + endpoint_id: str, + source_environment: dict[Any, Any], + instance_name: str, + ) -> dict: endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( @@ -643,7 +668,13 @@ def get_endpoint_instance( instance_name, ) - def get_endpoint_source_options(self, ctxt, endpoint_id, env, option_names): + def get_endpoint_source_options( + self, + ctxt: context.RequestContext, + endpoint_id: str, + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( @@ -659,7 +690,13 @@ def get_endpoint_source_options(self, ctxt, endpoint_id, env, option_names): ctxt, endpoint.type, endpoint.connection_info, env, option_names ) - def get_endpoint_destination_options(self, ctxt, endpoint_id, env, option_names): + def get_endpoint_destination_options( + self, + ctxt: context.RequestContext, + endpoint_id: str, + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( @@ -674,7 +711,9 @@ def get_endpoint_destination_options(self, ctxt, endpoint_id, env, option_names) ctxt, endpoint.type, endpoint.connection_info, env, option_names ) - def get_endpoint_networks(self, ctxt, endpoint_id, env): + def get_endpoint_networks( + self, ctxt: context.RequestContext, endpoint_id: str, env: dict[Any, Any] + ) -> list[dict]: endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( @@ -690,7 +729,9 @@ def get_endpoint_networks(self, ctxt, endpoint_id, env): ctxt, endpoint.type, endpoint.connection_info, env ) - def get_endpoint_storage(self, ctxt, endpoint_id, env): + def get_endpoint_storage( + self, ctxt: context.RequestContext, endpoint_id: str, env: dict[Any, Any] + ) -> dict[str, list]: endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( @@ -706,7 +747,12 @@ def get_endpoint_storage(self, ctxt, endpoint_id, env): ctxt, endpoint.type, endpoint.connection_info, env ) - def get_endpoint_inventory_csv(self, ctxt, endpoint_id, source_environment): + def get_endpoint_inventory_csv( + self, + ctxt: context.RequestContext, + endpoint_id: str, + source_environment: dict[Any, Any], + ) -> str: endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( @@ -722,7 +768,9 @@ def get_endpoint_inventory_csv(self, ctxt, endpoint_id, source_environment): ctxt, endpoint.type, endpoint.connection_info, source_environment ) - def validate_endpoint_connection(self, ctxt, endpoint_id): + def validate_endpoint_connection( + self, ctxt: context.RequestContext, endpoint_id: str + ) -> tuple: endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( @@ -736,7 +784,9 @@ def validate_endpoint_connection(self, ctxt, endpoint_id): ctxt, endpoint.type, endpoint.connection_info ) - def validate_endpoint_target_environment(self, ctxt, endpoint_id, target_env): + def validate_endpoint_target_environment( + self, ctxt: context.RequestContext, endpoint_id: str, target_env: dict + ) -> tuple[bool, None]: endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( ctxt, @@ -749,7 +799,9 @@ def validate_endpoint_target_environment(self, ctxt, endpoint_id, target_env): ctxt, endpoint.type, target_env ) - def validate_endpoint_source_environment(self, ctxt, endpoint_id, source_env): + def validate_endpoint_source_environment( + self, ctxt: context.RequestContext, endpoint_id: str, source_env: dict + ) -> tuple[bool, None]: endpoint = self.get_endpoint(ctxt, endpoint_id) worker_rpc = self._get_worker_service_rpc_for_specs( @@ -763,13 +815,15 @@ def validate_endpoint_source_environment(self, ctxt, endpoint_id, source_env): ctxt, endpoint.type, source_env ) - def get_available_providers(self, ctxt): + def get_available_providers(self, ctxt: context.RequestContext) -> dict[str, dict]: worker_rpc = rpc_worker_client.WorkerClient.from_service_definition( self._scheduler_client.get_any_worker_service(ctxt) ) return worker_rpc.get_available_providers(ctxt) - def get_provider_schemas(self, ctxt, platform_name, provider_type): + def get_provider_schemas( + self, ctxt: context.RequestContext, platform_name: str, provider_type: int + ) -> dict[str, dict]: # TODO(aznashwan): merge or version/namespace schemas for each worker? worker_rpc = rpc_worker_client.WorkerClient.from_service_definition( self._scheduler_client.get_any_worker_service(ctxt) @@ -778,13 +832,13 @@ def get_provider_schemas(self, ctxt, platform_name, provider_type): @staticmethod def _create_task( - instance, - task_type, - execution, - depends_on=None, - on_error=False, - on_error_only=False, - ): + instance: str, + task_type: str, + execution: models.TasksExecution, + depends_on: list | None = None, + on_error: bool = False, + on_error_only: bool = False, + ) -> models.Task: """Creates a task with the given properties. NOTE: for on_error and on_error_only tasks, the parent dependencies who @@ -828,7 +882,9 @@ def _create_task( return task - def _get_task_origin(self, ctxt, action): + def _get_task_origin( + self, ctxt: context.RequestContext, action: models.Deployment | models.Transfer + ) -> dict: endpoint = self.get_endpoint(ctxt, action.origin_endpoint_id) return { "connection_info": endpoint.connection_info, @@ -836,7 +892,9 @@ def _get_task_origin(self, ctxt, action): "source_environment": action.source_environment, } - def _get_task_destination(self, ctxt, action): + def _get_task_destination( + self, ctxt: context.RequestContext, action: models.Deployment | models.Transfer + ) -> dict: endpoint = self.get_endpoint(ctxt, action.destination_endpoint_id) return { "connection_info": endpoint.connection_info, @@ -846,14 +904,14 @@ def _get_task_destination(self, ctxt, action): def _get_worker_service_rpc_for_task( self, - ctxt, - task, - origin_endpoint, - destination_endpoint, - retry_count=5, - retry_period=2, - random_choice=True, - ): + ctxt: context.RequestContext, + task: models.Task, + origin_endpoint: models.Endpoint, + destination_endpoint: models.Endpoint, + retry_count: int = 5, + retry_period: int = 2, + random_choice: bool = True, + ) -> rpc_worker_client.WorkerClient: worker_service = None try: worker_service = self._scheduler_client.get_worker_service_for_task( @@ -884,14 +942,14 @@ def _get_worker_service_rpc_for_task( def _begin_tasks( self, - ctxt, - action, - execution, + ctxt: context.RequestContext, + action: models.Deployment | models.Transfer, + execution: models.TasksExecution, task_info_override=None, - scheduling_retry_count=5, - scheduling_retry_period=2, - delete_trust_id=True, - ): + scheduling_retry_count: int = 5, + scheduling_retry_period: int = 2, + delete_trust_id: bool = True, + ) -> None: """Starts all non-error-only tasks which have no depencies.""" if not ctxt.trust_id: keystone.create_trust(ctxt) @@ -962,7 +1020,9 @@ def _begin_tasks( % (execution.id) ) - def _check_execution_tasks_sanity(self, execution, initial_task_info): + def _check_execution_tasks_sanity( + self, execution: models.TasksExecution, initial_task_info: dict + ) -> None: """Checks whether the given execution's tasks are: - properly odered and not set to deadlock off the bat - properly manipulate the task_info in the right order @@ -973,7 +1033,9 @@ def _check_execution_tasks_sanity(self, execution, initial_task_info): if initial_task_info is None: initial_task_info = {} - def _check_task_cls_param_requirements(task, instance_task_info_keys): + def _check_task_cls_param_requirements( + task: models.Task, instance_task_info_keys: set[str] + ) -> list[str]: task_cls = tasks_factory.get_task_runner_class(task.task_type) missing_params = [ p @@ -1083,8 +1145,12 @@ def _check_task_cls_param_requirements(task, instance_task_info_keys): @transfer_synchronized def execute_transfer_tasks( - self, ctxt, transfer_id, shutdown_instances, auto_deploy=False - ): + self, + ctxt: context.RequestContext, + transfer_id: str, + shutdown_instances: bool, + auto_deploy: bool = False, + ) -> dict: transfer = self._get_transfer(ctxt, transfer_id, include_task_info=True) self._check_transfer_running_executions(ctxt, transfer) self._check_minion_pools_for_action(ctxt, transfer) @@ -1353,16 +1419,16 @@ def execute_transfer_tasks( @transfer_synchronized def get_transfer_tasks_executions( self, - ctxt, - transfer_id, - include_tasks=False, - include_task_info=False, - marker=None, - limit=None, - sort_keys=None, - sort_dirs=None, - filters=None, - ): + ctxt: context.RequestContext, + transfer_id: str, + include_tasks: bool = False, + include_task_info: bool = False, + marker: str | None = None, + limit: int | None = None, + sort_keys: list[str] = None, + sort_dirs: list[str] = None, + filters: dict[str, str] = None, + ) -> list: return db_api.get_transfer_tasks_executions( ctxt, transfer_id, @@ -1378,8 +1444,12 @@ def get_transfer_tasks_executions( @tasks_execution_synchronized def get_transfer_tasks_execution( - self, ctxt, transfer_id, execution_id, include_task_info=False - ): + self, + ctxt: context.RequestContext, + transfer_id: str, + execution_id: str, + include_task_info: bool = False, + ) -> dict | None: return self._get_transfer_tasks_execution( ctxt, transfer_id, @@ -1389,7 +1459,9 @@ def get_transfer_tasks_execution( ) @tasks_execution_synchronized - def delete_transfer_tasks_execution(self, ctxt, transfer_id, execution_id): + def delete_transfer_tasks_execution( + self, ctxt: context.RequestContext, transfer_id: str, execution_id: str + ) -> None: execution = self._get_transfer_tasks_execution(ctxt, transfer_id, execution_id) if execution.status in constants.ACTIVE_EXECUTION_STATUSES: raise exception.InvalidActionTasksExecutionState( @@ -1400,7 +1472,13 @@ def delete_transfer_tasks_execution(self, ctxt, transfer_id, execution_id): db_api.delete_transfer_tasks_execution(ctxt, execution_id) @tasks_execution_synchronized - def cancel_transfer_tasks_execution(self, ctxt, transfer_id, execution_id, force): + def cancel_transfer_tasks_execution( + self, + ctxt: context.RequestContext, + transfer_id: str, + execution_id: str, + force: bool, + ) -> None: execution = self._get_transfer_tasks_execution(ctxt, transfer_id, execution_id) if execution.status not in constants.ACTIVE_EXECUTION_STATUSES: raise exception.InvalidTransferState( @@ -1415,8 +1493,12 @@ def cancel_transfer_tasks_execution(self, ctxt, transfer_id, execution_id, force @staticmethod def _get_transfer_tasks_execution( - ctxt, transfer_id, execution_id, include_task_info=False, to_dict=False - ): + ctxt: context.RequestContext, + transfer_id: str, + execution_id: str, + include_task_info: bool = False, + to_dict: bool = False, + ) -> models.TasksExecution | dict: execution = db_api.get_transfer_tasks_execution( ctxt, transfer_id, @@ -1433,15 +1515,15 @@ def _get_transfer_tasks_execution( @staticmethod def get_transfers( - ctxt, - include_tasks_executions=False, - include_task_info=False, - marker=None, - limit=None, - sort_keys=None, - sort_dirs=None, - filters=None, - ): + ctxt: context.RequestContext, + include_tasks_executions: bool = False, + include_task_info: bool = False, + marker: str | None = None, + limit: int | None = None, + sort_keys: list[str] = None, + sort_dirs: list[str] = None, + filters: dict[str, str] = None, + ) -> list[dict]: return db_api.get_transfers( ctxt, include_tasks_executions=include_tasks_executions, @@ -1455,7 +1537,12 @@ def get_transfers( ) @transfer_synchronized - def get_transfer(self, ctxt, transfer_id, include_task_info=False): + def get_transfer( + self, + ctxt: context.RequestContext, + transfer_id: str, + include_task_info: bool = False, + ) -> dict | None: transfer = self._get_transfer( ctxt, transfer_id, include_task_info=include_task_info, to_dict=True ) @@ -1470,14 +1557,16 @@ def get_transfer(self, ctxt, transfer_id, include_task_info=False): return transfer @transfer_synchronized - def delete_transfer(self, ctxt, transfer_id): + def delete_transfer(self, ctxt: context.RequestContext, transfer_id: str) -> None: transfer = self._get_transfer(ctxt, transfer_id) self._check_transfer_running_executions(ctxt, transfer) self._check_delete_reservation_for_transfer(transfer) self._delete_transfer_schedules(ctxt, transfer_id) db_api.delete_transfer(ctxt, transfer_id) - def _delete_transfer_schedules(self, ctxt, transfer_id): + def _delete_transfer_schedules( + self, ctxt: context.RequestContext, transfer_id: str + ) -> None: for schedule in db_api.get_transfer_schedules(ctxt, transfer_id=transfer_id): try: db_api.delete_transfer_schedule( @@ -1495,7 +1584,9 @@ def _delete_transfer_schedules(self, ctxt, transfer_id): ) @transfer_synchronized - def delete_transfer_disks(self, ctxt, transfer_id): + def delete_transfer_disks( + self, ctxt: context.RequestContext, transfer_id: str + ) -> dict: transfer = self._get_transfer(ctxt, transfer_id, include_task_info=True) self._check_transfer_running_executions(ctxt, transfer) @@ -1552,7 +1643,11 @@ def delete_transfer_disks(self, ctxt, transfer_id): return self.get_transfer_tasks_execution(ctxt, transfer_id, execution.id) @staticmethod - def _check_endpoints(ctxt, origin_endpoint, destination_endpoint): + def _check_endpoints( + ctxt: context.RequestContext, + origin_endpoint: models.Endpoint, + destination_endpoint: models.Endpoint, + ) -> None: if origin_endpoint.id == destination_endpoint.id: raise exception.SameDestination( "The origin and destination endpoints cannot be the same. " @@ -1566,23 +1661,23 @@ def _check_endpoints(ctxt, origin_endpoint, destination_endpoint): def create_instances_transfer( self, - ctxt, - transfer_scenario, - origin_endpoint_id, - destination_endpoint_id, - origin_minion_pool_id, - destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, - source_environment, - destination_environment, - instances, - network_map, - storage_mappings, - notes=None, + ctxt: context.RequestContext, + transfer_scenario: str, + origin_endpoint_id: str, + destination_endpoint_id: str, + origin_minion_pool_id: str | None, + destination_minion_pool_id: str | None, + instance_osmorphing_minion_pool_mappings: dict[Any, Any], + source_environment: dict, + destination_environment: dict[str, dict], + instances: list[str], + network_map: dict[Any, Any], + storage_mappings: dict[Any, Any], + notes: str = None, user_scripts=None, - clone_disks=True, - skip_os_morphing=False, - ): + clone_disks: bool = True, + skip_os_morphing: bool = False, + ) -> dict: clustered = len(instances) > 1 supported_scenarios = [ constants.TRANSFER_SCENARIO_REPLICA, @@ -1631,7 +1726,13 @@ def create_instances_transfer( LOG.info("Transfer created: %s", transfer.id) return self.get_transfer(ctxt, transfer.id) - def _get_transfer(self, ctxt, transfer_id, include_task_info=False, to_dict=False): + def _get_transfer( + self, + ctxt: context.RequestContext, + transfer_id: str, + include_task_info: bool = False, + to_dict: bool = False, + ) -> models.Transfer | dict: transfer = db_api.get_transfer( ctxt, transfer_id, include_task_info=include_task_info, to_dict=to_dict ) @@ -1641,15 +1742,15 @@ def _get_transfer(self, ctxt, transfer_id, include_task_info=False, to_dict=Fals @staticmethod def get_deployments( - ctxt, - include_tasks, - include_task_info=False, + ctxt: context.RequestContext, + include_tasks: bool, + include_task_info: bool = False, marker=None, limit=None, - sort_keys=None, - sort_dirs=None, - filters=None, - ): + sort_keys: list | None = None, + sort_dirs: list | None = None, + filters: dict | None = None, + ) -> list: return db_api.get_deployments( ctxt, include_tasks, @@ -1663,12 +1764,19 @@ def get_deployments( ) @deployment_synchronized - def get_deployment(self, ctxt, deployment_id, include_task_info=False): + def get_deployment( + self, + ctxt: context.RequestContext, + deployment_id: str, + include_task_info: bool = False, + ) -> dict | None: return self._get_deployment( ctxt, deployment_id, include_task_info=include_task_info, to_dict=True ) - def _check_running_transfer_deployments(self, ctxt, transfer_id): + def _check_running_transfer_deployments( + self, ctxt: context.RequestContext, transfer_id: str + ) -> None: deployments = db_api.get_transfer_deployments(ctxt, transfer_id) for d in deployments: execution = self._get_execution_for_deployment( @@ -1680,7 +1788,9 @@ def _check_running_transfer_deployments(self, ctxt, transfer_id): ) @staticmethod - def _check_running_executions(ctxt, transfer): + def _check_running_executions( + ctxt: context.RequestContext, transfer: models.Transfer + ) -> None: running_executions = db_api.get_transfer_tasks_executions( ctxt, transfer.id, filters={"status": constants.ACTIVE_EXECUTION_STATUSES} ) @@ -1690,14 +1800,19 @@ def _check_running_executions(ctxt, transfer): % ([e.id for e in running_executions]) ) - def _check_transfer_running_executions(self, ctxt, transfer): + def _check_transfer_running_executions( + self, ctxt: context.RequestContext, transfer: models.Transfer + ) -> None: self._check_running_executions(ctxt, transfer) self._check_running_transfer_deployments(ctxt, transfer.id) @staticmethod def _get_last_transfer_execution( - ctxt, transfer_id, status=None, execution_type=None - ): + ctxt: context.RequestContext, + transfer_id: str, + status: str | None = None, + execution_type: str | None = None, + ) -> models.TasksExecution: filters = {} if status is not None: filters["status"] = status @@ -1713,7 +1828,12 @@ def _get_last_transfer_execution( ) return executions[0] if executions else None - def _check_transfer_deploy_ability(self, ctxt, transfer, force=False): + def _check_transfer_deploy_ability( + self, + ctxt: context.RequestContext, + transfer: models.Transfer, + force: bool = False, + ) -> None: if force: return @@ -1729,13 +1849,21 @@ def _check_transfer_deploy_ability(self, ctxt, transfer, force=False): "once in order to be deployed" ) - def _get_provider_types(self, ctxt, endpoint): + def _get_provider_types( + self, ctxt: context.RequestContext, endpoint: models.Endpoint + ) -> list[int]: provider_types = self.get_available_providers(ctxt).get(endpoint.type) if provider_types is None: raise exception.NotFound("No provider found for: %s" % endpoint.type) return provider_types["types"] - def _validate_deployment_inputs(self, ctxt, deployment, transfer, force): + def _validate_deployment_inputs( + self, + ctxt: context.RequestContext, + deployment: models.Deployment, + transfer: models.Transfer, + force: bool, + ) -> None: self._check_transfer_running_executions(ctxt, transfer) self._check_transfer_deploy_ability(ctxt, transfer, force) for instance, info in transfer.info.items(): @@ -1749,7 +1877,9 @@ def _validate_deployment_inputs(self, ctxt, deployment, transfer, force): self._check_minion_pools_for_action(ctxt, deployment) self._check_reservation_for_transfer(ctxt, transfer) - def _execute_deployment(self, ctxt, deployment, force): + def _execute_deployment( + self, ctxt: context.RequestContext, deployment: models.Deployment, force: bool + ) -> None: transfer = self._get_transfer( ctxt, deployment.transfer_id, include_task_info=True ) @@ -1992,7 +2122,9 @@ def _execute_deployment(self, ctxt, deployment, force): else: self._begin_tasks(ctxt, deployment, execution) - def confirm_deployer_completed(self, ctxt, deployment_id, force=False): + def confirm_deployer_completed( + self, ctxt: context.RequestContext, deployment_id: str, force: bool = False + ) -> None: try: with lockutils.lock( constants.DEPLOYMENT_LOCK_NAME_FORMAT % deployment_id, external=True @@ -2017,7 +2149,12 @@ def confirm_deployer_completed(self, ctxt, deployment_id, force=False): raise @deployment_synchronized - def report_deployer_failure(self, ctxt, deployment_id, deployer_error_details): + def report_deployer_failure( + self, + ctxt: context.RequestContext, + deployment_id: str, + deployer_error_details: str, + ) -> None: error_status = constants.EXECUTION_STATUS_ERROR expected_status = constants.EXECUTION_STATUS_PENDING try: @@ -2044,7 +2181,7 @@ def report_deployer_failure(self, ctxt, deployment_id, deployer_error_details): finally: db_api.set_action_last_execution_status(ctxt, deployment_id, error_status) - def _normalize_user_scripts(self, user_scripts, instances): + def _normalize_user_scripts(self, user_scripts: dict, instances: list[str]) -> dict: """Removes instance user_scripts if said instance is not one of the selected instances for the replica/migration""" if user_scripts is None: @@ -2065,16 +2202,16 @@ def _normalize_user_scripts(self, user_scripts, instances): @transfer_synchronized def deploy_transfer_instances( self, - ctxt, - transfer_id, - force=False, - wait_for_execution=None, - clone_disks=None, - instance_osmorphing_minion_pool_mappings=None, - skip_os_morphing=None, - user_scripts=None, - trust_id=None, - ): + ctxt: context.RequestContext, + transfer_id: str, + force: bool = False, + wait_for_execution: str | None = None, + clone_disks: bool = None, + instance_osmorphing_minion_pool_mappings: dict | None = None, + skip_os_morphing: bool = None, + user_scripts: dict | None = None, + trust_id: str | None = None, + ) -> dict: transfer = self._get_transfer(ctxt, transfer_id, include_task_info=True) if user_scripts is None: @@ -2137,7 +2274,9 @@ def deploy_transfer_instances( return self.get_deployment(ctxt, deployment.id) - def _get_instance_scripts(self, user_scripts, instance): + def _get_instance_scripts( + self, user_scripts: dict, instance: str + ) -> dict[str, dict]: user_scripts = user_scripts or {} ret = { "global": user_scripts.get("global", {}), @@ -2149,12 +2288,16 @@ def _get_instance_scripts(self, user_scripts, instance): ret["instances"][instance] = instance_script return ret - def _deallocate_minion_machines_for_action(self, ctxt, action): + def _deallocate_minion_machines_for_action( + self, ctxt: context.RequestContext, action: models.Deployment | models.Transfer + ) -> None: return self._minion_manager_client.deallocate_minion_machines_for_action( ctxt, action.base_id ) - def _check_minion_pools_for_action(self, ctxt, action): + def _check_minion_pools_for_action( + self, ctxt: context.RequestContext, action: models.Deployment | models.Transfer + ) -> None: self._minion_manager_client.validate_minion_pool_selections_for_action( ctxt, action ) @@ -2163,8 +2306,11 @@ def _check_minion_pools_for_action(self, ctxt, action): ) def _update_task_info_for_minion_allocations( - self, ctxt, action, minion_machine_allocations - ): + self, + ctxt: context.RequestContext, + action: models.Deployment | models.Transfer, + minion_machine_allocations: dict[str, dict], + ) -> None: for instance in action.instances: instance_minion_machines = minion_machine_allocations.get(instance, {}) @@ -2228,8 +2374,12 @@ def _update_task_info_for_minion_allocations( ) def _get_execution_for_deployment( - self, ctxt, deployment, requery=False, raise_on_executions_empty=True - ): + self, + ctxt: context.RequestContext, + deployment: models.Deployment, + requery: bool = False, + raise_on_executions_empty: bool = True, + ) -> models.TasksExecution | None: if requery: deployment = self._get_deployment(ctxt, deployment.id) @@ -2249,8 +2399,11 @@ def _get_execution_for_deployment( @transfer_synchronized def confirm_transfer_minions_allocation( - self, ctxt, transfer_id, minion_machine_allocations - ): + self, + ctxt: context.RequestContext, + transfer_id: str, + minion_machine_allocations: dict[str, dict], + ) -> None: transfer = self._get_transfer(ctxt, transfer_id, include_task_info=True) awaiting_minions_status = constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS @@ -2278,8 +2431,11 @@ def confirm_transfer_minions_allocation( @transfer_synchronized def report_transfer_minions_allocation_error( - self, ctxt, transfer_id, minion_allocation_error_details - ): + self, + ctxt: context.RequestContext, + transfer_id: str, + minion_allocation_error_details: str, + ) -> None: transfer = self._get_transfer(ctxt, transfer_id) awaiting_minions_status = constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS if transfer.last_execution_status != awaiting_minions_status: @@ -2312,8 +2468,11 @@ def report_transfer_minions_allocation_error( @deployment_synchronized def confirm_deployment_minions_allocation( - self, ctxt, deployment_id, minion_machine_allocations - ): + self, + ctxt: context.RequestContext, + deployment_id: str, + minion_machine_allocations: dict[str, dict], + ) -> None: deployment = self._get_deployment(ctxt, deployment_id, include_task_info=True) awaiting_minions_status = constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS @@ -2332,8 +2491,11 @@ def confirm_deployment_minions_allocation( @deployment_synchronized def report_deployment_minions_allocation_error( - self, ctxt, deployment_id, minion_allocation_error_details - ): + self, + ctxt: context.RequestContext, + deployment_id: str, + minion_allocation_error_details: str, + ) -> None: deployment = self._get_deployment(ctxt, deployment_id) awaiting_minions_status = constants.EXECUTION_STATUS_AWAITING_MINION_ALLOCATIONS if deployment.last_execution_status != awaiting_minions_status: @@ -2358,8 +2520,12 @@ def report_deployment_minions_allocation_error( ) def _get_deployment( - self, ctxt, deployment_id, include_task_info=False, to_dict=False - ): + self, + ctxt: context.RequestContext, + deployment_id: str, + include_task_info: bool = False, + to_dict: bool = False, + ) -> models.Deployment | dict: deployment = db_api.get_deployment( ctxt, deployment_id, include_task_info=include_task_info, to_dict=to_dict ) @@ -2369,7 +2535,9 @@ def _get_deployment( ) return deployment - def _delete_deployment(self, ctxt, deployment_id): + def _delete_deployment( + self, ctxt: context.RequestContext, deployment_id: str + ) -> None: deployment = self._get_deployment(ctxt, deployment_id) execution = self._get_execution_for_deployment( ctxt, deployment, raise_on_executions_empty=False @@ -2383,10 +2551,14 @@ def _delete_deployment(self, ctxt, deployment_id): db_api.delete_deployment(ctxt, deployment_id) @deployment_synchronized - def delete_deployment(self, ctxt, deployment_id): + def delete_deployment( + self, ctxt: context.RequestContext, deployment_id: str + ) -> None: self._delete_deployment(ctxt, deployment_id) - def _cancel_deployment(self, ctxt, deployment_id, force): + def _cancel_deployment( + self, ctxt: context.RequestContext, deployment_id: str, force: bool + ) -> None: deployment = self._get_deployment(ctxt, deployment_id) if len(deployment.executions) != 1: raise exception.InvalidDeploymentState( @@ -2415,10 +2587,18 @@ def _cancel_deployment(self, ctxt, deployment_id, force): self._cancel_tasks_execution(ctxt, execution, force=force) @deployment_synchronized - def cancel_deployment(self, ctxt, deployment_id, force): + def cancel_deployment( + self, ctxt: context.RequestContext, deployment_id: str, force: bool + ) -> None: self._cancel_deployment(ctxt, deployment_id, force) - def _cancel_tasks_execution(self, ctxt, execution, requery=True, force=False): + def _cancel_tasks_execution( + self, + ctxt: context.RequestContext, + execution: models.TasksExecution, + requery: bool = True, + force: bool = False, + ) -> None: """Cancels a running Execution by: - telling workers to kill any already running non-on-error tasks - cancelling any non-on-error tasks which are pending @@ -2641,8 +2821,12 @@ def _cancel_tasks_execution(self, ctxt, execution, requery=True, force=False): ) def _abort_peer_sync_barrier_tasks_on_error( - self, ctxt, execution, errored_task, error_message - ): + self, + ctxt: context.RequestContext, + execution: models.TasksExecution, + errored_task: models.Task, + error_message: str, + ) -> None: """Mark peer tasks stuck in SYNCING on the same barrier as failed.""" if errored_task.task_type not in constants.TASK_TYPES_TO_SYNC: return @@ -2667,7 +2851,9 @@ def _abort_peer_sync_barrier_tasks_on_error( ), ) - def _update_reservation_fulfillment_for_execution(self, ctxt, execution): + def _update_reservation_fulfillment_for_execution( + self, ctxt: context.RequestContext, execution: models.TasksExecution + ) -> None: """Updates the reservation fulfillment status for the parent transfer action of the given execution based on its type. @@ -2730,7 +2916,12 @@ def _update_reservation_fulfillment_for_execution(self, ctxt, execution): f"{transfer_action.scenario}." ) - def _set_tasks_execution_status(self, ctxt, execution, new_execution_status): + def _set_tasks_execution_status( + self, + ctxt: context.RequestContext, + execution: models.TasksExecution, + new_execution_status: str, + ) -> None: previous_execution_status = execution.status LOG.info( "Tasks execution %(id)s (action %(action)s) status updated " @@ -2817,7 +3008,9 @@ def _set_tasks_execution_status(self, ctxt, execution, new_execution_status): ) @parent_tasks_execution_synchronized - def set_task_host(self, ctxt, task_id, host): + def set_task_host( + self, ctxt: context.RequestContext, task_id: str, host: str + ) -> None: """Saves the ID of the worker host which has accepted the task to the DB and marks the task as STARTING.""" task = db_api.get_task(ctxt, task_id) @@ -2858,7 +3051,9 @@ def set_task_host(self, ctxt, task_id, host): LOG.info("Successfully set host for task with ID '%s' to '%s'", task_id, host) @parent_tasks_execution_synchronized - def set_task_process(self, ctxt, task_id, process_id): + def set_task_process( + self, ctxt: context.RequestContext, task_id: str, process_id: int + ) -> None: """Sets the ID of the Worker-side process for the given task, and marks the task as actually 'RUNNING'.""" task = db_api.get_task(ctxt, task_id) @@ -2896,8 +3091,12 @@ def set_task_process(self, ctxt, task_id, process_id): ) def _check_clean_execution_deadlock( - self, ctxt, execution, task_statuses=None, requery=True - ): + self, + ctxt: context.RequestContext, + execution: models.TasksExecution, + task_statuses: dict | None = None, + requery: bool = True, + ) -> str: """Checks whether an execution is deadlocked. Deadlocked executions have no currently running/pending tasks but some remaining scheduled tasks. @@ -2952,7 +3151,12 @@ def _check_clean_execution_deadlock( determined_state = constants.EXECUTION_STATUS_DEADLOCKED return determined_state - def _get_execution_status(self, ctxt, execution, requery=False): + def _get_execution_status( + self, + ctxt: context.RequestContext, + execution: models.TasksExecution, + requery: bool = False, + ) -> str: """Returns the global status of an execution. RUNNING - at least one task is RUNNING, STARTING, PENDING or CANCELLING COMPLETED - all non-error-only tasks are COMPLETED @@ -3016,7 +3220,13 @@ def _get_execution_status(self, ctxt, execution, requery=False): ) return status - def _advance_execution_state(self, ctxt, execution, requery=True, instance=None): + def _advance_execution_state( + self, + ctxt: context.RequestContext, + execution: models.TasksExecution, + requery: bool = True, + instance: str | None = None, + ) -> list[str]: """Advances the state of the execution by starting/refreshing the state of all child tasks. If the execution has finalized (either completed or error'd), @@ -3085,7 +3295,7 @@ def _advance_execution_state(self, ctxt, execution, requery=True, instance=None) started_tasks = [] - def _start_task(task): + def _start_task(task: models.Task) -> str: task_info = None if task.instance not in action.info: LOG.error( @@ -3371,15 +3581,24 @@ def _start_task(task): return started_tasks @staticmethod - def _update_transfer_volumes_info(ctxt, transfer_id, instance, updated_task_info): + def _update_transfer_volumes_info( + ctxt: context.RequestContext, + transfer_id: str, + instance: str, + updated_task_info: dict[str, list], + ) -> None: """WARN: the lock for the Transfer must be pre-acquired.""" db_api.update_transfer_action_info_for_instance( ctxt, transfer_id, instance, updated_task_info ) def _update_volumes_info_for_deployment_parent_transfer( - self, ctxt, deployment_id, instance, updated_task_info - ): + self, + ctxt: context.RequestContext, + deployment_id: str, + instance: str, + updated_task_info: dict[str, list], + ) -> None: deployment = db_api.get_deployment(ctxt, deployment_id) transfer_id = deployment.transfer_id @@ -3395,7 +3614,13 @@ def _update_volumes_info_for_deployment_parent_transfer( ctxt, transfer_id, instance, updated_task_info ) - def _handle_task_sync_barrier(self, ctxt, task, execution, action): + def _handle_task_sync_barrier( + self, + ctxt: context.RequestContext, + task: models.Task, + execution: models.TasksExecution, + action: models.Deployment | models.Transfer, + ) -> bool: """Implements a cross-instance sync barrier for clustered actions. If the completed task's type is in TASK_TYPES_TO_SYNC and the @@ -3463,7 +3688,12 @@ def _handle_task_sync_barrier(self, ctxt, task, execution, action): return True - def _assign_clustered_export_disk_owners(self, ctxt, execution, action): + def _assign_clustered_export_disk_owners( + self, + ctxt: context.RequestContext, + execution: models.TasksExecution, + action: models.Transfer, + ) -> None: """Assign an ``owner`` to every disk in each instance's export_info. The owner of a disk is the first instance (in ``action.instances`` @@ -3520,7 +3750,13 @@ def _assign_clustered_export_disk_owners(self, ctxt, execution, action): {d.get("id"): d.get("owner") for d in disks}, ) - def _handle_synced_tasks(self, ctxt, task_type, execution, action): + def _handle_synced_tasks( + self, + ctxt: context.RequestContext, + task_type: str, + execution: models.TasksExecution, + action: models.Transfer, + ) -> None: """Runs type-specific logic after all instances reach a sync barrier. Called once all tasks of a given type across all instances have @@ -3536,10 +3772,18 @@ def _handle_synced_tasks(self, ctxt, task_type, execution, action): execution.id, ) - def _handle_post_task_actions(self, ctxt, task, execution, task_info): + def _handle_post_task_actions( + self, + ctxt: context.RequestContext, + task: models.Task, + execution: models.TasksExecution, + task_info: dict, + ) -> None: task_type = task.task_type - def _check_other_tasks_running(execution, current_task): + def _check_other_tasks_running( + execution: models.TasksExecution, current_task: models.Task + ) -> bool: still_running = False for other_task in execution.tasks: if other_task.id == current_task.id: @@ -3783,7 +4027,9 @@ def _check_other_tasks_running(execution, current_task): ) @parent_tasks_execution_synchronized - def task_completed(self, ctxt, task_id, task_result): + def task_completed( + self, ctxt: context.RequestContext, task_id: str, task_result: dict + ) -> None: LOG.info("Task completed: %s", task_id) task = db_api.get_task(ctxt, task_id) @@ -4082,7 +4328,9 @@ def confirm_task_cancellation(self, ctxt, task_id, cancellation_details): self._advance_execution_state(ctxt, execution, requery=False) @parent_tasks_execution_synchronized - def set_task_error(self, ctxt, task_id, exception_details): + def set_task_error( + self, ctxt: context.RequestContext, task_id: str, exception_details: str + ) -> None: LOG.error( "Received error confirmation for task: %(task_id)s - %(ex)s", {"task_id": task_id, "ex": exception_details}, @@ -4258,8 +4506,13 @@ def add_task_event(self, ctxt, task_id, level, message): @task_synchronized def add_task_progress_update( - self, ctxt, task_id, message, initial_step=0, total_steps=0 - ): + self, + ctxt: context.RequestContext, + task_id: str, + message: str, + initial_step: int = 0, + total_steps: int = 0, + ) -> models.TaskProgressUpdate: LOG.info("Adding task progress update: %s", task_id) task = db_api.get_task(ctxt, task_id) if task.status not in constants.ACTIVE_TASK_STATUSES: @@ -4276,13 +4529,13 @@ def add_task_progress_update( @task_synchronized def update_task_progress_update( self, - ctxt, - task_id, - progress_update_index, - new_current_step, + ctxt: context.RequestContext, + task_id: str, + progress_update_index: int, + new_current_step: int, new_total_steps=None, new_message=None, - ): + ) -> None: LOG.info( "Updating progress update with index '%s' for task %s: %s", progress_update_index, @@ -4299,7 +4552,12 @@ def update_task_progress_update( ) @staticmethod - def _get_transfer_schedule(ctxt, transfer_id, schedule_id, expired=True): + def _get_transfer_schedule( + ctxt: context.RequestContext, + transfer_id: str, + schedule_id: str, + expired: bool = True, + ) -> models.TransferSchedule: schedule = db_api.get_transfer_schedule( ctxt, transfer_id, schedule_id, expired=expired ) @@ -4312,14 +4570,14 @@ def _get_transfer_schedule(ctxt, transfer_id, schedule_id, expired=True): def create_transfer_schedule( self, - ctxt, - transfer_id, - schedule, - enabled, + ctxt: context.RequestContext, + transfer_id: str, + schedule: dict[str, int], + enabled: bool, exp_date, - shutdown_instance, - auto_deploy, - ): + shutdown_instance: bool, + auto_deploy: bool, + ) -> models.TransferSchedule: keystone.create_trust(ctxt) transfer = self._get_transfer(ctxt, transfer_id) transfer_schedule = models.TransferSchedule() @@ -4341,7 +4599,13 @@ def create_transfer_schedule( return self.get_transfer_schedule(ctxt, transfer_id, transfer_schedule.id) @schedule_synchronized - def update_transfer_schedule(self, ctxt, transfer_id, schedule_id, updated_values): + def update_transfer_schedule( + self, + ctxt: context.RequestContext, + transfer_id: str, + schedule_id: str, + updated_values: dict, + ) -> models.TransferSchedule: db_api.update_transfer_schedule( ctxt, transfer_id, @@ -4352,7 +4616,9 @@ def update_transfer_schedule(self, ctxt, transfer_id, schedule_id, updated_value ) return self._get_transfer_schedule(ctxt, transfer_id, schedule_id) - def _cleanup_schedule_resources(self, ctxt, schedule): + def _cleanup_schedule_resources( + self, ctxt: context.RequestContext, schedule: models.TransferSchedule + ) -> None: self._transfer_cron_client.unregister(ctxt, schedule) if schedule.trust_id: tmp_trust = context.get_admin_context(trust_id=schedule.trust_id) @@ -4366,7 +4632,9 @@ def _cleanup_schedule_resources(self, ctxt, schedule): ) @schedule_synchronized - def delete_transfer_schedule(self, ctxt, transfer_id, schedule_id): + def delete_transfer_schedule( + self, ctxt: context.RequestContext, transfer_id: str, schedule_id: str + ) -> None: transfer = self._get_transfer(ctxt, transfer_id) transfer_status = transfer.last_execution_status valid_statuses = list( @@ -4390,19 +4658,32 @@ def delete_transfer_schedule(self, ctxt, transfer_id, schedule_id): ) @transfer_synchronized - def get_transfer_schedules(self, ctxt, transfer_id=None, expired=True): + def get_transfer_schedules( + self, + ctxt: context.RequestContext, + transfer_id: str | None = None, + expired: bool = True, + ) -> list[models.TransferSchedule]: return db_api.get_transfer_schedules( ctxt, transfer_id=transfer_id, expired=expired ) @schedule_synchronized - def get_transfer_schedule(self, ctxt, transfer_id, schedule_id, expired=True): + def get_transfer_schedule( + self, + ctxt: context.RequestContext, + transfer_id: str, + schedule_id: str, + expired: bool = True, + ) -> models.TransferSchedule | None: return self._get_transfer_schedule( ctxt, transfer_id, schedule_id, expired=expired ) @transfer_synchronized - def update_transfer(self, ctxt, transfer_id, updated_properties): + def update_transfer( + self, ctxt: context.RequestContext, transfer_id: str, updated_properties: dict + ) -> dict: transfer = self._get_transfer(ctxt, transfer_id, include_task_info=True) minion_pool_fields = [ @@ -4514,7 +4795,7 @@ def update_transfer(self, ctxt, transfer_id, updated_properties): return self.get_transfer_tasks_execution(ctxt, transfer_id, execution.id) - def get_diagnostics(self, ctxt): + def get_diagnostics(self, ctxt: context.RequestContext) -> dict: diagnostics = utils.get_diagnostics_info() if self._licensing_client: diagnostics['licensing_status'] = ( @@ -4529,7 +4810,13 @@ def get_diagnostics(self, ctxt): ) return diagnostics - def create_region(self, ctxt, region_name, description="", enabled=True): + def create_region( + self, + ctxt: context.RequestContext, + region_name: str, + description: str = "", + enabled: bool = True, + ) -> models.Region: region = models.Region() region.id = str(uuid.uuid4()) region.name = region_name @@ -4538,18 +4825,25 @@ def create_region(self, ctxt, region_name, description="", enabled=True): db_api.add_region(ctxt, region) return self.get_region(ctxt, region.id) - def get_regions(self, ctxt): + def get_regions(self, ctxt: context.RequestContext) -> list[models.Region]: return db_api.get_regions(ctxt) @region_synchronized - def get_region(self, ctxt, region_id): + def get_region( + self, ctxt: context.RequestContext, region_id: str + ) -> models.Region | None: region = db_api.get_region(ctxt, region_id) if not region: raise exception.NotFound("Region with ID '%s' not found." % region_id) return region @region_synchronized - def update_region(self, ctxt, region_id, updated_values): + def update_region( + self, + ctxt: context.RequestContext, + region_id: str, + updated_values: dict[str, str], + ) -> models.Region: LOG.info( "Attempting to update region '%s' with payload: %s", region_id, @@ -4560,22 +4854,22 @@ def update_region(self, ctxt, region_id, updated_values): return db_api.get_region(ctxt, region_id) @region_synchronized - def delete_region(self, ctxt, region_id): + def delete_region(self, ctxt: context.RequestContext, region_id: str) -> None: # TODO(aznashwan): add checks for endpoints/services # associated to the region before deletion: db_api.delete_region(ctxt, region_id) def register_service( self, - ctxt, - host, - binary, - topic, - enabled, - mapped_regions=None, - providers=None, - specs=None, - ): + ctxt: context.RequestContext, + host: str, + binary: str, + topic: str, + enabled: bool, + mapped_regions: list | None = None, + providers: dict | None = None, + specs: dict | None = None, + ) -> models.Service: service = db_api.find_service(ctxt, host, binary, topic=topic) if service: raise exception.Conflict( @@ -4631,7 +4925,9 @@ def register_service( return self.get_service(ctxt, service.id) - def check_service_registered(self, ctxt, host, binary, topic): + def check_service_registered( + self, ctxt: context.RequestContext, host: str, binary: str, topic: str + ) -> None: props = "host='%s', binary='%s', topic='%s'" % (host, binary, topic) LOG.debug("Checking for existence of service with properties: %s", props) service = db_api.find_service(ctxt, host, binary, topic=topic) @@ -4643,18 +4939,22 @@ def check_service_registered(self, ctxt, host, binary, topic): ) return service - def get_services(self, ctxt): + def get_services(self, ctxt: context.RequestContext) -> list[models.Service]: return db_api.get_services(ctxt) @service_synchronized - def get_service(self, ctxt, service_id): + def get_service( + self, ctxt: context.RequestContext, service_id: str + ) -> models.Service: service = db_api.get_service(ctxt, service_id) if not service: raise exception.NotFound("Service with ID '%s' not found." % service_id) return service @service_synchronized - def update_service(self, ctxt, service_id, updated_values): + def update_service( + self, ctxt: context.RequestContext, service_id: str, updated_values: dict + ) -> models.Service: LOG.info( "Attempting to update service '%s' with payload: %s", service_id, @@ -4665,5 +4965,5 @@ def update_service(self, ctxt, service_id, updated_values): return db_api.get_service(ctxt, service_id) @service_synchronized - def delete_service(self, ctxt, service_id): + def delete_service(self, ctxt: context.RequestContext, service_id: str) -> None: db_api.delete_service(ctxt, service_id) diff --git a/coriolis/conductor/rpc/utils.py b/coriolis/conductor/rpc/utils.py index 52e121f3a..d2cf63221 100644 --- a/coriolis/conductor/rpc/utils.py +++ b/coriolis/conductor/rpc/utils.py @@ -13,14 +13,14 @@ def check_create_registration_for_service( conductor_rpc, request_context, - host, - binary, - topic, - enabled=False, + host: str, + binary: str, + topic: str, + enabled: bool = False, mapped_regions=None, - providers=None, - specs=None, - retry_period=30, + providers: dict | None = None, + specs: dict | None = None, + retry_period: int = 30, ): """Checks with the conductor whether or not a service has already been registered for this host and topic and creates one if not. diff --git a/coriolis/conf.py b/coriolis/conf.py index e2dfbcf40..a381878d0 100644 --- a/coriolis/conf.py +++ b/coriolis/conf.py @@ -4,7 +4,7 @@ from oslo_config import cfg -def init_common_opts(): +def init_common_opts() -> None: opts = [ cfg.IntOpt( 'default_requests_timeout', diff --git a/coriolis/context.py b/coriolis/context.py index 2c6ed69f9..52449c655 100644 --- a/coriolis/context.py +++ b/coriolis/context.py @@ -14,25 +14,25 @@ class RequestContext(context.RequestContext): def __init__( self, - user, - project_id, - is_admin=None, - roles=None, + user: str | None, + project_id: str | None, + is_admin: bool | None = None, + roles: list | None = None, project_name=None, remote_address=None, timestamp=None, - request_id=None, + request_id: str | None = None, auth_token=None, - overwrite=True, + overwrite: bool = True, domain_name=None, domain_id=None, user_domain_name=None, user_domain_id=None, project_domain_name=None, project_domain_id=None, - show_deleted=None, - trust_id=None, - delete_trust_id=False, + show_deleted: bool | None = None, + trust_id: str | None = None, + delete_trust_id: bool = False, **kwargs, ): @@ -62,7 +62,7 @@ def __init__( self.trust_id = trust_id self.delete_trust_id = delete_trust_id - def to_dict(self): + def to_dict(self) -> dict: result = super(RequestContext, self).to_dict() result['user'] = self.user result['project_id'] = self.project_id @@ -83,16 +83,16 @@ def to_dict(self): return result @classmethod - def from_dict(cls, values): + def from_dict(cls, values: dict) -> "RequestContext": return cls(**values) - def to_policy_values(self): + def to_policy_values(self) -> context._DeprecatedPolicyValues: policy = super(RequestContext, self).to_policy_values() # TODO(aznashwan): determine if there are any other custom # context params we'd like to be used for policy validation: return policy - def can(self, action, target=None, fatal=True): + def can(self, action: str, target: dict | None = None, fatal: bool = True) -> bool: """Validates policies allow the requested action to be perfomed in the given context, and raises otherwise. """ @@ -113,5 +113,5 @@ def can(self, action, target=None, fatal=True): return result -def get_admin_context(trust_id=None): +def get_admin_context(trust_id: str | None = None) -> RequestContext: return RequestContext(user=None, project_id=None, is_admin=True, trust_id=trust_id) diff --git a/coriolis/cron/cron.py b/coriolis/cron/cron.py index 7f6ec0c40..5e84d22d6 100644 --- a/coriolis/cron/cron.py +++ b/coriolis/cron/cron.py @@ -3,6 +3,7 @@ import sys import threading import time +from typing import Callable import schedule from oslo_log import log @@ -18,14 +19,14 @@ class CronJob(object): def __init__( self, - name, - description, - schedule, - enabled, + name: str, + description: str, + schedule: dict[str, int], + enabled: bool, expires, on_success, on_error, - job_callable, + job_callable: Callable, *args, **kw, ): @@ -80,7 +81,7 @@ def __init__( raise exception.CoriolisException("Invalid expires") self._expires = expires - def _compare(self, pairs): + def _compare(self, pairs: list) -> list[bool]: # we don't support the full cron syntax. Either exact matches # or wildcard matches are allowed. Expressions such as: # */10 (every 10 units) @@ -93,13 +94,13 @@ def _compare(self, pairs): ret.append(False) return ret - def is_expired(self): + def is_expired(self) -> bool: now = timeutils.utcnow() if self._expires and self._expires < now: return True return False - def should_run(self, dt): + def should_run(self, dt: datetime.datetime) -> bool: # NOTE(gsamfira): I find your lack of strong typing....disturbing if type(dt) is not datetime.datetime: raise exception.CoriolisException("Invalid datetime object") @@ -117,12 +118,12 @@ def should_run(self, dt): compared = self._compare(pairs) return False not in compared - def _send_status(self, queue, status): + def _send_status(self, queue: queue.Queue, status: dict) -> None: if not queue: return queue.put(status) - def start(self, status_queue=None): + def start(self, status_queue: queue.Queue = None) -> None: result = None exc_info = None try: @@ -156,7 +157,7 @@ def __init__(self): self._threads = [] self._semaphore = threading.Semaphore(value=1) - def register(self, job): + def register(self, job: CronJob) -> None: if not isinstance(job, CronJob): raise ValueError("Invalid job class") name = job.name @@ -164,14 +165,14 @@ def register(self, job): with self._semaphore: self._jobs[name] = job - def unregister(self, name): + def unregister(self, name: str) -> None: job = self._jobs.get(name) if job: LOG.debug("Unregistering cron job with name '%s'", name) with self._semaphore: del self._jobs[name] - def unregister_jobs_with_prefix(self, prefix): + def unregister_jobs_with_prefix(self, prefix: str) -> None: jobs = [job for job in self._jobs if job.startswith(prefix)] if jobs: LOG.debug( @@ -184,7 +185,7 @@ def unregister_jobs_with_prefix(self, prefix): for job in jobs: del self._jobs[job] - def _check_jobs(self): + def _check_jobs(self) -> None: LOG.debug("Checking cron jobs") jobs = self._jobs.copy() job_nr = len(jobs) @@ -209,12 +210,12 @@ def _check_jobs(self): % {"seconds": delta.seconds, "jobs": spawned} ) - def _loop(self): + def _loop(self) -> None: while not self._should_stop: schedule.run_pending() time.sleep(0.2) - def _result_loop(self): + def _result_loop(self) -> None: while not self._should_stop: job_info = self._queue.get() result = job_info["result"] @@ -232,7 +233,7 @@ def _result_loop(self): "Job %(desc)s returned: %(ret)r" % {"desc": desc, "ret": result} ) - def _janitor(self): + def _janitor(self) -> None: # remove expired jobs from memory. The check for expired # jobs runs once every minute. while not self._should_stop: @@ -248,7 +249,7 @@ def _janitor(self): # No need to run very often. Once a minute should do time.sleep(60) - def start(self): + def start(self) -> None: schedule.every().minute.do(self._check_jobs) self._threads.append(utils.start_thread(self._loop)) # TODO(lpetrut): consider using "schedule" for the janitor job to avoid @@ -256,5 +257,5 @@ def start(self): self._threads.append(utils.start_thread(self._janitor)) self._threads.append(utils.start_thread(self._result_loop)) - def stop(self): + def stop(self) -> None: self._should_stop = True diff --git a/coriolis/data_transfer.py b/coriolis/data_transfer.py index b59dff3f3..639b102cc 100644 --- a/coriolis/data_transfer.py +++ b/coriolis/data_transfer.py @@ -37,7 +37,7 @@ } -def _get_session_and_address(): +def _get_session_and_address() -> tuple: if not CONF.compressor_address: return None, None @@ -61,7 +61,7 @@ def _get_session_and_address(): return (requests.Session(), "http://%s/" % CONF.compressor_address) -def compression_proxy(content, fmt): +def compression_proxy(content: bytes, fmt: str) -> tuple[bytes, bool]: if fmt not in constants.VALID_COMPRESSION_FORMATS: raise exception.CoriolisException( "Invalid compression format requested: %s" % fmt @@ -101,7 +101,9 @@ def compression_proxy(content, fmt): return data, compress -def encode_data(msg_id, path, offset, content, compress=True): +def encode_data( + msg_id: int, path: str, offset: int, content: bytes, compress: bool = True +) -> bytes: inflated_content = path.encode() + b'\0' + struct.pack(" bytes: return struct.pack(" bool: """Indicates if the request context is a normal user.""" if not context: return False @@ -64,7 +65,11 @@ def _model_query(context, *args): return session.query(*args) -def _update_sqlalchemy_object_fields(obj, updateable_fields, values_to_update): +def _update_sqlalchemy_object_fields( + obj: models.Endpoint | models.MinionMachine | models.Service, + updateable_fields: list[str], + values_to_update: dict, +) -> None: """Updates the given 'values_to_update' on the provided sqlalchemy object as long as they are included as 'updateable_fields'. :param obj: object: sqlalchemy object @@ -101,7 +106,10 @@ def _update_sqlalchemy_object_fields(obj, updateable_fields, values_to_update): def _get_transfer_schedules_filter( - context, transfer_id=None, schedule_id=None, expired=True + context, + transfer_id: str | None = None, + schedule_id: str | None = None, + expired: bool = True, ): now = timeutils.utcnow() q = _soft_delete_aware_query(context, models.TransferSchedule) @@ -146,7 +154,7 @@ def _soft_delete_aware_query(context, *args, **kwargs): @enginefacade.reader -def get_endpoints(context): +def get_endpoints(context) -> list[Any]: q = _soft_delete_aware_query(context, models.Endpoint).options( orm.joinedload('mapped_regions') ) @@ -156,7 +164,7 @@ def get_endpoints(context): @enginefacade.reader -def get_endpoint(context, endpoint_id): +def get_endpoint(context, endpoint_id: str) -> models.Endpoint: q = _soft_delete_aware_query(context, models.Endpoint).options( orm.joinedload('mapped_regions') ) @@ -166,14 +174,14 @@ def get_endpoint(context, endpoint_id): @enginefacade.writer -def add_endpoint(context, endpoint): +def add_endpoint(context, endpoint: models.Endpoint) -> None: endpoint.user_id = context.user endpoint.project_id = context.project_id _session(context).add(endpoint) @enginefacade.writer -def update_endpoint(context, endpoint_id, updated_values): +def update_endpoint(context, endpoint_id: str, updated_values: dict) -> None: endpoint = get_endpoint(context, endpoint_id) if not endpoint: raise exception.NotFound("Endpoint with ID '%s' found" % endpoint_id) @@ -184,7 +192,7 @@ def update_endpoint(context, endpoint_id, updated_values): "(type: %s): %s" % (type(updated_values), updated_values) ) - def _try_unmap_regions(region_ids): + def _try_unmap_regions(region_ids: Collection[Any]) -> None: for region_to_unmap in region_ids: try: LOG.debug( @@ -282,7 +290,7 @@ def _try_unmap_regions(region_ids): @enginefacade.writer -def delete_endpoint(context, endpoint_id): +def delete_endpoint(context, endpoint_id: str) -> None: endpoint = get_endpoint(context, endpoint_id) args = {"id": endpoint_id} if is_user_context(context): @@ -304,16 +312,16 @@ def delete_endpoint(context, endpoint_id): @enginefacade.reader def get_transfer_tasks_executions( context, - transfer_id, - include_tasks=False, - include_task_info=False, + transfer_id: str, + include_tasks: bool = False, + include_task_info: bool = False, marker=None, - limit=None, + limit: int | None = None, sort_keys: list[str] | None = None, sort_dirs: list[str] | None = None, filters: dict | None = None, - to_dict=False, -): + to_dict: bool = False, +) -> list: q = _soft_delete_aware_query(context, models.TasksExecution) q = q.join(models.Transfer) if include_task_info: @@ -367,8 +375,12 @@ def get_transfer_tasks_executions( @enginefacade.reader def get_transfer_tasks_execution( - context, transfer_id, execution_id, include_task_info=False, to_dict=False -): + context, + transfer_id: str, + execution_id: str, + include_task_info: bool = False, + to_dict: bool = False, +) -> models.TasksExecution | dict: q = _soft_delete_aware_query(context, models.TasksExecution).join(models.Transfer) if include_task_info: q = q.options(orm.joinedload('action').undefer('info')) @@ -385,7 +397,7 @@ def get_transfer_tasks_execution( @enginefacade.writer -def add_transfer_tasks_execution(context, execution): +def add_transfer_tasks_execution(context, execution: models.TasksExecution) -> None: if is_user_context(context): if execution.action.project_id != context.project_id: raise exception.NotAuthorized() @@ -403,7 +415,7 @@ def add_transfer_tasks_execution(context, execution): @enginefacade.writer -def delete_transfer_tasks_execution(context, execution_id): +def delete_transfer_tasks_execution(context, execution_id: str) -> None: q = _soft_delete_aware_query(context, models.TasksExecution).filter( models.TasksExecution.id == execution_id ) @@ -420,7 +432,11 @@ def delete_transfer_tasks_execution(context, execution_id): @enginefacade.reader -def get_transfer_schedules(context, transfer_id=None, expired=True): +def get_transfer_schedules( + context, + transfer_id: str | None = None, + expired: bool = True, +) -> list[models.TransferSchedule]: sched_filter = _get_transfer_schedules_filter( context, transfer_id=transfer_id, expired=expired ) @@ -428,7 +444,12 @@ def get_transfer_schedules(context, transfer_id=None, expired=True): @enginefacade.reader -def get_transfer_schedule(context, transfer_id, schedule_id, expired=True): +def get_transfer_schedule( + context, + transfer_id: str, + schedule_id: str, + expired: bool = True, +) -> models.TransferSchedule: sched_filter = _get_transfer_schedules_filter( context, transfer_id=transfer_id, schedule_id=schedule_id, expired=expired ) @@ -438,12 +459,12 @@ def get_transfer_schedule(context, transfer_id, schedule_id, expired=True): @enginefacade.writer def update_transfer_schedule( context, - transfer_id, - schedule_id, - updated_values, + transfer_id: str, + schedule_id: str, + updated_values: dict, pre_update_callable=None, - post_update_callable=None, -): + post_update_callable: Callable | None = None, +) -> None: # NOTE(gsamfira): we need to refactor the DB layer a bit to allow # two-phase transactions or at least allow running these functions # inside a single transaction block. @@ -470,11 +491,11 @@ def update_transfer_schedule( @enginefacade.writer def delete_transfer_schedule( context, - transfer_id, - schedule_id, + transfer_id: str, + schedule_id: str, pre_delete_callable=None, - post_delete_callable=None, -): + post_delete_callable: Callable | None = None, +) -> None: # NOTE(gsamfira): we need to refactor the DB layer a bit to allow # two-phase transactions or at least allow running these functions # inside a single transaction block. @@ -503,7 +524,11 @@ def delete_transfer_schedule( @enginefacade.writer -def add_transfer_schedule(context, schedule, post_create_callable=None): +def add_transfer_schedule( + context, + schedule: models.TransferSchedule, + post_create_callable: Callable | None = None, +) -> None: # NOTE(gsamfira): we need to refactor the DB layer a bit to allow # two-phase transactions or at least allow running these functions # inside a single transaction block. @@ -523,15 +548,15 @@ def _get_transfer_with_tasks_executions_options(q): def get_transfers( context, transfer_scenario=None, - include_tasks_executions=False, - include_task_info=False, - marker=None, - limit=None, + include_tasks_executions: bool = False, + include_task_info: bool = False, + marker: models.Transfer | str | None = None, + limit: int | None = None, sort_keys: list[str] | None = None, sort_dirs: list[str] | None = None, filters: dict | None = None, - to_dict=False, -): + to_dict: bool = False, +) -> list[dict]: q = _soft_delete_aware_query(context, models.Transfer) if include_tasks_executions: q = _get_transfer_with_tasks_executions_options(q) @@ -582,8 +607,12 @@ def get_transfers( @enginefacade.reader def get_transfer( - context, transfer_id, transfer_scenario=None, include_task_info=False, to_dict=False -): + context, + transfer_id: str, + transfer_scenario=None, + include_task_info: bool = False, + to_dict: bool = False, +) -> models.Transfer | dict: q = _soft_delete_aware_query(context, models.Transfer) if include_task_info: q = q.options(orm.undefer('info')) @@ -602,7 +631,9 @@ def get_transfer( @enginefacade.reader -def get_endpoint_transfers_count(context, endpoint_id, transfer_scenario=None): +def get_endpoint_transfers_count( + context, endpoint_id: str, transfer_scenario=None +) -> int: scenario_filter_kwargs = {} if transfer_scenario: @@ -628,14 +659,14 @@ def get_endpoint_transfers_count(context, endpoint_id, transfer_scenario=None): @enginefacade.writer -def add_transfer(context, transfer): +def add_transfer(context, transfer: models.Transfer) -> None: transfer.user_id = context.user transfer.project_id = context.project_id _session(context).add(transfer) @enginefacade.writer -def _delete_transfer_action(context, cls, id): +def _delete_transfer_action(context, cls, id: str) -> None: args = {"base_id": id} if is_user_context(context): args["project_id"] = context.project_id @@ -649,12 +680,12 @@ def _delete_transfer_action(context, cls, id): @enginefacade.writer -def delete_transfer(context, transfer_id): +def delete_transfer(context, transfer_id: str) -> None: _delete_transfer_action(context, models.Transfer, transfer_id) @enginefacade.reader -def get_transfer_deployments(context, transfer_id): +def get_transfer_deployments(context, transfer_id: str) -> list[models.Deployment]: q = _soft_delete_aware_query(context, models.Deployment) q = q.join("transfer") q = q.options(orm.joinedload("executions")) @@ -666,15 +697,15 @@ def get_transfer_deployments(context, transfer_id): @enginefacade.reader def get_deployments( context, - include_tasks=False, - include_task_info=False, + include_tasks: bool = False, + include_task_info: bool = False, marker=None, limit=None, sort_keys: list[str] | None = None, sort_dirs: list[str] | None = None, filters: dict | None = None, - to_dict=False, -): + to_dict: bool = False, +) -> list: q = _soft_delete_aware_query(context, models.Deployment) if include_tasks: q = _get_deployment_task_query_options(q) @@ -720,7 +751,9 @@ def get_deployments( return result -def _get_tasks_with_details_options(query): +def _get_tasks_with_details_options( + query, +): return ( query.options(orm.joinedload("action")) .options(orm.joinedload("tasks").joinedload("progress_updates")) @@ -728,7 +761,9 @@ def _get_tasks_with_details_options(query): ) -def _get_deployment_task_query_options(query): +def _get_deployment_task_query_options( + query, +): return ( query.options( orm.joinedload("executions") @@ -741,7 +776,12 @@ def _get_deployment_task_query_options(query): @enginefacade.reader -def get_deployment(context, deployment_id, include_task_info=False, to_dict=False): +def get_deployment( + context, + deployment_id: str, + include_task_info: bool = False, + to_dict: bool = False, +) -> models.Deployment | dict | None: q = _soft_delete_aware_query(context, models.Deployment) q = _get_deployment_task_query_options(q) if include_task_info: @@ -757,19 +797,24 @@ def get_deployment(context, deployment_id, include_task_info=False, to_dict=Fals @enginefacade.writer -def add_deployment(context, deployment): +def add_deployment(context, deployment: models.Deployment) -> None: deployment.user_id = context.user or deployment.transfer.user_id deployment.project_id = context.project_id or deployment.transfer.project_id _session(context).add(deployment) @enginefacade.writer -def delete_deployment(context, deployment_id): +def delete_deployment(context, deployment_id: str) -> None: _delete_transfer_action(context, models.Deployment, deployment_id) @enginefacade.writer -def set_execution_status(context, execution_id, status, update_action_status=True): +def set_execution_status( + context, + execution_id: str, + status: str, + update_action_status: bool = True, +) -> models.TasksExecution: execution = _soft_delete_aware_query(context, models.TasksExecution).join( models.TasksExecution.action ) @@ -788,7 +833,11 @@ def set_execution_status(context, execution_id, status, update_action_status=Tru @enginefacade.reader -def get_action(context, action_id, include_task_info=False): +def get_action( + context, + action_id: str, + include_task_info: bool = False, +) -> models.Deployment | models.Transfer: action = _soft_delete_aware_query(context, models.BaseTransferAction) if include_task_info: action = action.options(orm.undefer('info')) @@ -803,15 +852,22 @@ def get_action(context, action_id, include_task_info=False): @enginefacade.writer -def set_action_last_execution_status(context, action_id, last_execution_status): +def set_action_last_execution_status( + context, + action_id: str, + last_execution_status: str, +) -> None: action = get_action(context, action_id) action.last_execution_status = last_execution_status @enginefacade.writer def update_transfer_action_info_for_instance( - context, action_id, instance, new_instance_info -): + context, + action_id: str, + instance: str, + new_instance_info: dict, +) -> dict: """Updates the info for the given action with the provided dict. Returns the updated value. Sub-fields of the dict already in the info will get overwritten entirely! @@ -862,7 +918,12 @@ def update_transfer_action_info_for_instance( @enginefacade.writer -def set_transfer_action_result(context, action_id, instance, result): +def set_transfer_action_result( + context, + action_id: str, + instance: str, + result: dict, +) -> dict: """Adds the result for the given 'instance' in the 'transfer_result' JSON in the 'base_transfer_action' table. """ @@ -878,7 +939,7 @@ def set_transfer_action_result(context, action_id, instance, result): @enginefacade.reader -def get_tasks_execution(context, execution_id): +def get_tasks_execution(context, execution_id: str) -> models.TasksExecution: q = _soft_delete_aware_query(context, models.TasksExecution) q = q.join(models.BaseTransferAction) q = q.options(orm.joinedload("action")) @@ -891,7 +952,7 @@ def get_tasks_execution(context, execution_id): return execution -def _get_task(context, task_id): +def _get_task(context, task_id: str) -> models.Task: task = _soft_delete_aware_query(context, models.Task).filter_by(id=task_id).first() if not task: raise exception.NotFound("Task not found: %s" % task_id) @@ -899,14 +960,24 @@ def _get_task(context, task_id): @enginefacade.writer -def set_task_status(context, task_id, status, exception_details=None): +def set_task_status( + context, + task_id: str, + status: str, + exception_details: str | None = None, +) -> None: task = _get_task(context, task_id) task.status = status task.exception_details = exception_details @enginefacade.writer -def set_task_host_properties(context, task_id, host=None, process_id=None): +def set_task_host_properties( + context, + task_id: str, + host: str | None = None, + process_id: int | None = None, +) -> None: task = _get_task(context, task_id) if host: task.host = host @@ -915,7 +986,7 @@ def set_task_host_properties(context, task_id, host=None, process_id=None): @enginefacade.reader -def get_task(context, task_id): +def get_task(context, task_id: str) -> models.Task: q = _soft_delete_aware_query(context, models.Task) return q.filter_by(id=task_id).first() @@ -947,7 +1018,9 @@ def _get_last_task_event(context, task_id): @enginefacade.reader -def _get_last_task_progress_update(context, task_id): +def _get_last_task_progress_update( + context, task_id: str +) -> models.TaskProgressUpdate | None: q = _soft_delete_aware_query(context, models.TaskProgressUpdate) last_update = ( q.filter(models.TaskProgressUpdate.task_id == task_id) @@ -958,7 +1031,7 @@ def _get_last_task_progress_update(context, task_id): @enginefacade.reader -def _get_last_minion_pool_event(context, pool_id): +def _get_last_minion_pool_event(context, pool_id: str) -> models.MinionPoolEvent | None: q = _soft_delete_aware_query(context, models.MinionPoolEvent) last_event = ( q.filter(models.MinionPoolEvent.pool_id == pool_id) @@ -980,7 +1053,9 @@ def _get_last_minion_pool_progress_update(context, pool_id): @enginefacade.writer -def add_minion_pool_event(context, pool_id, level, message): +def add_minion_pool_event( + context, pool_id: str, level: str, message: str +) -> models.MinionPoolEvent: pool_event = models.MinionPoolEvent() pool_event.id = str(uuid.uuid4()) pool_event.pool_id = pool_id @@ -1049,7 +1124,9 @@ def update_minion_pool_progress_update( return pool_progress_update -def _get_progress_update(context, task_id, index): +def _get_progress_update( + context, task_id: str, index: int +) -> models.TaskProgressUpdate: q = _soft_delete_aware_query(context, models.TaskProgressUpdate) return q.filter( models.TaskProgressUpdate.task_id == task_id, @@ -1058,7 +1135,13 @@ def _get_progress_update(context, task_id, index): @enginefacade.writer -def add_task_progress_update(context, task_id, message, initial_step=0, total_steps=0): +def add_task_progress_update( + context, + task_id: str, + message: str, + initial_step: int = 0, + total_steps: int = 0, +) -> models.TaskProgressUpdate: task_progress_update = models.TaskProgressUpdate() task_event_id = str(uuid.uuid4()) task_progress_update.id = task_event_id @@ -1087,12 +1170,12 @@ def add_task_progress_update(context, task_id, message, initial_step=0, total_st @enginefacade.writer def update_task_progress_update( context, - task_id, - update_index, - new_current_step, + task_id: str, + update_index: int, + new_current_step: int, new_total_steps=None, new_message=None, -): +) -> None: task_progress_update = _get_progress_update(context, task_id, update_index) if not task_progress_update: raise exception.NotFound( @@ -1135,7 +1218,7 @@ def update_task_progress_update( @enginefacade.writer -def update_transfer(context, transfer_id, updated_values): +def update_transfer(context, transfer_id: str, updated_values: dict) -> None: transfer = get_transfer(context, transfer_id) if not transfer: raise exception.NotFound("Transfer not found") @@ -1181,12 +1264,12 @@ def update_transfer(context, transfer_id, updated_values): @enginefacade.writer -def add_region(context, region): +def add_region(context, region: models.Region) -> None: _session(context).add(region) @enginefacade.reader -def get_regions(context): +def get_regions(context) -> list[models.Region]: q = _soft_delete_aware_query(context, models.Region) q = q.options(orm.joinedload('mapped_endpoints')) q = q.options(orm.joinedload('mapped_services')) @@ -1194,7 +1277,7 @@ def get_regions(context): @enginefacade.reader -def get_region(context, region_id): +def get_region(context, region_id: str) -> models.Region | None: q = _soft_delete_aware_query(context, models.Region) q = q.options(orm.joinedload('mapped_endpoints')) q = q.options(orm.joinedload('mapped_services')) @@ -1202,7 +1285,11 @@ def get_region(context, region_id): @enginefacade.writer -def update_region(context, region_id, updated_values): +def update_region( + context, + region_id: str, + updated_values: dict[str, str], +) -> None: if not region_id: raise exception.InvalidInput("No region ID specified for updating.") region = get_region(context, region_id) @@ -1214,7 +1301,7 @@ def update_region(context, region_id, updated_values): @enginefacade.writer -def delete_region(context, region_id): +def delete_region(context, region_id: str) -> None: region = get_region(context, region_id) count = ( _soft_delete_aware_query(context, models.Region) @@ -1233,7 +1320,10 @@ def delete_region(context, region_id): @enginefacade.writer -def add_endpoint_region_mapping(context, endpoint_region_mapping): +def add_endpoint_region_mapping( + context, + endpoint_region_mapping: models.EndpointRegionMapping, +) -> None: region_id = endpoint_region_mapping.region_id endpoint_id = endpoint_region_mapping.endpoint_id @@ -1256,7 +1346,7 @@ def get_endpoint_region_mapping(context, endpoint_id, region_id): @enginefacade.writer -def delete_endpoint_region_mapping(context, endpoint_id, region_id): +def delete_endpoint_region_mapping(context, endpoint_id: str, region_id: str) -> None: args = {"endpoint_id": endpoint_id, "region_id": region_id} # TODO(aznashwan): many-to-many realtionships have no sane way of # supporting soft deletion from the sqlalchemy layer wihout @@ -1280,7 +1370,11 @@ def delete_endpoint_region_mapping(context, endpoint_id, region_id): @enginefacade.reader -def get_region_mappings_for_endpoint(context, endpoint_id, enabled_regions_only=False): +def get_region_mappings_for_endpoint( + context, + endpoint_id: str, + enabled_regions_only: bool = False, +) -> list[Any]: q = _soft_delete_aware_query(context, models.EndpointRegionMapping) q = q.join(models.Region) q = q.filter(models.EndpointRegionMapping.endpoint_id == endpoint_id) @@ -1298,12 +1392,12 @@ def get_mapped_endpoints_for_region(context, region_id): @enginefacade.writer -def add_service(context, service): +def add_service(context, service: models.Service) -> None: _session(context).add(service) @enginefacade.reader -def get_services(context): +def get_services(context) -> list[models.Service]: q = _soft_delete_aware_query(context, models.Service).options( orm.joinedload('mapped_regions') ) @@ -1311,7 +1405,7 @@ def get_services(context): @enginefacade.reader -def get_service(context, service_id): +def get_service(context, service_id: str) -> models.Service | None: q = _soft_delete_aware_query(context, models.Service).options( orm.joinedload('mapped_regions') ) @@ -1319,7 +1413,12 @@ def get_service(context, service_id): @enginefacade.reader -def find_service(context, host, binary, topic=None): +def find_service( + context, + host: str, + binary: str, + topic: str | None = None, +) -> models.Service | None: args = {"host": host, "binary": binary} if topic: args["topic"] = topic @@ -1332,7 +1431,7 @@ def find_service(context, host, binary, topic=None): @enginefacade.writer -def update_service(context, service_id, updated_values): +def update_service(context, service_id: str, updated_values: dict) -> None: if not service_id: raise exception.InvalidInput("No service ID specified for updating.") service = get_service(context, service_id) @@ -1345,7 +1444,7 @@ def update_service(context, service_id, updated_values): "(type: %s): %s" % (type(updated_values), updated_values) ) - def _try_unmap_regions(region_ids): + def _try_unmap_regions(region_ids: Collection[str]) -> None: for region_to_unmap in region_ids: try: LOG.debug( @@ -1442,7 +1541,7 @@ def _try_unmap_regions(region_ids): @enginefacade.writer -def delete_service(context, service_id): +def delete_service(context, service_id: str) -> None: service = get_service(context, service_id) count = ( _soft_delete_aware_query(context, models.Service) @@ -1459,7 +1558,10 @@ def delete_service(context, service_id): @enginefacade.writer -def add_service_region_mapping(context, service_region_mapping): +def add_service_region_mapping( + context, + service_region_mapping: models.ServiceRegionMapping, +) -> None: region_id = service_region_mapping.region_id service_id = service_region_mapping.service_id @@ -1482,7 +1584,7 @@ def get_service_region_mapping(context, service_id, region_id): @enginefacade.writer -def delete_service_region_mapping(context, service_id, region_id): +def delete_service_region_mapping(context, service_id: str, region_id: str) -> None: args = {"service_id": service_id, "region_id": region_id} # TODO(aznashwan): many-to-many realtionships have no sane way of # supporting soft deletion from the sqlalchemy layer wihout @@ -1501,7 +1603,11 @@ def delete_service_region_mapping(context, service_id, region_id): @enginefacade.reader -def get_region_mappings_for_service(context, service_id, enabled_regions_only=False): +def get_region_mappings_for_service( + context, + service_id: str, + enabled_regions_only: bool = False, +) -> list[models.ServiceRegionMapping]: q = _soft_delete_aware_query(context, models.ServiceRegionMapping) q = q.join(models.Region) q = q.filter(models.ServiceRegionMapping.service_id == service_id) @@ -1519,7 +1625,7 @@ def get_mapped_services_for_region(context, region_id): @enginefacade.writer -def add_minion_machine(context, minion_machine): +def add_minion_machine(context, minion_machine: models.MinionMachine) -> None: minion_machine.user_id = context.user minion_machine.project_id = context.project_id # inherit pool user/tenant if none are given: @@ -1533,7 +1639,9 @@ def add_minion_machine(context, minion_machine): @enginefacade.reader -def get_minion_machines(context, allocated_action_id=None): +def get_minion_machines( + context, allocated_action_id: str | None = None +) -> list[models.MinionMachine]: q = _soft_delete_aware_query(context, models.MinionMachine) if allocated_action_id: q = q.filter(models.MinionMachine.allocated_action == allocated_action_id) @@ -1541,13 +1649,17 @@ def get_minion_machines(context, allocated_action_id=None): @enginefacade.reader -def get_minion_machine(context, minion_machine_id): +def get_minion_machine(context, minion_machine_id: str) -> models.MinionMachine | None: q = _soft_delete_aware_query(context, models.MinionMachine) return q.filter(models.MinionMachine.id == minion_machine_id).first() @enginefacade.writer -def update_minion_machine(context, minion_machine_id, updated_values): +def update_minion_machine( + context, + minion_machine_id: str, + updated_values: dict, +) -> None: if not minion_machine_id: raise exception.InvalidInput("No minion_machine ID specified for updating.") minion_machine = get_minion_machine(context, minion_machine_id) @@ -1569,7 +1681,9 @@ def update_minion_machine(context, minion_machine_id, updated_values): @enginefacade.writer -def set_minion_machine_allocation_status(context, minion_machine_id, status): +def set_minion_machine_allocation_status( + context, minion_machine_id: str, status: str +) -> None: machine = get_minion_machine(context, minion_machine_id) if not machine: raise exception.NotFound( @@ -1590,11 +1704,11 @@ def set_minion_machine_allocation_status(context, minion_machine_id, status): @enginefacade.writer def set_minion_machines_allocation_statuses( context, - minion_machine_ids, - action_id, - allocation_status, - refresh_allocation_time=True, -): + minion_machine_ids: list[str], + action_id: str | None, + allocation_status: str, + refresh_allocation_time: bool = True, +) -> None: machines = get_minion_machines(context) existing_machine_id_mappings = {machine.id: machine for machine in machines} missing = [ @@ -1625,7 +1739,7 @@ def set_minion_machines_allocation_statuses( @enginefacade.writer -def delete_minion_machine(context, minion_machine_id): +def delete_minion_machine(context, minion_machine_id: str) -> None: # TODO(aznashwan): update models to be soft-delete-aware to # avoid needing to hard-delete here: count = ( @@ -1638,14 +1752,14 @@ def delete_minion_machine(context, minion_machine_id): @enginefacade.writer -def add_minion_pool(context, minion_pool): +def add_minion_pool(context, minion_pool: models.MinionPool) -> None: minion_pool.user_id = context.user minion_pool.project_id = context.project_id _session(context).add(minion_pool) @enginefacade.writer -def delete_minion_pool(context, minion_pool_id): +def delete_minion_pool(context, minion_pool_id: str) -> None: args = {"id": minion_pool_id} if is_user_context(context): args["project_id"] = context.project_id @@ -1661,11 +1775,11 @@ def delete_minion_pool(context, minion_pool_id): @enginefacade.reader def get_minion_pool( context, - minion_pool_id, - include_machines=False, - include_events=False, - include_progress_updates=False, -): + minion_pool_id: str, + include_machines: bool = False, + include_events: bool = False, + include_progress_updates: bool = False, +) -> models.MinionPool | None: q = _soft_delete_aware_query(context, models.MinionPool) if include_machines: q = q.options(orm.selectinload('minion_machines')) @@ -1681,11 +1795,11 @@ def get_minion_pool( @enginefacade.reader def get_minion_pools( context, - include_machines=False, - include_events=False, - include_progress_updates=False, - to_dict=True, -): + include_machines: bool = False, + include_events: bool = False, + include_progress_updates: bool = False, + to_dict: bool = True, +) -> list: q = _soft_delete_aware_query(context, models.MinionPool) q = q.filter() if is_user_context(context): @@ -1710,7 +1824,7 @@ def get_minion_pools( @enginefacade.writer -def set_minion_pool_status(context, minion_pool_id, status): +def set_minion_pool_status(context, minion_pool_id: str, status: str) -> None: pool = get_minion_pool(context, minion_pool_id, include_machines=False) if not pool: raise exception.NotFound("Minion pool '%s' not found" % minion_pool_id) @@ -1725,7 +1839,11 @@ def set_minion_pool_status(context, minion_pool_id, status): @enginefacade.writer -def update_minion_pool(context, minion_pool_id, updated_values): +def update_minion_pool( + context, + minion_pool_id: str, + updated_values: dict, +) -> None: lifecycle = get_minion_pool(context, minion_pool_id, include_machines=False) if not lifecycle: raise exception.NotFound("Minion pool '%s' not found" % minion_pool_id) @@ -1764,11 +1882,11 @@ def update_minion_pool(context, minion_pool_id, updated_values): def process_sort_params( - sort_keys, - sort_dirs, - default_keys=None, - default_dir='desc', -): + sort_keys: list | None, + sort_dirs: list | None, + default_keys: list | None = None, + default_dir: str = 'desc', +) -> tuple[list, list]: """Process the sort parameters to include default keys. Creates a list of sort keys and a list of sort directions. Adds the default diff --git a/coriolis/db/sqlalchemy/migration.py b/coriolis/db/sqlalchemy/migration.py index 978bf6fed..43b892b19 100644 --- a/coriolis/db/sqlalchemy/migration.py +++ b/coriolis/db/sqlalchemy/migration.py @@ -20,13 +20,13 @@ LEGACY_FINAL_VERSION = 24 -def _get_alembic_config(): +def _get_alembic_config() -> alembic_config.Config: config = alembic_config.Config(ALEMBIC_INI_PATH) config.set_main_option("script_location", ALEMBIC_DIR) return config -def _stamp_legacy_database_if_needed(engine, config): +def _stamp_legacy_database_if_needed(engine, config: alembic_config.Config) -> None: """Transition a sqlalchemy-migrate managed database to alembic. If this database was previously managed by the old sqlalchemy-migrate @@ -57,7 +57,7 @@ def _stamp_legacy_database_if_needed(engine, config): command.stamp(config, "%03d" % legacy_version) -def db_sync(engine, version=None): +def db_sync(engine, version=None) -> None: config = _get_alembic_config() _stamp_legacy_database_if_needed(engine, config) config.attributes["connection"] = engine.connect() diff --git a/coriolis/db/sqlalchemy/models.py b/coriolis/db/sqlalchemy/models.py index 2cd6bb6a4..232d308ab 100644 --- a/coriolis/db/sqlalchemy/models.py +++ b/coriolis/db/sqlalchemy/models.py @@ -94,7 +94,7 @@ class TaskProgressUpdate( total_steps = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=True) message = sqlalchemy.Column(sqlalchemy.Text, nullable=True) - def to_dict(self): + def to_dict(self) -> dict: result = { "id": self.id, "task_id": self.task_id, @@ -178,7 +178,7 @@ class Task(BASE, models.TimestampMixin, models.SoftDeleteMixin, models.ModelBase order_by=(TaskProgressUpdate.index), ) - def to_dict(self): + def to_dict(self) -> dict: result = { "id": self.id, "execution_id": self.execution_id, @@ -228,7 +228,7 @@ class TasksExecution( number = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) type = sqlalchemy.Column(sqlalchemy.String(255)) - def to_dict(self): + def to_dict(self) -> dict: result = { "id": self.id, "action_id": self.action_id, @@ -308,7 +308,9 @@ class BaseTransferAction( 'polymorphic_on': type, } - def to_dict(self, include_task_info=True, include_executions=True): + def to_dict( + self, include_task_info: bool = True, include_executions: bool = True + ) -> dict: result = { "base_id": self.base_id, "user_id": self.user_id, @@ -364,7 +366,9 @@ class Transfer(BaseTransferAction): 'polymorphic_identity': 'transfer', } - def to_dict(self, include_task_info=True, include_executions=True): + def to_dict( + self, include_task_info: bool = True, include_executions: bool = True + ) -> dict: base = super(Transfer, self).to_dict( include_task_info=include_task_info, include_executions=include_executions ) @@ -393,7 +397,9 @@ class Deployment(BaseTransferAction): 'polymorphic_identity': 'deployment', } - def to_dict(self, include_task_info=True, include_tasks=True): + def to_dict( + self, include_task_info: bool = True, include_tasks: bool = True + ) -> dict: base = super(Deployment, self).to_dict( include_task_info=include_task_info, include_executions=include_tasks ) @@ -545,7 +551,7 @@ class MinionMachine( provider_properties = sqlalchemy.Column(types.Json, nullable=True) - def to_dict(self): + def to_dict(self) -> dict: result = { "id": self.id, "user_id": self.user_id, @@ -615,8 +621,11 @@ class MinionPool(BASE, models.TimestampMixin, models.ModelBase, models.SoftDelet ) def to_dict( - self, include_machines=True, include_events=True, include_progress_updates=True - ): + self, + include_machines: bool = True, + include_events: bool = True, + include_progress_updates: bool = True, + ) -> dict: base = { "id": self.id, "name": self.name, diff --git a/coriolis/db/sqlalchemy/types.py b/coriolis/db/sqlalchemy/types.py index 2d3fcc601..997abb71b 100644 --- a/coriolis/db/sqlalchemy/types.py +++ b/coriolis/db/sqlalchemy/types.py @@ -24,7 +24,7 @@ class LongText(types.TypeDecorator): impl = types.Text - def load_dialect_impl(self, dialect): + def load_dialect_impl(self, dialect) -> mysql.LONGTEXT: if dialect.name == 'mysql': return dialect.type_descriptor(mysql.LONGTEXT()) else: @@ -42,20 +42,36 @@ def load_dialect_impl(self, dialect): class Json(LongText): - def process_bind_param(self, value, dialect): + def process_bind_param( + self, + value: dict | None, + dialect, + ) -> str: return jsonutils.dumps(value) - def process_result_value(self, value, dialect): + def process_result_value( + self, + value: str, + dialect, + ) -> dict | None: if value is None: return None return jsonutils.loads(value) class Bson(Blob): - def process_bind_param(self, value, dialect): + def process_bind_param( + self, + value: dict | None, + dialect, + ) -> bytes: return zlib.compress(jsonutils.dumps(value).encode('utf-8')) - def process_result_value(self, value, dialect): + def process_result_value( + self, + value: bytes, + dialect, + ) -> dict: if value is None: return None data = None @@ -69,16 +85,24 @@ def process_result_value(self, value, dialect): class List(types.TypeDecorator): impl = types.Text - def load_dialect_impl(self, dialect): + def load_dialect_impl(self, dialect) -> mysql.LONGTEXT: if dialect.name == 'mysql': return dialect.type_descriptor(mysql.LONGTEXT()) else: return self.impl - def process_bind_param(self, value, dialect): + def process_bind_param( + self, + value: list | None, + dialect, + ) -> str: return jsonutils.dumps(value) - def process_result_value(self, value, dialect): + def process_result_value( + self, + value: str, + dialect, + ) -> list | None: if value is None: return None return jsonutils.loads(value) diff --git a/coriolis/deployer_manager/rpc/client.py b/coriolis/deployer_manager/rpc/client.py index 267c07c33..14621fef5 100644 --- a/coriolis/deployer_manager/rpc/client.py +++ b/coriolis/deployer_manager/rpc/client.py @@ -27,7 +27,13 @@ def __init__(self, timeout=None): timeout = CONF.deployer_manager.deployer_manager_rpc_timeout super(DeployerManagerClient, self).__init__(target, timeout=timeout) - def execute_auto_deployment(self, ctxt, transfer_id, deployer_id, **kwargs): + def execute_auto_deployment( + self, + ctxt, + transfer_id: str, + deployer_id: str, + **kwargs, + ) -> None: self._cast( ctxt, 'execute_auto_deployment', diff --git a/coriolis/deployer_manager/rpc/server.py b/coriolis/deployer_manager/rpc/server.py index 7ed31ba7c..3c07b1876 100644 --- a/coriolis/deployer_manager/rpc/server.py +++ b/coriolis/deployer_manager/rpc/server.py @@ -22,12 +22,12 @@ def __init__(self): self._init_loop() @property - def _rpc_conductor_client(self): + def _rpc_conductor_client(self) -> rpc_conductor_client.ConductorClient: if not getattr(self, '_conductor_client_instance', None): self._conductor_client_instance = rpc_conductor_client.ConductorClient() return self._conductor_client_instance - def _wait_for_pending_deployment_status_change(self, deployment_id): + def _wait_for_pending_deployment_status_change(self, deployment_id: str) -> None: pending = constants.EXECUTION_STATUS_PENDING LOG.info( f"Waiting for deployment '{deployment_id}' to be out of '{pending}' status" @@ -55,7 +55,7 @@ def _wait_for_pending_deployment_status_change(self, deployment_id): f"'{pending}' status." ) - def _check_deployer_status(self, deployment_id): + def _check_deployer_status(self, deployment_id: str) -> None: active_statuses = [ constants.EXECUTION_STATUS_UNEXECUTED, constants.EXECUTION_STATUS_RUNNING, @@ -137,10 +137,12 @@ def _loop(self): ) time.sleep(10) - def _init_loop(self): + def _init_loop(self) -> None: utils.start_thread(self._loop) - def execute_auto_deployment(self, ctxt, transfer_id, deployer_id, **kwargs): + def execute_auto_deployment( + self, ctxt: context.RequestContext, transfer_id: str, deployer_id: str, **kwargs + ) -> None: LOG.debug( f"Creating deployment for deployer ID '{deployer_id}' of transfer " f"'{transfer_id}'" diff --git a/coriolis/deployments/api.py b/coriolis/deployments/api.py index a449279bd..0d052ea99 100644 --- a/coriolis/deployments/api.py +++ b/coriolis/deployments/api.py @@ -11,13 +11,13 @@ def __init__(self): def deploy_transfer_instances( self, ctxt, - transfer_id, - instance_osmorphing_minion_pool_mappings, - clone_disks=False, - force=False, - skip_os_morphing=False, - user_scripts=None, - ): + transfer_id: str, + instance_osmorphing_minion_pool_mappings: dict[str, str], + clone_disks: bool = False, + force: bool = False, + skip_os_morphing: bool = False, + user_scripts: dict | None = None, + ) -> dict: return self._rpc_client.deploy_transfer_instances( ctxt, transfer_id, @@ -30,23 +30,23 @@ def deploy_transfer_instances( user_scripts=user_scripts, ) - def delete(self, ctxt, deployment_id): + def delete(self, ctxt, deployment_id: str) -> None: self._rpc_client.delete_deployment(ctxt, deployment_id) - def cancel(self, ctxt, deployment_id, force): + def cancel(self, ctxt, deployment_id: str, force: bool) -> None: self._rpc_client.cancel_deployment(ctxt, deployment_id, force) def get_deployments( self, ctxt, - include_tasks=False, - include_task_info=False, + include_tasks: bool = False, + include_task_info: bool = False, marker=None, limit=None, - sort_keys=None, - sort_dirs=None, - filters=None, - ): + sort_keys: list | None = None, + sort_dirs: list | None = None, + filters: dict | None = None, + ) -> list: return self._rpc_client.get_deployments( ctxt, include_tasks, @@ -58,7 +58,12 @@ def get_deployments( filters=filters, ) - def get_deployment(self, ctxt, deployment_id, include_task_info=False): + def get_deployment( + self, + ctxt, + deployment_id: str, + include_task_info: bool = False, + ) -> dict: return self._rpc_client.get_deployment( ctxt, deployment_id, include_task_info=include_task_info ) diff --git a/coriolis/diagnostics/api.py b/coriolis/diagnostics/api.py index a6e703843..c8bc931dd 100644 --- a/coriolis/diagnostics/api.py +++ b/coriolis/diagnostics/api.py @@ -13,7 +13,7 @@ def __init__(self): self._cron_cli = cron_rpc.TransferCronClient() self._worker_cli = worker_rpc.WorkerClient() - def get(self, ctxt): + def get(self, ctxt) -> list[dict]: diag = self._conductor_cli.get_all_diagnostics(ctxt) api = utils.get_diagnostics_info() diag.append(api) diff --git a/coriolis/endpoint_options/api.py b/coriolis/endpoint_options/api.py index 1828a0ef2..9adc3670f 100644 --- a/coriolis/endpoint_options/api.py +++ b/coriolis/endpoint_options/api.py @@ -1,6 +1,7 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. + from coriolis.conductor.rpc import client as rpc_conductor_client from coriolis.minion_manager.rpc import client as rpc_minion_manager_client @@ -13,29 +14,45 @@ def __init__(self): self._rpc_conductor_client = rpc_conductor_client.ConductorClient() def get_endpoint_source_options( - self, ctxt, endpoint_id, env=None, option_names=None - ): + self, + ctxt, + endpoint_id: str, + env: dict | None = None, + option_names: dict | None = None, + ) -> list[dict]: return self._rpc_conductor_client.get_endpoint_source_options( ctxt, endpoint_id, env, option_names ) def get_endpoint_destination_options( - self, ctxt, endpoint_id, env=None, option_names=None - ): + self, + ctxt, + endpoint_id: str, + env: dict | None = None, + option_names: dict | None = None, + ) -> list[dict]: return self._rpc_conductor_client.get_endpoint_destination_options( ctxt, endpoint_id, env, option_names ) def get_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, env=None, option_names=None - ): + self, + ctxt, + endpoint_id: str, + env: dict | None = None, + option_names: dict | None = None, + ) -> list[dict]: return ( self._rpc_minion_manager_client.get_endpoint_source_minion_pool_options )(ctxt, endpoint_id, env, option_names) def get_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, env=None, option_names=None - ): + self, + ctxt, + endpoint_id: str, + env: dict | None = None, + option_names: dict | None = None, + ) -> list[dict]: return ( self._rpc_minion_manager_client.get_endpoint_destination_minion_pool_options )(ctxt, endpoint_id, env, option_names) diff --git a/coriolis/endpoint_resources/api.py b/coriolis/endpoint_resources/api.py index 94d952ea8..85d9dbb84 100644 --- a/coriolis/endpoint_resources/api.py +++ b/coriolis/endpoint_resources/api.py @@ -1,6 +1,8 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from coriolis.conductor.rpc import client as rpc_client @@ -11,13 +13,13 @@ def __init__(self): def get_endpoint_instances( self, ctxt, - endpoint_id, - source_environment, + endpoint_id: str, + source_environment: dict[Any, Any], marker=None, limit=None, - instance_name_pattern=None, - refresh=False, - ): + instance_name_pattern: str | None = None, + refresh: bool = False, + ) -> list[dict]: return self._rpc_client.get_endpoint_instances( ctxt, endpoint_id, @@ -29,19 +31,38 @@ def get_endpoint_instances( ) def get_endpoint_instance( - self, ctxt, endpoint_id, source_environment, instance_name - ): + self, + ctxt, + endpoint_id: str, + source_environment: dict[Any, Any], + instance_name: str, + ) -> dict: return self._rpc_client.get_endpoint_instance( ctxt, endpoint_id, source_environment, instance_name ) - def get_endpoint_networks(self, ctxt, endpoint_id, env): + def get_endpoint_networks( + self, + ctxt, + endpoint_id: str, + env: dict[Any, Any], + ) -> list[dict]: return self._rpc_client.get_endpoint_networks(ctxt, endpoint_id, env) - def get_endpoint_storage(self, ctxt, endpoint_id, env): + def get_endpoint_storage( + self, + ctxt, + endpoint_id: str, + env: dict[Any, Any], + ) -> dict[str, list]: return self._rpc_client.get_endpoint_storage(ctxt, endpoint_id, env) - def get_endpoint_inventory_csv(self, ctxt, endpoint_id, source_environment): + def get_endpoint_inventory_csv( + self, + ctxt, + endpoint_id: str, + source_environment: dict[Any, Any], + ) -> str: return self._rpc_client.get_endpoint_inventory_csv( ctxt, endpoint_id, source_environment ) diff --git a/coriolis/endpoints/api.py b/coriolis/endpoints/api.py index 7b5b05331..38c2b49b7 100644 --- a/coriolis/endpoints/api.py +++ b/coriolis/endpoints/api.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from coriolis import utils from coriolis.conductor.rpc import client as rpc_conductor_client from coriolis.minion_manager.rpc import client as rpc_minion_manager_client @@ -14,53 +16,80 @@ def __init__(self): ) def create( - self, ctxt, name, endpoint_type, description, connection_info, mapped_regions + self, + ctxt, + name: str, + endpoint_type: str, + description: str, + connection_info: dict[str, str], + mapped_regions: list[str], ): return self._rpc_conductor_client.create_endpoint( ctxt, name, endpoint_type, description, connection_info, mapped_regions ) - def update(self, ctxt, endpoint_id, properties): + def update( + self, + ctxt, + endpoint_id: str, + properties: dict, + ): return self._rpc_conductor_client.update_endpoint(ctxt, endpoint_id, properties) - def delete(self, ctxt, endpoint_id): + def delete(self, ctxt, endpoint_id: str) -> None: self._rpc_conductor_client.delete_endpoint(ctxt, endpoint_id) - def get_endpoints(self, ctxt): + def get_endpoints(self, ctxt) -> list[Any]: return self._rpc_conductor_client.get_endpoints(ctxt) - def get_endpoint(self, ctxt, endpoint_id): + def get_endpoint(self, ctxt, endpoint_id: str): return self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id) - def validate_connection(self, ctxt, endpoint_id): + def validate_connection(self, ctxt, endpoint_id: str) -> tuple: return self._rpc_conductor_client.validate_endpoint_connection( ctxt, endpoint_id ) @utils.bad_request_on_error("Invalid destination environment: %s") - def validate_target_environment(self, ctxt, endpoint_id, target_env): + def validate_target_environment( + self, + ctxt, + endpoint_id: str, + target_env: dict, + ) -> tuple[bool, None]: return self._rpc_conductor_client.validate_endpoint_target_environment( ctxt, endpoint_id, target_env ) @utils.bad_request_on_error("Invalid source environment: %s") - def validate_source_environment(self, ctxt, endpoint_id, source_env): + def validate_source_environment( + self, + ctxt, + endpoint_id: str, + source_env: dict, + ) -> tuple[bool, None]: return self._rpc_conductor_client.validate_endpoint_source_environment( ctxt, endpoint_id, source_env ) @utils.bad_request_on_error("Invalid source minion pool environment: %s") def validate_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, pool_environment - ): + self, + ctxt, + endpoint_id: str, + pool_environment: dict[Any, Any], + ) -> tuple[bool, None]: return ( self._rpc_minion_manager_client.validate_endpoint_source_minion_pool_options )(ctxt, endpoint_id, pool_environment) @utils.bad_request_on_error("Invalid destination minion pool environment: %s") def validate_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, pool_environment - ): + self, + ctxt, + endpoint_id: str, + pool_environment: dict[Any, Any], + ) -> tuple[bool, None]: return ( self._rpc_minion_manager_client.validate_endpoint_destination_minion_pool_options )(ctxt, endpoint_id, pool_environment) diff --git a/coriolis/events.py b/coriolis/events.py index 93e6de514..7b1211252 100644 --- a/coriolis/events.py +++ b/coriolis/events.py @@ -18,11 +18,14 @@ class EventManager(object, with_metaclass(abc.ABCMeta)): - def __init__(self, event_handler): + def __init__( + self, + event_handler, + ): self._event_handler = event_handler self._perc_steps = {} - def _call_event_handler(self, method_name, *args, **kwargs): + def _call_event_handler(self, method_name: str, *args, **kwargs): if self._event_handler: method_obj = getattr(self._event_handler, str(method_name), None) if not method_obj: @@ -32,7 +35,9 @@ def _call_event_handler(self, method_name, *args, **kwargs): ) return method_obj(*args, **kwargs) - def add_percentage_step(self, message, total_steps, initial_step=0): + def add_percentage_step( + self, message: str, total_steps: int, initial_step: int = 0 + ) -> _PercStepData: if total_steps < 0: LOG.warn("Max percentage value was negative (%s). Reset to 0", total_steps) total_steps = 0 @@ -64,7 +69,7 @@ def add_percentage_step(self, message, total_steps, initial_step=0): return self._perc_steps[progress_update_id] - def set_percentage_step(self, step, new_current_step): + def set_percentage_step(self, step: _PercStepData, new_current_step: int) -> None: perc_step = self._perc_steps.get(step.progress_update_id, None) if perc_step is None: return @@ -95,7 +100,7 @@ def set_percentage_step(self, step, new_current_step): del perc_step self._perc_steps[perc_id] = _PercStepData(perc_id, perc, 0, total_steps) - def progress_update(self, message): + def progress_update(self, message: str) -> None: self._call_event_handler('add_progress_update', message, return_event=False) def info(self, message): @@ -113,19 +118,23 @@ def error(self, message): class BaseEventHandler(object, with_metaclass(abc.ABCMeta)): @abc.abstractmethod def add_progress_update( - self, message, initial_step=0, total_steps=0, return_event=False + self, + message: str, + initial_step: int = 0, + total_steps: int = 0, + return_event: bool = False, ): pass @abc.abstractmethod def update_progress_update( self, - update_identifier, - new_current_step, + update_identifier: int, + new_current_step: int, new_total_steps=None, new_message=None, - sync=False, - ): + sync: bool = False, + ) -> None: pass @classmethod diff --git a/coriolis/exception.py b/coriolis/exception.py index dd8a95ebe..7a42acc07 100644 --- a/coriolis/exception.py +++ b/coriolis/exception.py @@ -33,7 +33,7 @@ class ConvertedException(webob.exc.WSGIHTTPException): - def __init__(self, code=500, title="", explanation=""): + def __init__(self, code: int = 500, title: str = "", explanation: str = ""): self.code = code # There is a strict rule about constructing status line for HTTP: # '...Status-Line, consisting of the protocol version followed by a @@ -72,7 +72,7 @@ class CoriolisException(Exception): headers = {} safe = False - def __init__(self, message=None, **kwargs): + def __init__(self, message: str | None = None, **kwargs): self.kwargs = kwargs if 'code' not in self.kwargs: @@ -111,7 +111,7 @@ def __init__(self, message=None, **kwargs): self.msg = message super(CoriolisException, self).__init__(message) - def _should_format(self, message): + def _should_format(self, message: str) -> bool: return message is None or '%(message)' in self.message def __unicode__(self): @@ -509,7 +509,7 @@ class MinionMachineCommandTimeout(CoriolisException): class SSHCommandFailed(CoriolisException): - def __init__(self, message=None, exit_code=None, **kwargs): + def __init__(self, message: str = None, exit_code: int = None, **kwargs): super(SSHCommandFailed, self).__init__(message, **kwargs) self.exit_code = exit_code diff --git a/coriolis/i18n.py b/coriolis/i18n.py index 61127f3a7..59db5f7d1 100644 --- a/coriolis/i18n.py +++ b/coriolis/i18n.py @@ -42,7 +42,7 @@ def enable_lazy(enable=True): return i18n.enable_lazy(enable) -def translate(value, user_locale=None): +def translate(value: str, user_locale=None) -> str: return i18n.translate(value, user_locale) diff --git a/coriolis/keystone.py b/coriolis/keystone.py index 9f008525c..96a1e5402 100644 --- a/coriolis/keystone.py +++ b/coriolis/keystone.py @@ -1,6 +1,7 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. + from keystoneauth1 import exceptions as ks_exceptions from keystoneauth1 import loading from keystoneauth1 import session as ks_session @@ -62,7 +63,7 @@ def _get_verify_option(): return verify -def create_trust(ctxt): +def create_trust(ctxt) -> None: if ctxt.trust_id: return diff --git a/coriolis/licensing/client.py b/coriolis/licensing/client.py index 61237e221..313a54234 100644 --- a/coriolis/licensing/client.py +++ b/coriolis/licensing/client.py @@ -29,7 +29,7 @@ def __init__(self, base_url, appliance_id=None, allow_untrusted=False): self._appliance_id = appliance_id @classmethod - def from_env(cls): + def from_env(cls) -> None: """Retuns a `LicensingClient` object instatiated using the following env vars: LICENSING_SERVER_BASE_URL="https://10.7.2.3:37667/v1" diff --git a/coriolis/minion_manager/rpc/client.py b/coriolis/minion_manager/rpc/client.py index 31da9c6d1..04ecded99 100644 --- a/coriolis/minion_manager/rpc/client.py +++ b/coriolis/minion_manager/rpc/client.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + import oslo_messaging as messaging from oslo_config import cfg from oslo_log import log as logging @@ -77,15 +79,23 @@ def add_minion_pool_event(self, ctxt, minion_pool_id, level, message): message=message, ) - def get_diagnostics(self, ctxt): + def get_diagnostics(self, ctxt) -> dict: return self._call(ctxt, 'get_diagnostics') - def validate_minion_pool_selections_for_action(self, ctxt, action): + def validate_minion_pool_selections_for_action( + self, + ctxt, + action, + ) -> None: return self._call( ctxt, 'validate_minion_pool_selections_for_action', action=action ) - def allocate_minion_machines_for_transfer(self, ctxt, transfer): + def allocate_minion_machines_for_transfer( + self, + ctxt, + transfer, + ) -> None: return self._cast( ctxt, 'allocate_minion_machines_for_transfer', transfer=transfer ) @@ -94,9 +104,9 @@ def allocate_minion_machines_for_deployment( self, ctxt, deployment, - include_transfer_minions=True, - include_osmorphing_minions=True, - ): + include_transfer_minions: bool = True, + include_osmorphing_minions: bool = True, + ) -> None: return self._cast( ctxt, 'allocate_minion_machines_for_deployment', @@ -105,12 +115,12 @@ def allocate_minion_machines_for_deployment( include_osmorphing_minions=include_osmorphing_minions, ) - def deallocate_minion_machine(self, ctxt, minion_machine_id): + def deallocate_minion_machine(self, ctxt, minion_machine_id: str) -> None: return self._cast( ctxt, 'deallocate_minion_machine', minion_machine_id=minion_machine_id ) - def deallocate_minion_machines_for_action(self, ctxt, action_id): + def deallocate_minion_machines_for_action(self, ctxt, action_id: str) -> None: return self._cast( ctxt, 'deallocate_minion_machines_for_action', action_id=action_id ) @@ -118,17 +128,17 @@ def deallocate_minion_machines_for_action(self, ctxt, action_id): def create_minion_pool( self, ctxt, - name, - endpoint_id, - pool_platform, - pool_os_type, - environment_options, - minimum_minions, - maximum_minions, - minion_max_idle_time, - minion_retention_strategy, + name: str, + endpoint_id: str, + pool_platform: str, + pool_os_type: str, + environment_options: dict[Any, Any], + minimum_minions: int, + maximum_minions: int, + minion_max_idle_time: int, + minion_retention_strategy: str, notes=None, - skip_allocation=False, + skip_allocation: bool = False, ): return self._call( ctxt, @@ -146,24 +156,34 @@ def create_minion_pool( skip_allocation=skip_allocation, ) - def allocate_minion_pool(self, ctxt, minion_pool_id): + def allocate_minion_pool(self, ctxt, minion_pool_id: str): return self._call(ctxt, "allocate_minion_pool", minion_pool_id=minion_pool_id) - def refresh_minion_pool(self, ctxt, minion_pool_id): + def refresh_minion_pool(self, ctxt, minion_pool_id: str): return self._call(ctxt, "refresh_minion_pool", minion_pool_id=minion_pool_id) - def deallocate_minion_pool(self, ctxt, minion_pool_id, force=False): + def deallocate_minion_pool( + self, + ctxt, + minion_pool_id: str, + force: bool = False, + ): return self._call( ctxt, "deallocate_minion_pool", minion_pool_id=minion_pool_id, force=force ) - def get_minion_pools(self, ctxt): + def get_minion_pools(self, ctxt) -> list: return self._call(ctxt, 'get_minion_pools') - def get_minion_pool(self, ctxt, minion_pool_id): + def get_minion_pool(self, ctxt, minion_pool_id: str): return self._call(ctxt, 'get_minion_pool', minion_pool_id=minion_pool_id) - def update_minion_pool(self, ctxt, minion_pool_id, updated_values): + def update_minion_pool( + self, + ctxt, + minion_pool_id: str, + updated_values: dict, + ): return self._call( ctxt, 'update_minion_pool', @@ -171,12 +191,16 @@ def update_minion_pool(self, ctxt, minion_pool_id, updated_values): updated_values=updated_values, ) - def delete_minion_pool(self, ctxt, minion_pool_id): + def delete_minion_pool(self, ctxt, minion_pool_id: str) -> None: return self._call(ctxt, 'delete_minion_pool', minion_pool_id=minion_pool_id) def get_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, env, option_names - ): + self, + ctxt, + endpoint_id: str, + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_source_minion_pool_options', @@ -186,8 +210,12 @@ def get_endpoint_source_minion_pool_options( ) def get_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, env, option_names - ): + self, + ctxt, + endpoint_id: str, + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_destination_minion_pool_options', @@ -197,8 +225,11 @@ def get_endpoint_destination_minion_pool_options( ) def validate_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, pool_environment - ): + self, + ctxt, + endpoint_id: str, + pool_environment: dict[Any, Any], + ) -> tuple[bool, None]: return self._call( ctxt, 'validate_endpoint_source_minion_pool_options', @@ -207,8 +238,11 @@ def validate_endpoint_source_minion_pool_options( ) def validate_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, pool_environment - ): + self, + ctxt, + endpoint_id: str, + pool_environment: dict[Any, Any], + ) -> tuple[bool, None]: return self._call( ctxt, 'validate_endpoint_destination_minion_pool_options', @@ -218,7 +252,7 @@ def validate_endpoint_destination_minion_pool_options( class MinionManagerPoolRpcEventHandler(events.BaseEventHandler): - def __init__(self, ctxt, pool_id): + def __init__(self, ctxt, pool_id: str): self._ctxt = ctxt self._pool_id = pool_id self._rpc_minion_manager_client_instance = None diff --git a/coriolis/minion_manager/rpc/server.py b/coriolis/minion_manager/rpc/server.py index f3e26d5f2..4ccd1103d 100644 --- a/coriolis/minion_manager/rpc/server.py +++ b/coriolis/minion_manager/rpc/server.py @@ -4,6 +4,7 @@ import datetime import math import uuid +from typing import Any, ValuesView from oslo_config import cfg from oslo_log import log as logging @@ -47,7 +48,11 @@ ) -def _trigger_pool_refresh(ctxt, minion_manager_client, minion_pool_id): +def _trigger_pool_refresh( + ctxt: context.RequestContext, + minion_manager_client: rpc_minion_manager_client.MinionManagerClient, + minion_pool_id: str, +) -> None: try: minion_manager_client.refresh_minion_pool(ctxt, minion_pool_id) except exception.InvalidMinionPoolState as ex: @@ -80,7 +85,7 @@ def __init__(self): utils.get_exception_details(), ) - def _init_pools_refresh_cron_jobs(self): + def _init_pools_refresh_cron_jobs(self) -> None: minion_pools = db_api.get_minion_pools( self._admin_ctxt, include_machines=False, @@ -124,7 +129,9 @@ def _init_pools_refresh_cron_jobs(self): utils.get_exception_details(), ) - def _register_refresh_jobs_for_minion_pool(self, minion_pool, period_minutes=None): + def _register_refresh_jobs_for_minion_pool( + self, minion_pool: models.MinionPool, period_minutes: int | None = None + ) -> None: if period_minutes is None: period_minutes = ( CONF.minion_manager.minion_pool_default_refresh_period_minutes @@ -181,8 +188,8 @@ def _register_refresh_jobs_for_minion_pool(self, minion_pool, period_minutes=Non ) def _unregister_refresh_jobs_for_minion_pool( - self, minion_pool, raise_on_error=True - ): + self, minion_pool: models.MinionPool, raise_on_error: bool = True + ) -> None: job_prefix = MINION_POOL_REFRESH_JOB_PREFIX_FORMAT % (minion_pool.id) try: self._cron.unregister_jobs_with_prefix(job_prefix) @@ -216,29 +223,33 @@ def _rpc_worker_client(self): return self._worker_client_instance @property - def _rpc_scheduler_client(self): + def _rpc_scheduler_client(self) -> rpc_scheduler_client.SchedulerClient: if not getattr(self, '_scheduler_client_instance', None): self._scheduler_client_instance = rpc_scheduler_client.SchedulerClient() return self._scheduler_client_instance @property - def _rpc_conductor_client(self): + def _rpc_conductor_client(self) -> rpc_conductor_client.ConductorClient: if not getattr(self, '_conductor_client_instance', None): self._conductor_client_instance = rpc_conductor_client.ConductorClient() return self._conductor_client_instance @property - def _rpc_minion_manager_client(self): + def _rpc_minion_manager_client( + self, + ) -> rpc_minion_manager_client.MinionManagerClient: if not getattr(self, '_minion_manager_client_instance', None): self._minion_manager_client_instance = ( rpc_minion_manager_client.MinionManagerClient() ) return self._minion_manager_client_instance - def get_diagnostics(self, ctxt): + def get_diagnostics(self, ctxt: context.RequestContext) -> dict: return utils.get_diagnostics_info() - def _check_minion_pool_capability(self, ctxt, platform_name, provider_type): + def _check_minion_pool_capability( + self, ctxt: context.RequestContext, platform_name: str, provider_type: int + ) -> None: # Raises 405 error if the platform does not support minion pools. provider_types = ( self._rpc_conductor_client.get_available_providers(ctxt) @@ -251,8 +262,12 @@ def _check_minion_pool_capability(self, ctxt, platform_name, provider_type): ) def get_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, env, option_names - ): + self, + ctxt: context.RequestContext, + endpoint_id: str, + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id) self._check_minion_pool_capability( ctxt, endpoint['type'], constants.PROVIDER_TYPE_SOURCE_MINION_POOL @@ -275,8 +290,12 @@ def get_endpoint_source_minion_pool_options( ) def get_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, env, option_names - ): + self, + ctxt: context.RequestContext, + endpoint_id: str, + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id) self._check_minion_pool_capability( ctxt, endpoint['type'], constants.PROVIDER_TYPE_DESTINATION_MINION_POOL @@ -299,8 +318,11 @@ def get_endpoint_destination_minion_pool_options( ) def validate_endpoint_source_minion_pool_options( - self, ctxt, endpoint_id, pool_environment - ): + self, + ctxt: context.RequestContext, + endpoint_id: str, + pool_environment: dict[Any, Any], + ) -> tuple[bool, None]: endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id) self._check_minion_pool_capability( ctxt, endpoint['type'], constants.PROVIDER_TYPE_SOURCE_MINION_POOL @@ -323,8 +345,11 @@ def validate_endpoint_source_minion_pool_options( ) def validate_endpoint_destination_minion_pool_options( - self, ctxt, endpoint_id, pool_environment - ): + self, + ctxt: context.RequestContext, + endpoint_id: str, + pool_environment: dict[Any, Any], + ) -> tuple[bool, None]: endpoint = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id) self._check_minion_pool_capability( ctxt, endpoint['type'], constants.PROVIDER_TYPE_DESTINATION_MINION_POOL @@ -346,7 +371,13 @@ def validate_endpoint_destination_minion_pool_options( ctxt, endpoint['type'], pool_environment ) - def _add_minion_pool_event(self, ctxt, minion_pool_id, level, message): + def _add_minion_pool_event( + self, + ctxt: context.RequestContext, + minion_pool_id: str, + level: str, + message: str, + ) -> None: LOG.info("Minion pool event for pool %s: %s", minion_pool_id, message) pool = db_api.get_minion_pool(ctxt, minion_pool_id) db_api.add_minion_pool_event(ctxt, pool.id, level, message) @@ -405,8 +436,11 @@ def update_minion_pool_progress_update( ) def _check_keys_for_action_dict( - self, action, required_action_properties, operation=None - ): + self, + action: dict, + required_action_properties: list[str], + operation: str | None = None, + ) -> None: if not isinstance(action, dict): raise exception.InvalidInput( "Action must be a dict, got '%s': %s" % (type(action), action) @@ -418,7 +452,9 @@ def _check_keys_for_action_dict( "%s: %s. Got %s" % (operation, missing, action) ) - def validate_minion_pool_selections_for_action(self, ctxt, action): + def validate_minion_pool_selections_for_action( + self, ctxt: context.RequestContext, action: dict + ) -> None: """Validates the minion pool selections for a given action.""" required_action_properties = [ 'id', @@ -448,7 +484,7 @@ def validate_minion_pool_selections_for_action(self, ctxt, action): ) } - def _get_pool(pool_id): + def _get_pool(pool_id: str) -> models.MinionPool: pool = minion_pools.get(pool_id) if not pool: raise exception.NotFound( @@ -456,7 +492,11 @@ def _get_pool(pool_id): ) return pool - def _check_pool_minion_count(minion_pool, instances, minion_pool_type=""): + def _check_pool_minion_count( + minion_pool: models.MinionPool, + instances: list[str], + minion_pool_type: str = "", + ) -> None: desired_minion_count = len(instances) if minion_pool.status != constants.MINION_POOL_STATUS_ALLOCATED: raise exception.InvalidMinionPoolState( @@ -639,7 +679,9 @@ def _check_pool_minion_count(minion_pool, instances, minion_pool_type=""): strutils.mask_dict_password(action), ) - def allocate_minion_machines_for_transfer(self, ctxt, transfer): + def allocate_minion_machines_for_transfer( + self, ctxt: context.RequestContext, transfer: dict + ) -> None: try: self._run_machine_allocation_subflow_for_action( ctxt, @@ -667,11 +709,11 @@ def allocate_minion_machines_for_transfer(self, ctxt, transfer): def allocate_minion_machines_for_deployment( self, - ctxt, - deployment, - include_transfer_minions=True, - include_osmorphing_minions=True, - ): + ctxt: context.RequestContext, + deployment: dict, + include_transfer_minions: bool = True, + include_osmorphing_minions: bool = True, + ) -> None: try: self._run_machine_allocation_subflow_for_action( ctxt, @@ -699,13 +741,13 @@ def allocate_minion_machines_for_deployment( def _make_minion_machine_allocation_subflow_for_action( self, - ctxt, - minion_pool, - action_id, - action_instances, - subflow_name, - inject_for_tasks=None, - ): + ctxt: context.RequestContext, + minion_pool: models.MinionPool, + action_id: str, + action_instances: list[str], + subflow_name: str, + inject_for_tasks: dict | None = None, + ) -> dict: """Creates a subflow for allocating minion machines from the provided minion pool to the given action (one for each instance) @@ -747,7 +789,9 @@ def _make_minion_machine_allocation_subflow_for_action( ) ) - def _select_machine(minion_pool, exclude=None): + def _select_machine( + minion_pool: models.MinionPool, exclude: ValuesView[Any] | None = None + ) -> models.MinionMachine: selected_machine = None # NOTE(aznashwan): this will iterate through machines in a set # order every time, thus ensuring that some are preferred over @@ -964,12 +1008,12 @@ def _select_machine(minion_pool, exclude=None): def _run_machine_allocation_subflow_for_action( self, - ctxt, - action, - action_type, - include_transfer_minions=True, - include_osmorphing_minions=True, - ): + ctxt: context.RequestContext, + action: dict, + action_type: str, + include_transfer_minions: bool = True, + include_osmorphing_minions: bool = True, + ) -> linear_flow.Flow: """Defines and starts a taskflow subflow for allocating minion machines for the given action. If there are no more minion machines available, upscaling will occur. @@ -1041,7 +1085,9 @@ def _run_machine_allocation_subflow_for_action( ) pools_used = [] - def _check_pool_allocation_status(minion_pool, minion_pool_type): + def _check_pool_allocation_status( + minion_pool: models.MinionPool, minion_pool_type: str + ) -> None: if minion_pool.status != constants.MINION_POOL_STATUS_ALLOCATED: raise exception.InvalidMinionPoolState( "Minion Pool '%s' is an invalid state ('%s') to be " @@ -1360,7 +1406,9 @@ def _cleanup_machines_with_statuses_for_action( ) db_api.delete_minion_machine(ctxt, machine.id) - def deallocate_minion_machine(self, ctxt, minion_machine_id): + def deallocate_minion_machine( + self, ctxt: context.RequestContext, minion_machine_id: str + ) -> None: minion_machine = db_api.get_minion_machine(ctxt, minion_machine_id) if not minion_machine: @@ -1406,7 +1454,9 @@ def deallocate_minion_machine(self, ctxt, minion_machine_id): "Successfully deallocated minion machine with '%s'.", minion_machine.id ) - def deallocate_minion_machines_for_action(self, ctxt, action_id): + def deallocate_minion_machines_for_action( + self, ctxt: context.RequestContext, action_id: str + ) -> None: allocated_minion_machines = db_api.get_minion_machines( ctxt, allocated_action_id=action_id @@ -1487,13 +1537,13 @@ def deallocate_minion_machines_for_action(self, ctxt, action_id): def _get_healtchcheck_flow_for_minion_machine( self, - minion_pool, - minion_machine, - allocate_to_action=None, - machine_status_on_success=constants.MINION_MACHINE_STATUS_AVAILABLE, # noqa: E501 - power_on_machine=True, - inject_for_tasks=None, - ): + minion_pool: models.MinionPool, + minion_machine: models.MinionMachine, + allocate_to_action: str | None = None, + machine_status_on_success: str = constants.MINION_MACHINE_STATUS_AVAILABLE, # noqa: E501 + power_on_machine: bool = True, + inject_for_tasks: dict | None = None, + ) -> graph_flow.Flow: """Returns a taskflow graph flow with a healtcheck task and redeployment subflow on error.""" # define healthcheck subflow for each machine: @@ -1599,7 +1649,12 @@ def _get_healtchcheck_flow_for_minion_machine( return machine_healthcheck_subflow - def _get_minion_pool_refresh_flow(self, ctxt, minion_pool, requery=True): + def _get_minion_pool_refresh_flow( + self, + ctxt: context.RequestContext, + minion_pool: models.MinionPool, + requery: bool = True, + ) -> unordered_flow.Flow: if requery: minion_pool = self._get_minion_pool( @@ -1817,7 +1872,9 @@ def _get_minion_pool_refresh_flow(self, ctxt, minion_pool, requery=True): return pool_refresh_flow @minion_manager_utils.minion_pool_synchronized_op - def refresh_minion_pool(self, ctxt, minion_pool_id): + def refresh_minion_pool( + self, ctxt: context.RequestContext, minion_pool_id: str + ) -> models.MinionPool: LOG.info("Attempting to healthcheck Minion Pool '%s'.", minion_pool_id) minion_pool = self._get_minion_pool( ctxt, @@ -1876,7 +1933,9 @@ def refresh_minion_pool(self, ctxt, minion_pool_id): include_progress_updates=True, ) - def _get_minion_pool_allocation_flow(self, minion_pool): + def _get_minion_pool_allocation_flow( + self, minion_pool: models.MinionPool + ) -> linear_flow.Flow: """Returns a taskflow.Flow object pertaining to all the tasks required for allocating a minion pool (validation, shared resource setup, and actual minion creation) @@ -1968,19 +2027,19 @@ def _get_minion_pool_allocation_flow(self, minion_pool): def create_minion_pool( self, - ctxt, - name, - endpoint_id, - pool_platform, - pool_os_type, - environment_options, - minimum_minions, - maximum_minions, - minion_max_idle_time, - minion_retention_strategy, + ctxt: context.RequestContext, + name: str, + endpoint_id: str, + pool_platform: str, + pool_os_type: str, + environment_options: dict[Any, Any], + minimum_minions: int, + maximum_minions: int, + minion_max_idle_time: int, + minion_retention_strategy: str, notes=None, - skip_allocation=False, - ): + skip_allocation: bool = False, + ) -> models.MinionPool: endpoint_dict = self._rpc_conductor_client.get_endpoint(ctxt, endpoint_id) minion_pool = models.MinionPool() @@ -2046,7 +2105,12 @@ def create_minion_pool( return self.get_minion_pool(ctxt, minion_pool.id) - def _get_pool_initial_taskflow_store_base(self, ctxt, minion_pool, endpoint_dict): + def _get_pool_initial_taskflow_store_base( + self, + ctxt: context.RequestContext, + minion_pool: models.MinionPool, + endpoint_dict: models.Endpoint, + ) -> dict: # NOTE: considering pools are associated to strictly one endpoint, # we can duplicate the 'origin/destination': origin_info = { @@ -2072,8 +2136,12 @@ def _get_pool_initial_taskflow_store_base(self, ctxt, minion_pool, endpoint_dict return initial_store def _check_pool_machines_in_use( - self, ctxt, minion_pool, raise_if_in_use=False, requery=False - ): + self, + ctxt: context.RequestContext, + minion_pool: models.MinionPool, + raise_if_in_use: bool = False, + requery: bool = False, + ) -> set[Any]: """Checks whether the given pool has any machines currently in-use. Returns a list of the used machines if so, or an empty list of not. """ @@ -2105,7 +2173,9 @@ def _check_pool_machines_in_use( return used_machines @minion_manager_utils.minion_pool_synchronized_op - def allocate_minion_pool(self, ctxt, minion_pool_id): + def allocate_minion_pool( + self, ctxt: context.RequestContext, minion_pool_id: str + ) -> models.MinionPool: LOG.info("Attempting to allocate Minion Pool '%s'.", minion_pool_id) minion_pool = self._get_minion_pool( ctxt, @@ -2168,7 +2238,9 @@ def allocate_minion_pool(self, ctxt, minion_pool_id): include_progress_updates=True, ) - def _get_minion_pool_deallocation_flow(self, minion_pool, raise_on_error=True): + def _get_minion_pool_deallocation_flow( + self, minion_pool: models.MinionPool, raise_on_error: bool = True + ) -> linear_flow.Flow: """Returns a taskflow.Flow object pertaining to all the tasks required for deallocating a minion pool (machines and shared resources) """ @@ -2229,7 +2301,12 @@ def _get_minion_pool_deallocation_flow(self, minion_pool, raise_on_error=True): return deallocation_flow - def _get_pool_deallocation_initial_store(self, ctxt, minion_pool, endpoint_dict): + def _get_pool_deallocation_initial_store( + self, + ctxt: context.RequestContext, + minion_pool: models.MinionPool, + endpoint_dict: models.Endpoint, + ) -> dict: base = self._get_pool_initial_taskflow_store_base( ctxt, minion_pool, endpoint_dict ) @@ -2239,7 +2316,9 @@ def _get_pool_deallocation_initial_store(self, ctxt, minion_pool, endpoint_dict) return base @minion_manager_utils.minion_pool_synchronized_op - def deallocate_minion_pool(self, ctxt, minion_pool_id, force=False): + def deallocate_minion_pool( + self, ctxt: context.RequestContext, minion_pool_id: str, force: bool = False + ) -> models.MinionPool: LOG.info("Attempting to deallocate Minion Pool '%s'.", minion_pool_id) minion_pool = self._get_minion_pool( ctxt, @@ -2332,7 +2411,9 @@ def deallocate_minion_pool(self, ctxt, minion_pool_id, force=False): include_progress_updates=True, ) - def get_minion_pools(self, ctxt, include_machines=True): + def get_minion_pools( + self, ctxt: context.RequestContext, include_machines: bool = True + ) -> list: return db_api.get_minion_pools( ctxt, include_machines=include_machines, @@ -2342,12 +2423,12 @@ def get_minion_pools(self, ctxt, include_machines=True): def _get_minion_pool( self, - ctxt, - minion_pool_id, - include_machines=False, - include_events=False, - include_progress_updates=False, - ): + ctxt: context.RequestContext, + minion_pool_id: str, + include_machines: bool = False, + include_events: bool = False, + include_progress_updates: bool = False, + ) -> models.MinionPool: minion_pool = db_api.get_minion_pool( ctxt, minion_pool_id, @@ -2363,7 +2444,9 @@ def _get_minion_pool( return minion_pool @minion_manager_utils.minion_pool_synchronized_op - def get_minion_pool(self, ctxt, minion_pool_id): + def get_minion_pool( + self, ctxt: context.RequestContext, minion_pool_id: str + ) -> models.MinionPool | None: return self._get_minion_pool( ctxt, minion_pool_id, @@ -2373,7 +2456,9 @@ def get_minion_pool(self, ctxt, minion_pool_id): ) @minion_manager_utils.minion_pool_synchronized_op - def update_minion_pool(self, ctxt, minion_pool_id, updated_values): + def update_minion_pool( + self, ctxt: context.RequestContext, minion_pool_id: str, updated_values: dict + ) -> models.MinionPool: minion_pool = self._get_minion_pool( ctxt, minion_pool_id, include_machines=False ) @@ -2411,7 +2496,9 @@ def update_minion_pool(self, ctxt, minion_pool_id, updated_values): ) @minion_manager_utils.minion_pool_synchronized_op - def delete_minion_pool(self, ctxt, minion_pool_id): + def delete_minion_pool( + self, ctxt: context.RequestContext, minion_pool_id: str + ) -> None: minion_pool = self._get_minion_pool(ctxt, minion_pool_id, include_machines=True) acceptable_deletion_statuses = [ constants.MINION_POOL_STATUS_DEALLOCATED, diff --git a/coriolis/minion_manager/rpc/tasks.py b/coriolis/minion_manager/rpc/tasks.py index 3801fefa7..7ace65f8c 100644 --- a/coriolis/minion_manager/rpc/tasks.py +++ b/coriolis/minion_manager/rpc/tasks.py @@ -74,24 +74,34 @@ class MinionManagerTaskEventMixin(object): # process" as well as forking child processes, it is safest to # re-instantiate the clients every time: @property - def _conductor_client(self): + def _conductor_client(self) -> rpc_conductor_client.ConductorClient: if not getattr(self, '_conductor_client_instance', None): self._conductor_client_instance = rpc_conductor_client.ConductorClient() return self._conductor_client_instance @property - def _minion_manager_client(self): + def _minion_manager_client(self) -> rpc_minion_manager_client.MinionManagerClient: if not getattr(self, '_minion_manager_client_instance', None): self._minion_manager_client_instance = ( rpc_minion_manager_client.MinionManagerClient() ) return self._minion_manager_client_instance - def _add_minion_pool_event(self, context, message, level=constants.TASK_EVENT_INFO): + def _add_minion_pool_event( + self, + context, + message: str, + level: str = constants.TASK_EVENT_INFO, + ) -> None: LOG.debug("Minion pool '%s' event: %s", self._minion_pool_id, message) db_api.add_minion_pool_event(context, self._minion_pool_id, level, message) - def _get_minion_machine(self, context, minion_machine_id, raise_if_not_found=False): + def _get_minion_machine( + self, + context, + minion_machine_id: str, + raise_if_not_found: bool = False, + ) -> models.MinionMachine | None: machine = db_api.get_minion_machine(context, minion_machine_id) if not machine and raise_if_not_found: raise exception.NotFound( @@ -99,27 +109,44 @@ def _get_minion_machine(self, context, minion_machine_id, raise_if_not_found=Fal ) return machine - def _set_minion_pool_status(self, ctxt, minion_pool_id, new_status): + def _set_minion_pool_status( + self, + ctxt, + minion_pool_id: str, + new_status: str, + ) -> None: with minion_manager_utils.get_minion_pool_lock(minion_pool_id, external=True): db_api.set_minion_pool_status(ctxt, minion_pool_id, new_status) def _update_minion_machine( - self, ctxt, minion_pool_id, minion_machine_id, updated_values - ): + self, + ctxt, + minion_pool_id: str, + minion_machine_id: str, + updated_values: dict, + ) -> None: with minion_manager_utils.get_minion_pool_lock(minion_pool_id, external=True): db_api.update_minion_machine(ctxt, minion_machine_id, updated_values) def _set_minion_machine_allocation_status( - self, ctxt, minion_pool_id, minion_machine_id, new_status - ): + self, + ctxt, + minion_pool_id: str, + minion_machine_id: str, + new_status: str, + ) -> None: with minion_manager_utils.get_minion_pool_lock(minion_pool_id, external=True): db_api.set_minion_machine_allocation_status( ctxt, minion_machine_id, new_status ) def _set_minion_machine_power_status( - self, ctxt, minion_pool_id, minion_machine_id, new_status - ): + self, + ctxt, + minion_pool_id: str, + minion_machine_id: str, + new_status: str, + ) -> None: self._update_minion_machine( ctxt, minion_pool_id, minion_machine_id, {"power_status": new_status} ) @@ -132,7 +159,7 @@ class _BaseReportMinionAllocationFailureForActionTask( reports a minion allocation failure to the conductor for the afferent transfer action.""" - def __init__(self, action_id, **kwargs): + def __init__(self, action_id: str, **kwargs): self._action_id = action_id self._task_name = self._get_task_name(action_id) super(_BaseReportMinionAllocationFailureForActionTask, self).__init__( @@ -140,18 +167,23 @@ def __init__(self, action_id, **kwargs): ) @abc.abstractmethod - def _get_task_name(self, action_id): + def _get_task_name(self, action_id: str) -> str: raise NotImplementedError("No allocation failure task name provided") @abc.abstractmethod - def _report_machine_allocation_failure(self, context, action_id, failure_str): + def _report_machine_allocation_failure( + self, + context, + action_id: str, + failure_str: str, + ) -> None: raise NotImplementedError("No allocation failure operation defined") - def execute(self, context): + def execute(self, context) -> None: super(_BaseReportMinionAllocationFailureForActionTask, self).execute() LOG.debug("Nothing to execute for task '%s'", self._task_name) - def revert(self, context, *args, **kwargs): + def revert(self, context, *args, **kwargs) -> None: super(_BaseReportMinionAllocationFailureForActionTask, self).revert( *args, **kwargs ) @@ -175,12 +207,17 @@ def revert(self, context, *args, **kwargs): class ReportMinionAllocationFailureForMigrationTask( _BaseReportMinionAllocationFailureForActionTask ): - def _get_task_name(self, action_id): + def _get_task_name(self, action_id: str) -> str: return MINION_POOL_REPORT_MIGRATION_ALLOCATION_FAILURE_TASK_NAME_FORMAT % ( action_id ) - def _report_machine_allocation_failure(self, context, action_id, failure_str): + def _report_machine_allocation_failure( + self, + context, + action_id: str, + failure_str: str, + ) -> None: self._conductor_client.report_deployment_minions_allocation_error( context, action_id, failure_str ) @@ -189,12 +226,17 @@ def _report_machine_allocation_failure(self, context, action_id, failure_str): class ReportMinionAllocationFailureForReplicaTask( _BaseReportMinionAllocationFailureForActionTask ): - def _get_task_name(self, action_id): + def _get_task_name(self, action_id: str) -> str: return MINION_POOL_REPORT_REPLICA_ALLOCATION_FAILURE_TASK_NAME_FORMAT % ( action_id ) - def _report_machine_allocation_failure(self, context, action_id, failure_str): + def _report_machine_allocation_failure( + self, + context, + action_id: str, + failure_str: str, + ) -> None: self._conductor_client.report_transfer_minions_allocation_error( context, action_id, failure_str ) @@ -207,7 +249,9 @@ class _BaseConfirmMinionAllocationForActionTask( to the conductor. """ - def __init__(self, action_id, allocated_machine_id_mappings, **kwargs): + def __init__( + self, action_id: str, allocated_machine_id_mappings: dict[str, dict], **kwargs + ): """ param allocated_machine_id_mappings: dict of the form: { @@ -230,22 +274,27 @@ def _get_action_label(self): ) @abc.abstractmethod - def _get_task_name(self, action_id): + def _get_task_name(self, action_id: str) -> str: raise NotImplementedError("No minion allocation confirmation task name defined") @abc.abstractmethod def _confirm_machine_allocation_for_action( - self, context, action_id, machine_allocations - ): + self, + context, + action_id: str, + machine_allocations: dict[str, dict], + ) -> None: raise NotImplementedError("No minion allocation confirmation operation defined") - def execute(self, context): + def execute(self, context) -> None: machines_cache = {} machine_allocations = {} def _check_minion_properties( - minion_machine, instance, minion_purpose="unknown" - ): + minion_machine: models.MinionMachine, + instance: str, + minion_purpose: str = "unknown", + ) -> None: if minion_machine.allocation_status != ( constants.MINION_MACHINE_STATUS_IN_USE ): @@ -434,14 +483,17 @@ class ConfirmMinionAllocationForMigrationTask( def _get_action_label(self): return "migration" - def _get_task_name(self, action_id): + def _get_task_name(self, action_id: str) -> str: return MINION_POOL_CONFIRM_MIGRATION_MINION_ALLOCATION_TASK_NAME_FORMAT % ( action_id ) def _confirm_machine_allocation_for_action( - self, context, action_id, machine_allocations - ): + self, + context, + action_id: str, + machine_allocations: dict[str, dict], + ) -> None: self._conductor_client.confirm_deployment_minions_allocation( context, action_id, machine_allocations ) @@ -451,14 +503,17 @@ class ConfirmMinionAllocationForReplicaTask(_BaseConfirmMinionAllocationForActio def _get_action_label(self): return "replica" - def _get_task_name(self, action_id): + def _get_task_name(self, action_id: str) -> str: return MINION_POOL_CONFIRM_REPLICA_MINION_ALLOCATION_TASK_NAME_FORMAT % ( action_id ) def _confirm_machine_allocation_for_action( - self, context, action_id, machine_allocations - ): + self, + context, + action_id: str, + machine_allocations: dict[str, dict], + ) -> None: self._conductor_client.confirm_transfer_minions_allocation( context, action_id, machine_allocations ) @@ -474,7 +529,11 @@ class UpdateMinionPoolStatusTask( default_provides = ["latest_status"] def __init__( - self, minion_pool_id, target_status, status_to_revert_to=None, **kwargs + self, + minion_pool_id: str, + target_status: str, + status_to_revert_to: str | None = None, + **kwargs, ): self._target_status = target_status @@ -488,7 +547,7 @@ def __init__( super(UpdateMinionPoolStatusTask, self).__init__(name=self._task_name, **kwargs) - def execute(self, context, *args): + def execute(self, context, *args) -> str: super(UpdateMinionPoolStatusTask, self).execute(*args) if not self._previous_status: @@ -603,7 +662,11 @@ class BaseMinionManangerTask( default_provides = 'task_info' def __init__( - self, minion_pool_id, minion_machine_id, main_task_runner_type, **kwargs + self, + minion_pool_id: str, + minion_machine_id: str, + main_task_runner_type: str, + **kwargs, ): self._minion_pool_id = minion_pool_id self._minion_machine_id = minion_machine_id @@ -620,10 +683,16 @@ def __init__( ) @abc.abstractmethod - def _get_task_name(self, minion_pool_id, minion_machine_id): + def _get_task_name(self, minion_pool_id: str, minion_machine_id: str) -> str: raise NotImplementedError("No task name providable") - def execute(self, context, origin, destination, task_info): + def execute( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + ) -> dict | None: LOG.info( "Starting minion pool task '%s' (runner type '%s')", self._task_name, @@ -639,7 +708,14 @@ def execute(self, context, origin, destination, task_info): ) return res - def revert(self, context, origin, destination, task_info, **kwargs): + def revert( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + **kwargs, + ) -> None: flow_failures = kwargs.get('flow_failures', {}) self._add_minion_pool_event( context, @@ -660,7 +736,13 @@ def revert(self, context, origin, destination, task_info, **kwargs): class ValidateMinionPoolOptionsTask(BaseMinionManangerTask): - def __init__(self, minion_pool_id, minion_machine_id, minion_pool_type, **kwargs): + def __init__( + self, + minion_pool_id: str, + minion_machine_id: str, + minion_pool_type: str, + **kwargs, + ): task_type = constants.TASK_TYPE_VALIDATE_SOURCE_MINION_POOL_OPTIONS if minion_pool_type != constants.PROVIDER_PLATFORM_SOURCE: task_type = constants.TASK_TYPE_VALIDATE_DESTINATION_MINION_POOL_OPTIONS @@ -668,10 +750,16 @@ def __init__(self, minion_pool_id, minion_machine_id, minion_pool_type, **kwargs minion_pool_id, minion_machine_id, task_type, **kwargs ) - def _get_task_name(self, minion_pool_id, minion_machine_id): + def _get_task_name(self, minion_pool_id: str, minion_machine_id: str) -> str: return MINION_POOL_VALIDATION_TASK_NAME_FORMAT % minion_pool_id - def execute(self, context, origin, destination, task_info): + def execute( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + ) -> None: self._add_minion_pool_event(context, "Validating minion pool options") _ = super(ValidateMinionPoolOptionsTask, self).execute( context, origin, destination, task_info @@ -688,7 +776,13 @@ def revert(self, context, origin, destination, task_info, **kwargs): class AllocateSharedPoolResourcesTask(BaseMinionManangerTask): - def __init__(self, minion_pool_id, minion_machine_id, minion_pool_type, **kwargs): + def __init__( + self, + minion_pool_id: str, + minion_machine_id: str, + minion_pool_type: str, + **kwargs, + ): resource_deployment_task_type = ( constants.TASK_TYPE_SET_UP_SOURCE_POOL_SHARED_RESOURCES @@ -711,10 +805,16 @@ def __init__(self, minion_pool_id, minion_machine_id, minion_pool_type, **kwargs **kwargs, ) - def _get_task_name(self, minion_pool_id, minion_machine_id): + def _get_task_name(self, minion_pool_id: str, minion_machine_id: str) -> str: return MINION_POOL_ALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT % (minion_pool_id) - def execute(self, context, origin, destination, task_info): + def execute( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + ) -> dict: with minion_manager_utils.get_minion_pool_lock( self._minion_pool_id, external=True ): @@ -774,7 +874,13 @@ def revert(self, context, origin, destination, task_info, **kwargs): class DeallocateSharedPoolResourcesTask(BaseMinionManangerTask): - def __init__(self, minion_pool_id, minion_machine_id, minion_pool_type, **kwargs): + def __init__( + self, + minion_pool_id: str, + minion_machine_id: str, + minion_pool_type: str, + **kwargs, + ): resource_deallocation_task = ( constants.TASK_TYPE_TEAR_DOWN_SOURCE_POOL_SHARED_RESOURCES @@ -787,12 +893,18 @@ def __init__(self, minion_pool_id, minion_machine_id, minion_pool_type, **kwargs minion_pool_id, minion_machine_id, resource_deallocation_task, **kwargs ) - def _get_task_name(self, minion_pool_id, minion_machine_id): + def _get_task_name(self, minion_pool_id: str, minion_machine_id: str) -> str: return MINION_POOL_DEALLOCATE_SHARED_RESOURCES_TASK_NAME_FORMAT % ( minion_pool_id ) - def execute(self, context, origin, destination, task_info): + def execute( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + ) -> dict: self._add_minion_pool_event(context, "Deallocating shared pool resources") if 'pool_shared_resources' not in task_info: raise exception.InvalidInput( @@ -823,11 +935,11 @@ def execute(self, context, origin, destination, task_info): class AllocateMinionMachineTask(BaseMinionManangerTask): def __init__( self, - minion_pool_id, - minion_machine_id, - minion_pool_type, - raise_on_cleanup_failure=True, - allocate_to_action=None, + minion_pool_id: str, + minion_machine_id: str, + minion_pool_type: str, + raise_on_cleanup_failure: bool = True, + allocate_to_action: str | None = None, **kwargs, ): resource_deployment_task_type = constants.TASK_TYPE_CREATE_SOURCE_MINION_MACHINE @@ -850,13 +962,19 @@ def __init__( **kwargs, ) - def _get_task_name(self, minion_pool_id, minion_machine_id): + def _get_task_name(self, minion_pool_id: str, minion_machine_id: str) -> str: return MINION_POOL_ALLOCATE_MACHINE_TASK_NAME_FORMAT % ( minion_pool_id, minion_machine_id, ) - def execute(self, context, origin, destination, task_info): + def execute( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + ) -> dict: minion_machine = self._get_minion_machine( context, self._minion_machine_id, raise_if_not_found=False ) @@ -990,7 +1108,14 @@ def execute(self, context, origin, destination, task_info): return task_info - def revert(self, context, origin, destination, task_info, **kwargs): + def revert( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + **kwargs, + ) -> None: minion_provider_properties = None task_info_minion_provider_properties = task_info.get( @@ -1135,10 +1260,10 @@ def revert(self, context, origin, destination, task_info, **kwargs): class DeallocateMinionMachineTask(BaseMinionManangerTask): def __init__( self, - minion_pool_id, - minion_machine_id, - minion_pool_type, - raise_on_cleanup_failure=True, + minion_pool_id: str, + minion_machine_id: str, + minion_pool_type: str, + raise_on_cleanup_failure: bool = True, **kwargs, ): resource_deletion_task_type = constants.TASK_TYPE_DELETE_SOURCE_MINION_MACHINE @@ -1155,13 +1280,19 @@ def __init__( **kwargs, ) - def _get_task_name(self, minion_pool_id, minion_machine_id): + def _get_task_name(self, minion_pool_id: str, minion_machine_id: str) -> str: return MINION_POOL_DEALLOCATE_MACHINE_TASK_NAME_FORMAT % ( minion_pool_id, minion_machine_id, ) - def execute(self, context, origin, destination, task_info): + def execute( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + ) -> dict: machine = self._get_minion_machine(context, self._minion_machine_id) if not machine: LOG.info( @@ -1240,11 +1371,11 @@ class HealthcheckMinionMachineTask(BaseMinionManangerTask): def __init__( self, - minion_pool_id, - minion_machine_id, - minion_pool_type, - fail_on_error=False, - machine_status_on_success=constants.MINION_MACHINE_STATUS_AVAILABLE, # noqa: E501 + minion_pool_id: str, + minion_machine_id: str, + minion_pool_type: str, + fail_on_error: bool = False, + machine_status_on_success: str = constants.MINION_MACHINE_STATUS_AVAILABLE, # noqa: E501 **kwargs, ): self._fail_on_error = fail_on_error @@ -1258,7 +1389,13 @@ def __init__( minion_pool_id, minion_machine_id, resource_healthcheck_task, **kwargs ) - def execute(self, context, origin, destination, task_info): + def execute( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + ) -> dict: res = {"healthy": True, "error": None} machine = self._get_minion_machine( @@ -1359,11 +1496,13 @@ def execute(self, context, origin, destination, task_info): return res - def _get_task_name(self, minion_pool_id, minion_machine_id): + def _get_task_name(self, minion_pool_id: str, minion_machine_id: str) -> str: return self.get_healthcheck_task_name(minion_pool_id, minion_machine_id) @classmethod - def get_healthcheck_task_name(cls, minion_pool_id, minion_machine_id): + def get_healthcheck_task_name( + cls, minion_pool_id: str, minion_machine_id: str + ) -> str: return MINION_POOL_HEALTHCHECK_MACHINE_TASK_NAME_FORMAT % ( minion_pool_id, minion_machine_id, @@ -1374,13 +1513,16 @@ class MinionMachineHealtchcheckDecider(object): """A callable to green/redlight further execution based on the result.""" def __init__( - self, minion_pool_id, minion_machine_id, on_successful_healthcheck=True + self, + minion_pool_id: str, + minion_machine_id: str, + on_successful_healthcheck: bool = True, ): self._minion_pool_id = minion_pool_id self._minion_machine_id = minion_machine_id self._on_success = on_successful_healthcheck - def __call__(self, history): + def __call__(self, history: dict[str, dict]) -> bool: healthcheck_task_name = HealthcheckMinionMachineTask.get_healthcheck_task_name( self._minion_pool_id, self._minion_machine_id ) @@ -1418,10 +1560,10 @@ def __call__(self, history): class PowerOnMinionMachineTask(BaseMinionManangerTask): def __init__( self, - minion_pool_id, - minion_machine_id, - minion_pool_type, - fail_on_error=True, + minion_pool_id: str, + minion_machine_id: str, + minion_pool_type: str, + fail_on_error: bool = True, **kwargs, ): self._fail_on_error = fail_on_error @@ -1432,13 +1574,19 @@ def __init__( minion_pool_id, minion_machine_id, power_on_task_type, **kwargs ) - def _get_task_name(self, minion_pool_id, minion_machine_id): + def _get_task_name(self, minion_pool_id: str, minion_machine_id: str) -> str: return MINION_POOL_POWER_ON_MACHINE_TASK_NAME_FORMAT % ( minion_pool_id, minion_machine_id, ) - def execute(self, context, origin, destination, task_info): + def execute( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + ) -> dict: machine = self._get_minion_machine( context, self._minion_machine_id, raise_if_not_found=True ) @@ -1518,11 +1666,11 @@ def execute(self, context, origin, destination, task_info): class PowerOffMinionMachineTask(BaseMinionManangerTask): def __init__( self, - minion_pool_id, - minion_machine_id, - minion_pool_type, - fail_on_error=True, - status_once_powered_off=constants.MINION_MACHINE_STATUS_AVAILABLE, + minion_pool_id: str, + minion_machine_id: str, + minion_pool_type: str, + fail_on_error: bool = True, + status_once_powered_off: str = constants.MINION_MACHINE_STATUS_AVAILABLE, **kwargs, ): self._fail_on_error = fail_on_error @@ -1534,13 +1682,19 @@ def __init__( minion_pool_id, minion_machine_id, power_on_task_type, **kwargs ) - def _get_task_name(self, minion_pool_id, minion_machine_id): + def _get_task_name(self, minion_pool_id: str, minion_machine_id: str) -> str: return MINION_POOL_POWER_OFF_MACHINE_TASK_NAME_FORMAT % ( minion_pool_id, minion_machine_id, ) - def execute(self, context, origin, destination, task_info): + def execute( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + ) -> dict: machine = self._get_minion_machine( context, self._minion_machine_id, raise_if_not_found=True ) diff --git a/coriolis/minion_manager/rpc/utils.py b/coriolis/minion_manager/rpc/utils.py index 697e4cb54..74508322b 100644 --- a/coriolis/minion_manager/rpc/utils.py +++ b/coriolis/minion_manager/rpc/utils.py @@ -2,19 +2,20 @@ # All Rights Reserved. import functools +from typing import Callable from oslo_concurrency import lockutils from coriolis import constants -def get_minion_pool_lock(minion_pool_id, external=True): +def get_minion_pool_lock(minion_pool_id: str, external: bool = True): return lockutils.lock( constants.MINION_POOL_LOCK_NAME_FORMAT % minion_pool_id, external=external ) -def minion_pool_synchronized(minion_pool_id, func): +def minion_pool_synchronized(minion_pool_id: str, func: Callable) -> Callable: @functools.wraps(func) def wrapper(*args, **kwargs): @lockutils.synchronized( @@ -28,7 +29,7 @@ def inner(): return wrapper -def minion_pool_synchronized_op(func): +def minion_pool_synchronized_op(func: Callable) -> Callable: @functools.wraps(func) def wrapper(self, ctxt, minion_pool_id, *args, **kwargs): return minion_pool_synchronized(minion_pool_id, func)( diff --git a/coriolis/minion_pools/api.py b/coriolis/minion_pools/api.py index a007f3339..6907bcbcd 100644 --- a/coriolis/minion_pools/api.py +++ b/coriolis/minion_pools/api.py @@ -1,6 +1,8 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from coriolis.minion_manager.rpc import client as rpc_client @@ -11,17 +13,17 @@ def __init__(self): def create( self, ctxt, - name, - endpoint_id, - pool_platform, - pool_os_type, - environment_options, - minimum_minions, - maximum_minions, - minion_max_idle_time, - minion_retention_strategy, + name: str, + endpoint_id: str, + pool_platform: str, + pool_os_type: str, + environment_options: dict[Any, Any], + minimum_minions: int, + maximum_minions: int, + minion_max_idle_time: int, + minion_retention_strategy: str, notes=None, - skip_allocation=False, + skip_allocation: bool = False, ): return self._rpc_client.create_minion_pool( ctxt, @@ -38,27 +40,37 @@ def create( skip_allocation=skip_allocation, ) - def update(self, ctxt, minion_pool_id, updated_values): + def update( + self, + ctxt, + minion_pool_id: str, + updated_values: dict, + ): return self._rpc_client.update_minion_pool( ctxt, minion_pool_id, updated_values=updated_values ) - def delete(self, ctxt, minion_pool_id): + def delete(self, ctxt, minion_pool_id: str) -> None: self._rpc_client.delete_minion_pool(ctxt, minion_pool_id) - def get_minion_pools(self, ctxt): + def get_minion_pools(self, ctxt) -> list: return self._rpc_client.get_minion_pools(ctxt) - def get_minion_pool(self, ctxt, minion_pool_id): + def get_minion_pool(self, ctxt, minion_pool_id: str): return self._rpc_client.get_minion_pool(ctxt, minion_pool_id) - def allocate_minion_pool(self, ctxt, minion_pool_id): + def allocate_minion_pool(self, ctxt, minion_pool_id: str): return self._rpc_client.allocate_minion_pool(ctxt, minion_pool_id) - def refresh_minion_pool(self, ctxt, minion_pool_id): + def refresh_minion_pool(self, ctxt, minion_pool_id: str): return self._rpc_client.refresh_minion_pool(ctxt, minion_pool_id) - def deallocate_minion_pool(self, ctxt, minion_pool_id, force=False): + def deallocate_minion_pool( + self, + ctxt, + minion_pool_id: str, + force: bool = False, + ): return self._rpc_client.deallocate_minion_pool( ctxt, minion_pool_id, force=force ) diff --git a/coriolis/osmorphing/base.py b/coriolis/osmorphing/base.py index 4b55915c8..d13bcbf47 100644 --- a/coriolis/osmorphing/base.py +++ b/coriolis/osmorphing/base.py @@ -7,6 +7,7 @@ import re import shlex import uuid +from typing import Any import yaml from oslo_log import log as logging @@ -135,14 +136,14 @@ class BaseOSMorphingTools(object, with_metaclass(abc.ABCMeta)): def __init__( self, conn, - os_root_dir, - os_root_device, + os_root_dir: str, + os_root_device: str, hypervisor, event_manager, - detected_os_info, - osmorphing_parameters, + detected_os_info: dict[str, str], + osmorphing_parameters: dict[Any, Any], operation_timeout, - nics_set_dhcp=True, + nics_set_dhcp: bool = True, ): self.check_detected_os_info_parameters(detected_os_info) @@ -166,7 +167,9 @@ def get_required_detected_os_info_fields(cls): raise NotImplementedError("Required OS params not defined.") @classmethod - def check_detected_os_info_parameters(cls, detected_os_info): + def check_detected_os_info_parameters( + cls, detected_os_info: dict[str, str] + ) -> bool: required_fields = cls.get_required_detected_os_info_fields() missing_os_info_fields = [ field for field in required_fields if field not in detected_os_info @@ -197,19 +200,19 @@ def check_os_supported(cls, detected_os_info): ) @abc.abstractmethod - def get_installed_packages(self): + def get_installed_packages(self) -> None: pass @abc.abstractmethod - def set_net_config(self, nics_info, dhcp): + def set_net_config(self, nics_info: list | None, dhcp: bool) -> None: pass @abc.abstractmethod - def get_packages(self): + def get_packages(self) -> tuple[list, list]: return [], [] @abc.abstractmethod - def run_user_script(self, user_script): + def run_user_script(self, user_script: str) -> None: pass @abc.abstractmethod @@ -219,7 +222,7 @@ def register_firstboot_script( index: int = 0, user_provided: bool = True, script_filename: str | None = None, - ): + ) -> None: """Register a script to be executed during the first replica boot. :param script: the script content @@ -237,15 +240,15 @@ def register_firstboot_script( pass @abc.abstractmethod - def pre_packages_install(self, package_names): + def pre_packages_install(self, package_names: list[str]) -> None: pass @abc.abstractmethod - def install_packages(self, package_names): + def install_packages(self, package_names: list[str]) -> None: pass @abc.abstractmethod - def post_packages_install(self, package_names): + def post_packages_install(self, package_names: list[str]) -> None: pass @abc.abstractmethod @@ -260,7 +263,7 @@ def uninstall_packages(self, package_names): def post_packages_uninstall(self, package_names): pass - def set_environment(self, environment): + def set_environment(self, environment: dict[str, str]) -> None: """Merges the given environment into the tools' own environment. The variables declared by the tools' constructor (such as @@ -289,14 +292,14 @@ class BaseLinuxOSMorphingTools(BaseOSMorphingTools): def __init__( self, conn, - os_root_dir, - os_root_dev, + os_root_dir: str, + os_root_dev: str, hypervisor, event_manager, - detected_os_info, - osmorphing_parameters, + detected_os_info: dict[str, str], + osmorphing_parameters: dict[Any, Any], operation_timeout=None, - nics_set_dhcp=True, + nics_set_dhcp: bool = True, ): super(BaseLinuxOSMorphingTools, self).__init__( conn, @@ -317,11 +320,13 @@ def datasource_list(self): return [] @classmethod - def get_required_detected_os_info_fields(cls): + def get_required_detected_os_info_fields(cls) -> list[str]: return REQUIRED_DETECTED_OS_FIELDS @classmethod - def _version_supported_util(cls, version, minimum, maximum=None): + def _version_supported_util( + cls, version: str, minimum: float, maximum=None + ) -> bool: """Parses version strings which are prefixed with a floating point and checks whether the value is between the provided minimum and maximum (excluding the maximum). @@ -395,7 +400,7 @@ def _parse_version_util(cls, version: str) -> Version: f"Could not parse version from release string: {version!r}" ) from exc - def get_packages(self): + def get_packages(self) -> tuple[list, list]: k_add = [h for h in self._packages.keys() if h is None or h == self._hypervisor] add = [ @@ -417,7 +422,7 @@ def get_packages(self): return add, remove - def run_user_script(self, user_script): + def run_user_script(self, user_script: str) -> None: if len(user_script) == 0: return @@ -442,10 +447,10 @@ def run_user_script(self, user_script): except Exception as err: raise exception.CoriolisException("Failed to run user script.") from err - def pre_packages_install(self, package_names): + def pre_packages_install(self, package_names: list[str]) -> None: self._copy_resolv_conf() - def post_packages_install(self, package_names): + def post_packages_install(self, package_names: list[str]) -> None: self._run_scheduled_grub2_update() self._restore_resolv_conf() @@ -458,7 +463,7 @@ def post_packages_uninstall(self, package_names): def get_update_grub2_command(self): raise NotImplementedError() - def _test_path(self, chroot_path): + def _test_path(self, chroot_path: str) -> bool: chroot_path = chroot_path.lstrip('/') path = os.path.join(self._os_root_dir, chroot_path) return utils.test_ssh_path(self._ssh, path) @@ -468,7 +473,7 @@ def _read_file(self, chroot_path): path = os.path.join(self._os_root_dir, chroot_path) return utils.read_ssh_file(self._ssh, path) - def _write_file(self, chroot_path, content): + def _write_file(self, chroot_path: str, content: str) -> None: chroot_path = chroot_path.lstrip('/') path = os.path.join(self._os_root_dir, chroot_path) utils.write_ssh_file(self._ssh, path, content) @@ -478,7 +483,7 @@ def _list_dir(self, chroot_path): path = os.path.join(self._os_root_dir, chroot_path) return utils.list_ssh_dir(self._ssh, path) - def _exec_cmd(self, cmd, timeout=None): + def _exec_cmd(self, cmd: str, timeout=None) -> str: if not timeout: timeout = self._osmorphing_operation_timeout try: @@ -494,7 +499,7 @@ def _exec_cmd(self, cmd, timeout=None): cmd=cmd, timeout=timeout ) from ex - def _exec_cmd_chroot(self, cmd, timeout=None): + def _exec_cmd_chroot(self, cmd: str, timeout=None) -> str: if not timeout: timeout = self._osmorphing_operation_timeout try: @@ -518,7 +523,7 @@ def _check_user_exists(self, username): except Exception: return False - def _write_file_sudo(self, chroot_path, content): + def _write_file_sudo(self, chroot_path: str, content: str) -> None: # NOTE: writes the file to a temp location due to permission issues tmp_file = 'tmp/%s' % str(uuid.uuid4()) self._write_file(tmp_file, content) @@ -670,7 +675,7 @@ def _write_nmconnection_configs(self, nics_info, nmconnection_files=None): ) self._exec_cmd_chroot("chmod 600 /%s" % cfg_path) - def _copy_resolv_conf(self): + def _copy_resolv_conf(self) -> None: resolv_conf = "etc/resolv.conf" resolv_conf_path = os.path.join(self._os_root_dir, resolv_conf) resolv_conf_path_old = "%s.old" % resolv_conf_path @@ -683,7 +688,7 @@ def _copy_resolv_conf(self): 'sudo cp -L --remove-destination /etc/resolv.conf %s' % resolv_conf_path ) - def _restore_resolv_conf(self): + def _restore_resolv_conf(self) -> None: resolv_conf = "etc/resolv.conf" resolv_conf_old = "%s.old" % resolv_conf resolv_conf_path = os.path.join(self._os_root_dir, resolv_conf) @@ -722,7 +727,7 @@ def _replace_fstab_entries_device_prefix( ) self._write_file(fstab_chroot_path, "\n".join(fstab_contents_lines)) - def _set_selinux_autorelabel(self): + def _set_selinux_autorelabel(self) -> None: LOG.debug("setting autorelabel on /") try: self._exec_cmd_chroot("touch /.autorelabel") @@ -796,7 +801,7 @@ def _create_cloudinit_user(self): def _has_systemd_chroot(self): return self._test_path("/lib/systemd/system") - def _configure_cloud_init(self): + def _configure_cloud_init(self) -> None: cloud_cfg_mods = {} if "cloud-init" not in self.get_packages()[0]: return @@ -843,7 +848,7 @@ def _configure_cloud_init(self): # unit service name renamed. self._enable_systemd_service(CLOUD_INIT_SERVICE_UNIT_NAME_FALLBACK) - def _test_path_chroot(self, path): + def _test_path_chroot(self, path: str) -> bool: # This method uses _exec_cmd_chroot() instead of SFTP stat() # because in some situations, the SSH user used may not have # execute rights on one or more of the folders that lead up @@ -933,7 +938,7 @@ def replace_in_cfg(opt, val): LOG.warning("TEMP CONFIG IS: %r" % cfg) @staticmethod - def _normalize_kernel_cmdline_args(args): + def _normalize_kernel_cmdline_args(args: list | None) -> list[str]: """Returns the given kernel command line arguments as a list.""" if isinstance(args, str): @@ -949,7 +954,9 @@ def _get_kernel_cmdline_arg_value(arg): return {"opt_type": "single", "opt_val": key} return {"opt_type": "key_val", "opt_key": key, "opt_val": value} - def _update_kernel_cmdline_args(self, args_to_add=None, args_to_remove=None): + def _update_kernel_cmdline_args( + self, args_to_add: list | None = None, args_to_remove: list | None = None + ) -> bool: """Updates the kernel command line arguments in the GRUB2 defaults. The arguments are edited in '/etc/default/grub' and the GRUB2 config @@ -983,7 +990,7 @@ def _update_kernel_cmdline_args(self, args_to_add=None, args_to_remove=None): return True - def _get_grub_default_conf(self): + def _get_grub_default_conf(self) -> None: grub_conf = "/etc/default/grub" if self._test_path_chroot(grub_conf): return grub_conf @@ -1029,7 +1036,7 @@ def _schedule_grub2_update(self): """ self._grub2_update_scheduled = True - def _run_scheduled_grub2_update(self): + def _run_scheduled_grub2_update(self) -> None: """Runs the deferred GRUB2 config regeneration, if one was scheduled. The pending flag is cleared so that a subsequent call is a no-op @@ -1177,7 +1184,7 @@ def register_firstboot_script( index: int = 0, user_provided: bool = True, script_filename: str | None = None, - ): + ) -> None: if not script: LOG.debug("Empty first-boot script, skipping...") diff --git a/coriolis/osmorphing/centos.py b/coriolis/osmorphing/centos.py index 7b777bb55..d8579f245 100644 --- a/coriolis/osmorphing/centos.py +++ b/coriolis/osmorphing/centos.py @@ -1,6 +1,8 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from oslo_log import log as logging from coriolis import exception, utils @@ -31,7 +33,7 @@ def check_os_supported(cls, detected_os_info): detected_os_info['release_version'], minimum=7 ) - def enable_repos(self, repo_names): + def enable_repos(self, repo_names: list[Any]): """Enable repositories for CentOS. Uses yum-config-manager for CentOS 7 and earlier, diff --git a/coriolis/osmorphing/debian.py b/coriolis/osmorphing/debian.py index 864d84fb4..035a13e61 100644 --- a/coriolis/osmorphing/debian.py +++ b/coriolis/osmorphing/debian.py @@ -4,6 +4,7 @@ import os import re from io import StringIO +from typing import Any import yaml from oslo_log import log as logging @@ -33,14 +34,14 @@ class BaseDebianMorphingTools(base.BaseLinuxOSMorphingTools): def __init__( self, conn, - os_root_dir, - os_root_dev, + os_root_dir: str, + os_root_dev: str, hypervisor, event_manager, - detected_os_info, - osmorphing_parameters, + detected_os_info: dict[str, str], + osmorphing_parameters: dict[Any, Any], operation_timeout=None, - nics_set_dhcp=True, + nics_set_dhcp: bool = True, ): super(BaseDebianMorphingTools, self).__init__( conn, @@ -70,7 +71,7 @@ def check_os_supported(cls, detected_os_info): detected_os_info['release_version'], minimum=8 ) - def disable_predictable_nic_names(self): + def disable_predictable_nic_names(self) -> None: self._update_kernel_cmdline_args(args_to_add=["net.ifnames=0", "biosdevname=0"]) def get_update_grub2_command(self): @@ -138,7 +139,7 @@ def _install_uefi_fallback_bootloader(self): ) self._schedule_grub2_update() - def set_net_config(self, nics_info, dhcp): + def set_net_config(self, nics_info: list | None, dhcp: bool) -> None: if not dhcp: LOG.info("Setting static IP configuration") self._setup_network_preservation(nics_info) @@ -162,7 +163,7 @@ def set_net_config(self, nics_info, dhcp): cfg_name = "%s/coriolis_netplan.yaml" % self.netplan_base self._write_file_sudo(cfg_name, new_cfg) - def _run_update_initramfs(self): + def _run_update_initramfs(self) -> None: # env LC_ALL=C suppresses Perl/shell locale warnings that would # otherwise appear in stdout and get mixed into the version list. # Using 'env' rather than a shell assignment prefix because @@ -191,14 +192,14 @@ def _run_update_initramfs(self): for version in kernel_versions: self._exec_cmd_chroot(f"update-initramfs -k {version} -u") - def get_installed_packages(self): + def get_installed_packages(self) -> None: cmd = "dpkg-query -f '${binary:Package}\\n' -W" try: self.installed_packages = self._exec_cmd_chroot(cmd).splitlines() except exception.CoriolisException: self.installed_packages = [] - def pre_packages_install(self, package_names): + def pre_packages_install(self, package_names: list[str]) -> None: super(BaseDebianMorphingTools, self).pre_packages_install(package_names) try: if package_names: @@ -223,7 +224,7 @@ def pre_packages_install(self, package_names): "error was: %s" % str(err) ) from err - def post_packages_install(self, package_names): + def post_packages_install(self, package_names: list[str]) -> None: self._configure_cloud_init() self._run_update_initramfs() if ( @@ -233,7 +234,7 @@ def post_packages_install(self, package_names): self._install_uefi_fallback_bootloader() super(BaseDebianMorphingTools, self).post_packages_install(package_names) - def install_packages(self, package_names): + def install_packages(self, package_names: list[str]) -> None: try: # NOTE(aznashwan): in the rare event that a replica sync occurs # while the source machine was midway through installing or diff --git a/coriolis/osmorphing/manager.py b/coriolis/osmorphing/manager.py index 211fae3ae..40535fdba 100644 --- a/coriolis/osmorphing/manager.py +++ b/coriolis/osmorphing/manager.py @@ -2,6 +2,7 @@ # All Rights Reserved. import itertools +from typing import Any from oslo_config import cfg from oslo_log import log as logging @@ -34,7 +35,7 @@ LOG = logging.getLogger(__name__) -def _get_proxy_settings(): +def _get_proxy_settings() -> dict: return { "url": CONF.proxy.url, "username": CONF.proxy.username, @@ -47,11 +48,11 @@ def run_os_detect( origin_provider, destination_provider, worker_connection, - os_type, - os_root_dir, - osmorphing_info, - tools_environment={}, -): + os_type: str, + os_root_dir: str, + osmorphing_info: dict, + tools_environment: dict[str, str] = {}, +) -> dict[str, str]: custom_export_os_detect_tools = origin_provider.get_custom_os_detect_tools( os_type, osmorphing_info ) @@ -80,8 +81,8 @@ def run_os_detect( def get_osmorphing_tools_class_for_provider( - provider, detected_os_info, os_type, osmorphing_info -): + provider, detected_os_info: dict[str, str], os_type: str, osmorphing_info: dict +) -> type[Any] | None: available_tools_cls = provider.get_os_morphing_tools(os_type, osmorphing_info) LOG.debug( "OSMorphing tools classes returned by provider '%s' for os_type '%s' " @@ -144,11 +145,11 @@ def get_osmorphing_tools_class_for_provider( def morph_image( origin_provider, destination_provider, - connection_info, - osmorphing_info, - user_scripts, + connection_info: dict, + osmorphing_info: dict, + user_scripts: list | None, event_handler, -): +) -> None: event_manager = events.EventManager(event_handler) os_type = osmorphing_info.get('os_type') @@ -229,12 +230,12 @@ def morph_image( def _morph_image( origin_provider, destination_provider, - connection_info, - osmorphing_info, - user_scripts, + connection_info: dict, + osmorphing_info: dict, + user_scripts: list[dict], event_handler, os_mount_tools, -): +) -> None: event_manager = events.EventManager(event_handler) os_type = osmorphing_info.get('os_type') diff --git a/coriolis/osmorphing/osdetect/base.py b/coriolis/osmorphing/osdetect/base.py index 619be2b22..0035f4ff0 100644 --- a/coriolis/osmorphing/osdetect/base.py +++ b/coriolis/osmorphing/osdetect/base.py @@ -23,7 +23,7 @@ class BaseOSDetectTools(object, with_metaclass(abc.ABCMeta)): - def __init__(self, conn, os_root_dir, operation_timeout): + def __init__(self, conn, os_root_dir: str, operation_timeout): self._conn = conn self._os_root_dir = os_root_dir self._environment = {} @@ -37,7 +37,7 @@ def returned_detected_os_info_fields(cls): ) @abc.abstractmethod - def detect_os(self): + def detect_os(self) -> dict[str, str]: """Attempts to detect the mounted OS and return all relevant release info as a dict. @@ -47,7 +47,7 @@ def detect_os(self): "`detect_os` not implemented for OS detection tools '%s'" % (type(self)) ) - def set_environment(self, environment): + def set_environment(self, environment: dict[str, str]) -> None: self._environment = environment # @abc.abstractmethod @@ -61,23 +61,25 @@ def set_environment(self, environment): class BaseLinuxOSDetectTools(BaseOSDetectTools): @classmethod - def returned_detected_os_info_fields(cls): + def returned_detected_os_info_fields(cls) -> list[str]: return REQUIRED_DETECTED_OS_FIELDS def _read_file(self, chroot_path): path = os.path.join(self._os_root_dir, chroot_path) return utils.read_ssh_file(self._conn, path) - def _read_config_file(self, chroot_path, check_exists=False): + def _read_config_file( + self, chroot_path: str, check_exists: bool = False + ) -> dict[str, str]: full_path = os.path.join(self._os_root_dir, chroot_path) return utils.read_ssh_ini_config_file( self._conn, full_path, check_exists=check_exists ) - def _get_os_release(self): + def _get_os_release(self) -> dict[str, str]: return self._read_config_file("etc/os-release", check_exists=True) - def _test_path(self, chroot_path): + def _test_path(self, chroot_path: str) -> bool: full_path = os.path.join(self._os_root_dir, chroot_path) return utils.test_ssh_path(self._conn, full_path) @@ -118,7 +120,7 @@ def _exec_cmd_chroot(self, cmd, timeout=None): class LinuxOSDetectUsingOSRelease(BaseLinuxOSDetectTools): """OS detection based on the standard /etc/os-release file.""" - def detect_os(self): + def detect_os(self) -> dict[str, str]: """Detect a Linux distro from /etc/os-release.""" os_release = self._get_os_release() if not os_release: diff --git a/coriolis/osmorphing/osdetect/coreos.py b/coriolis/osmorphing/osdetect/coreos.py index 95ac3b34d..7cf371b72 100644 --- a/coriolis/osmorphing/osdetect/coreos.py +++ b/coriolis/osmorphing/osdetect/coreos.py @@ -1,6 +1,8 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from coriolis import constants from coriolis.osmorphing.osdetect import base @@ -8,7 +10,7 @@ class CoreOSOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): + def detect_os(self) -> dict[Any, Any]: info = {} os_release = self._get_os_release() osid = os_release.get("ID") diff --git a/coriolis/osmorphing/osdetect/debian.py b/coriolis/osmorphing/osdetect/debian.py index f9f97c181..c73292deb 100644 --- a/coriolis/osmorphing/osdetect/debian.py +++ b/coriolis/osmorphing/osdetect/debian.py @@ -1,6 +1,8 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from coriolis import constants from coriolis.osmorphing.osdetect import base @@ -8,7 +10,7 @@ class DebianOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): + def detect_os(self) -> dict[Any, Any]: release = None base_info = { "os_type": constants.OS_TYPE_LINUX, diff --git a/coriolis/osmorphing/osdetect/manager.py b/coriolis/osmorphing/osdetect/manager.py index d0e10f962..a24664757 100644 --- a/coriolis/osmorphing/osdetect/manager.py +++ b/coriolis/osmorphing/osdetect/manager.py @@ -37,12 +37,12 @@ def _check_custom_os_detect_tools(custom_os_detect_tools): def detect_os( conn, - os_type, - os_root_dir, + os_type: str, + os_root_dir: str, operation_timeout, - tools_environment=None, - custom_os_detect_tools=None, -): + tools_environment: dict | None = None, + custom_os_detect_tools: list | None = None, +) -> dict[str, str]: """Iterates through all of the OS detection tools until one successfully identifies the OS/release and returns the release info from it. diff --git a/coriolis/osmorphing/osdetect/openwrt.py b/coriolis/osmorphing/osdetect/openwrt.py index 89182cb99..f5df1db8a 100644 --- a/coriolis/osmorphing/osdetect/openwrt.py +++ b/coriolis/osmorphing/osdetect/openwrt.py @@ -1,6 +1,8 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from coriolis import constants from coriolis.osmorphing.osdetect import base @@ -8,7 +10,7 @@ class OpenWRTOSDetectTools(base.BaseLinuxOSDetectTools): - def detect_os(self): + def detect_os(self) -> dict[Any, Any]: info = {} openwrt_release = self._read_config_file( "etc/openwrt_release", check_exists=True diff --git a/coriolis/osmorphing/osdetect/suse.py b/coriolis/osmorphing/osdetect/suse.py index ba5b6b778..9df3270df 100644 --- a/coriolis/osmorphing/osdetect/suse.py +++ b/coriolis/osmorphing/osdetect/suse.py @@ -2,6 +2,7 @@ # All Rights Reserved. import copy +from typing import Any from coriolis import constants from coriolis.osmorphing.osdetect import base @@ -20,7 +21,7 @@ def returned_detected_os_info_fields(cls): fields.append(DETECTED_SUSE_RELEASE_FIELD_NAME) return fields - def detect_os(self): + def detect_os(self) -> dict[Any, Any]: # TODO(aznashwan): implement separate detection for # SLES and openSUSE when the tools for them are separated. info = {} diff --git a/coriolis/osmorphing/osmount/base.py b/coriolis/osmorphing/osmount/base.py index b09545895..356cb282d 100644 --- a/coriolis/osmorphing/osmount/base.py +++ b/coriolis/osmorphing/osmount/base.py @@ -9,6 +9,7 @@ import os import re import uuid +from typing import Any from oslo_log import log as logging from six import with_metaclass @@ -27,11 +28,11 @@ class BaseOSMountTools(object, with_metaclass(abc.ABCMeta)): def __init__( self, - connection_info, + connection_info: dict, event_manager, - ignore_devices, + ignore_devices: list[str], operation_timeout, - osmorphing_info=None, + osmorphing_info: dict | None = None, ): self._event_manager = event_manager self._ignore_devices = ignore_devices @@ -42,7 +43,7 @@ def __init__( self._connect() @abc.abstractmethod - def _connect(self): + def _connect(self) -> None: pass @abc.abstractmethod @@ -50,41 +51,41 @@ def get_connection(self): pass @abc.abstractmethod - def check_os(self): + def check_os(self) -> bool: pass def setup(self): pass @abc.abstractmethod - def mount_os(self): + def mount_os(self) -> tuple[str, str]: pass @abc.abstractmethod - def dismount_os(self, root_dir): + def dismount_os(self, root_dir: str) -> None: pass - def set_proxy(self, proxy_settings): + def set_proxy(self, proxy_settings: dict): pass - def remove_encryption_artifacts(self, os_root_dir): + def remove_encryption_artifacts(self, os_root_dir: str): pass - def install_encryption_firstboot_setup(self, os_root_dir, os_morphing_tools): + def install_encryption_firstboot_setup(self, os_root_dir: str, os_morphing_tools): pass - def get_environment(self): + def get_environment(self) -> dict[str, str]: return self._environment @abc.abstractmethod - def run_user_script(self, user_script): + def run_user_script(self, user_script: str) -> None: """Run pre-os-mount user script.""" pass class BaseSSHOSMountTools(BaseOSMountTools): @utils.retry_on_error(max_attempts=5, sleep_seconds=3) - def _connect(self): + def _connect(self) -> None: connection_info = self._connection_info ip = connection_info["ip"] @@ -107,13 +108,13 @@ def _connect(self): ssh.set_log_channel("paramiko.morpher.%s.%s" % (ip, port)) self._ssh = ssh - def setup(self): + def setup(self) -> None: utils.check_env_command(self._ssh) if self._allow_ssh_env_vars(): self._ssh.close() self._connect() - def _allow_ssh_env_vars(self): + def _allow_ssh_env_vars(self) -> bool: self._exec_cmd('sudo sed -i -e "\$aAcceptEnv *" /etc/ssh/sshd_config') try: utils.restart_service(self._ssh, "sshd") @@ -125,7 +126,7 @@ def _allow_ssh_env_vars(self): utils.restart_service(self._ssh, 'ssh') return True - def _exec_cmd(self, cmd, timeout=None): + def _exec_cmd(self, cmd: str, timeout=None) -> str: if not timeout: timeout = self._osmount_operation_timeout try: @@ -137,7 +138,7 @@ def _exec_cmd(self, cmd, timeout=None): cmd=cmd, timeout=timeout ) from ex - def _exec_sudo_env_cmd(self, cmd, timeout=None): + def _exec_sudo_env_cmd(self, cmd: str, timeout=None) -> str: """ Runs a sudo command that also passes all the environment variables to the underlying command. Replaces sudo's -E flag, which is not currently @@ -186,7 +187,7 @@ def _get_pvs(self): LOG.debug("Physical Volume attributes: %s", pvs) return pvs - def _get_vgs(self): + def _get_vgs(self) -> dict[Any, Any]: """Returns a dict of the form: { "VG UUID": { "name": "", @@ -228,7 +229,7 @@ def _get_vgs(self): return vgs_uuid_map - def _check_vgs(self): + def _check_vgs(self) -> None: try: self._exec_cmd("sudo vgck") except Exception as ex: @@ -254,7 +255,7 @@ def _get_vgnames(self): LOG.debug("Volume group names: %s", vg_names) return vg_names - def _get_lv_paths(self): + def _get_lv_paths(self) -> list[Any]: """Returns list with paths of available LVM volumes.""" lvm_paths = [] out = self._exec_cmd("sudo lvdisplay -c").strip() @@ -271,11 +272,11 @@ def _get_lv_paths(self): def _check_mount_fstab_partitions( self, - os_root_dir, - skip_mounts=["/", "/boot"], - skip_filesystems=["swap"], - mountable_lvm_devs=None, - ): + os_root_dir: str, + skip_mounts: list[str] = ["/", "/boot"], + skip_filesystems: list[str] = ["swap"], + mountable_lvm_devs: list | None = None, + ) -> list[Any]: """Reads the contents of /etc/fstab from the VM's root directory and tries to mount all clearly identified (by UUID, label, or path) filesystems. Returns the list of the new directories which were @@ -415,7 +416,7 @@ def _check_mount_fstab_partitions( return new_mountpoints - def _get_symlink_target(self, symlink): + def _get_symlink_target(self, symlink: str) -> str: target = symlink try: target = self._exec_cmd('readlink -en %s' % symlink) @@ -428,7 +429,7 @@ def _get_symlink_target(self, symlink): return target - def _get_device_file_paths(self, symlink_list): + def _get_device_file_paths(self, symlink_list: list[Any]) -> list[Any]: """Reads a list of symlink paths, such as `/dev/GROUP/VOLUME0` or `/dev/mapper/GROUP-VOLUME0` and returns a list of respective links' target device file paths, such as `/dev/dm0` @@ -441,7 +442,7 @@ def _get_device_file_paths(self, symlink_list): dev_file_paths.append(dev_file) return dev_file_paths - def _get_mounted_devices(self): + def _get_mounted_devices(self) -> list[Any]: mounts = self._exec_cmd("cat /proc/mounts").splitlines() ret = [] mounted_device_numbers = [] @@ -482,7 +483,7 @@ def _get_mount_destinations(self): ret.add(colls[1]) return ret - def _get_volume_block_devices(self): + def _get_volume_block_devices(self) -> list[str]: # NOTE: depending on the version of the worker OS, scanning for just # the device NAME may lead to LVM volumes getting displayed as: # "- (dm-N)", @@ -515,7 +516,12 @@ def _get_volume_block_devices(self): LOG.info("Volume block devices: %s", volume_devs) return volume_devs - def _find_dev_with_contents(self, devices, all_files=None, one_of_files=None): + def _find_dev_with_contents( + self, + devices: list[str], + all_files: list | None = None, + one_of_files: list | None = None, + ) -> str | None: if all_files and one_of_files: raise exception.CoriolisException( "all_files and one_of_files are mutually exclusive" @@ -563,7 +569,7 @@ def _find_dev_with_contents(self, devices, all_files=None, one_of_files=None): return dev_name - def _mask_minion_efi_firmware(self, os_root_dir): + def _mask_minion_efi_firmware(self, os_root_dir: str) -> None: firmware_type = self._osmorphing_info.get("firmware_type") if firmware_type is None: firmware_type = self._osmorphing_info.get("osmorphing_parameters", {}).get( @@ -584,7 +590,7 @@ def _mask_minion_efi_firmware(self, os_root_dir): "sudo mount -t tmpfs tmpfs %s" % os.path.join(os_root_dir, "sys/firmware") ) - def _find_and_mount_root(self, devices): + def _find_and_mount_root(self, devices: list[str]) -> tuple[str, str]: files = ["etc", "bin", "sbin", "boot"] os_root_dir = None os_root_device = self._find_dev_with_contents(devices, all_files=files) @@ -654,7 +660,7 @@ def _find_and_mount_root(self, devices): return os_root_dir, os_root_device - def mount_os(self): + def mount_os(self) -> tuple[str, str]: # Prevent Coriolis mounts from being propagated to other namespaces # belonging to Systemd services. These would be leaked when running # "unmount -R" in this namespace, being especially troublesome if the @@ -753,7 +759,7 @@ def mount_os(self): return os_root_dir, os_root_device - def dismount_os(self, root_dir): + def dismount_os(self, root_dir: str) -> None: # Flush dirty pages to replica volumes before unmount. # Later volume detach would otherwise drop in-memory writes. self._exec_cmd("sudo sync") @@ -772,7 +778,7 @@ def dismount_os(self, root_dir): self._close_luks_devices() - def set_proxy(self, proxy_settings): + def set_proxy(self, proxy_settings: dict) -> None: url = proxy_settings.get('url') if not url: return @@ -793,7 +799,7 @@ def set_proxy(self, proxy_settings): LOG.debug("Proxy exclusions: %s", no_proxy) self._environment["no_proxy"] = '.'.join(no_proxy) - def run_user_script(self, user_script): + def run_user_script(self, user_script: str) -> None: if len(user_script) == 0: return diff --git a/coriolis/osmorphing/osmount/factory.py b/coriolis/osmorphing/osmount/factory.py index 9c51d84c2..eaee33c37 100644 --- a/coriolis/osmorphing/osmount/factory.py +++ b/coriolis/osmorphing/osmount/factory.py @@ -12,13 +12,13 @@ def get_os_mount_tools( - os_type, - connection_info, + os_type: str, + connection_info: dict, event_manager, - ignore_devices, + ignore_devices: list[str], operation_timeout, - osmorphing_info=None, -): + osmorphing_info: dict | None = None, +) -> ubuntu.UbuntuOSMountTools: os_mount_tools = { constants.OS_TYPE_LINUX: [ ubuntu.UbuntuOSMountTools, diff --git a/coriolis/osmorphing/osmount/luks_mixin.py b/coriolis/osmorphing/osmount/luks_mixin.py index 2f182a6aa..581a18977 100644 --- a/coriolis/osmorphing/osmount/luks_mixin.py +++ b/coriolis/osmorphing/osmount/luks_mixin.py @@ -5,6 +5,7 @@ import json import os import re +from typing import Callable, Iterator from oslo_config import cfg, types from oslo_log import log as logging @@ -45,7 +46,7 @@ _RESOURCES_DIR = os.path.join(os.path.dirname(__file__), "resources") -def _load_script(filename): +def _load_script(filename: str) -> str: with open(os.path.join(_RESOURCES_DIR, filename)) as fh: return fh.read() @@ -69,7 +70,7 @@ def validate_config(): ) -def _render_tpm2_pcrs(script_content, initramfs_tool): +def _render_tpm2_pcrs(script_content: str, initramfs_tool: str) -> str: """Substitute the configured TPM2 PCR list into the given script_content. The list is formatted based on the given initramfs_tool's enrollment @@ -98,7 +99,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._luks_opened = [] - def _unlock_luks_devices(self, dev_paths): + def _unlock_luks_devices(self, dev_paths: list[str]) -> None: """Open any LUKS-encrypted devices listed in dev_paths in-place. Reads osmorphing_info["encrypted_disks_passphrase"], a single @@ -125,7 +126,7 @@ def _unlock_luks_devices(self, dev_paths): self._close_luks_devices() raise - def _unlock_luks_device(self, dev_path, passphrase): + def _unlock_luks_device(self, dev_path: str, passphrase: str | None) -> str | None: """Open a single LUKS-encrypted device. :raises CoriolisException: if the device is LUKS-encrypted but no @@ -156,7 +157,7 @@ def _unlock_luks_device(self, dev_path, passphrase): return mapper_path - def _is_luks(self, dev_path): + def _is_luks(self, dev_path: str) -> bool: try: self._exec_cmd("sudo cryptsetup isLuks %s" % dev_path) return True @@ -169,14 +170,16 @@ def _is_luks(self, dev_path): return False raise - def _close_luks_devices(self): + def _close_luks_devices(self) -> None: """Close any LUKS mapper devices opened by _unlock_luks_devices.""" for mapper_name, _ in self._luks_opened: self._exec_cmd("sudo cryptsetup luksClose %s || true" % mapper_name) self._luks_opened = [] - def _write_remote_file(self, dest_path, content, mode=None): + def _write_remote_file( + self, dest_path: str, content: str, mode: str | None = None + ) -> None: """Write content to dest_path on the remote host via a temp file.""" tmp = self._exec_cmd("mktemp").strip() utils.write_ssh_file(self._ssh, tmp, content.encode("utf-8")) @@ -188,7 +191,7 @@ def _write_remote_file(self, dest_path, content, mode=None): self._exec_cmd("sudo mv %s %s" % (tmp, dest_path)) @contextlib.contextmanager - def _auth_luks(self, passphrase, key_path): + def _auth_luks(self, passphrase: str, key_path: str) -> Iterator[str]: """Write passphrase to key_path on the remote host; yield; delete.""" self._write_remote_file(key_path, passphrase) try: @@ -196,7 +199,7 @@ def _auth_luks(self, passphrase, key_path): finally: self._exec_cmd("sudo rm -f %s" % key_path) - def _get_tpm2_token_info(self, dev_path): + def _get_tpm2_token_info(self, dev_path: str) -> list[tuple]: """Return list of (token_id, keyslot_id) pairs for systemd-tpm2 tokens. Returns an empty list for LUKS1 devices, which do not support tokens. @@ -224,7 +227,7 @@ def _get_tpm2_token_info(self, dev_path): return results - def _remove_tpm2_tokens(self, dev_path, passphrase): + def _remove_tpm2_tokens(self, dev_path: str, passphrase: str) -> None: """Remove systemd-tpm2 tokens and kill their keyslots from dev_path. Token removal modifies only the LUKS2 header metadata. Keyslot @@ -266,7 +269,9 @@ def _remove_tpm2_tokens(self, dev_path, passphrase): utils.get_exception_details(), ) - def _transform_crypttab(self, os_root_dir, transform): + def _transform_crypttab( + self, os_root_dir: str, transform: Callable[[list], list] + ) -> bool: """Apply transform to each non-comment entry in /etc/crypttab. transform(parts) receives the split fields [name, device, keyfile, @@ -304,14 +309,14 @@ def _transform_crypttab(self, os_root_dir, transform): self._write_remote_file(crypttab_path, "".join(new_lines)) return True - def _remove_tpm2_crypttab_options(self, os_root_dir): + def _remove_tpm2_crypttab_options(self, os_root_dir: str) -> None: """Strip tpm2-* options from /etc/crypttab in the mounted OS. Prevents the initramfs from attempting TPM2 unsealing on the target, which would fail because the source TPM is not present there. """ - def _strip_tpm2(parts): + def _strip_tpm2(parts: list[str]) -> list[str]: if len(parts) < 4: return None @@ -325,7 +330,7 @@ def _strip_tpm2(parts): if self._transform_crypttab(os_root_dir, _strip_tpm2): LOG.info("Removed TPM2 options from /etc/crypttab in '%s'", os_root_dir) - def remove_encryption_artifacts(self, os_root_dir): + def remove_encryption_artifacts(self, os_root_dir: str) -> None: """Remove stale TPM2 tokens, kill their keyslots, and strip tpm2-* options from /etc/crypttab. @@ -345,13 +350,15 @@ def remove_encryption_artifacts(self, os_root_dir): self._remove_tpm2_crypttab_options(os_root_dir) - def _get_migration_keyfile_path(self, dev_path): + def _get_migration_keyfile_path(self, dev_path: str) -> str: return "%s/coriolis_%s.key" % (_LUKS_KEYFILE_DIR, os.path.basename(dev_path)) - def _get_luks_uuid(self, dev_path): + def _get_luks_uuid(self, dev_path: str) -> str: return self._exec_cmd("sudo cryptsetup luksUUID %s" % dev_path).strip() - def _update_crypttab_keyfile(self, os_root_dir, uuid_to_keyfile): + def _update_crypttab_keyfile( + self, os_root_dir: str, uuid_to_keyfile: dict[str, str] + ) -> None: """Update the keyfile column in crypttab for matching LUKS UUIDs.""" # cryptsetup-initramfs (Ubuntu 22.04+) only embeds crypttab entries # in the initramfs when the device is verifiable at build time OR when @@ -364,7 +371,7 @@ def _update_crypttab_keyfile(self, os_root_dir, uuid_to_keyfile): self._detect_initramfs_tool(os_root_dir) == "update-initramfs" ) - def _set_keyfile(parts): + def _set_keyfile(parts: list[str]) -> list[str]: if len(parts) < 2: return None @@ -398,7 +405,7 @@ def _set_keyfile(parts): LOG.info("Updated crypttab keyfile entries in '%s'", os_root_dir) - def _write_migration_keyfiles(self, os_root_dir): + def _write_migration_keyfiles(self, os_root_dir: str) -> None: """Write migration keyfiles into the OS and update crypttab.""" passphrase = self._osmorphing_info.get(constants.ENCRYPTED_DISKS_PASS) if not passphrase or not self._luks_opened: @@ -443,7 +450,9 @@ def _write_migration_keyfiles(self, os_root_dir): "keyfile-based LUKS auto-unlock." % os_root_dir ) - def _configure_dracut_keyfiles(self, os_root_dir, uuid_to_keyfile): + def _configure_dracut_keyfiles( + self, os_root_dir: str, uuid_to_keyfile: dict[str, str] + ) -> None: """Write a dracut.conf.d snippet to embed keyfiles in the initramfs.""" install_items = list(uuid_to_keyfile.values()) @@ -477,7 +486,7 @@ def _configure_dracut_keyfiles(self, os_root_dir, uuid_to_keyfile): ) LOG.info("Written dracut LUKS keyfile config at '%s'", _DRACUT_LUKS_CONF_PATH) - def _configure_initramfs_tools_keyfiles(self, os_root_dir): + def _configure_initramfs_tools_keyfiles(self, os_root_dir: str) -> None: """Set KEYFILE_PATTERN in cryptsetup-initramfs conf-hook. cryptsetup-initramfs (Debian / Ubuntu) only embeds keyfiles whose paths @@ -505,7 +514,7 @@ def _configure_initramfs_tools_keyfiles(self, os_root_dir): "Set KEYFILE_PATTERN in cryptsetup-initramfs conf-hook at '%s'", hook_abs ) - def _detect_initramfs_tool(self, os_root_dir): + def _detect_initramfs_tool(self, os_root_dir: str) -> str: initramfs_bins = ["usr/sbin/update-initramfs", "sbin/update-initramfs"] for initramfs_bin in initramfs_bins: path = os.path.join(os_root_dir, initramfs_bin) @@ -519,7 +528,7 @@ def _detect_initramfs_tool(self, os_root_dir): return None - def _build_dracut_include_args(self, os_root_dir): + def _build_dracut_include_args(self, os_root_dir: str) -> list[str]: """Return --include args that force-embed crypttab and LUKS keyfiles. dracut's install_items embeds the keyfile but does not guarantee that @@ -551,7 +560,7 @@ def _build_dracut_include_args(self, os_root_dir): return args - def _rebuild_initramfs(self, os_root_dir): + def _rebuild_initramfs(self, os_root_dir: str) -> None: """Rebuild the initramfs inside the mounted OS chroot. /dev, /proc, /sys, /run are already bind-mounted by mount_os(). @@ -567,7 +576,9 @@ def _rebuild_initramfs(self, os_root_dir): "initramfs for LUKS auto-unlock." % os_root_dir ) - def install_encryption_firstboot_setup(self, os_root_dir, os_morphing_tools): + def install_encryption_firstboot_setup( + self, os_root_dir: str, os_morphing_tools + ) -> None: """Install a firstboot script to re-enroll TPM2.""" if not self._luks_opened: return @@ -593,7 +604,7 @@ def install_encryption_firstboot_setup(self, os_root_dir, os_morphing_tools): script_content, user_provided=False, script_filename="luks-firstboot.sh" ) - def _rebuild_initramfs_dracut(self, os_root_dir): + def _rebuild_initramfs_dracut(self, os_root_dir: str) -> None: """Rebuild all dracut initramfs images inside the OS chroot.""" include_args = self._build_dracut_include_args(os_root_dir) @@ -626,7 +637,7 @@ def _rebuild_initramfs_dracut(self, os_root_dir): % (os_root_dir, kver, " ".join(include_args)) ) - def _fix_grub_luks_root(self, os_root_dir): + def _fix_grub_luks_root(self, os_root_dir: str) -> None: """Patch grub.cfg to use crypttab mapper names for LUKS root devices. update-grub names the root device after the mapper opened by osmount diff --git a/coriolis/osmorphing/osmount/ubuntu.py b/coriolis/osmorphing/osmount/ubuntu.py index bdda543ff..d67ac9332 100644 --- a/coriolis/osmorphing/osmount/ubuntu.py +++ b/coriolis/osmorphing/osmount/ubuntu.py @@ -10,12 +10,12 @@ class UbuntuOSMountTools(base.BaseLinuxOSMountTools): - def check_os(self): + def check_os(self) -> bool: os_info = utils.get_linux_os_info(self._ssh) if os_info and os_info[0] in ('Ubuntu', 'ubuntu'): return True - def setup(self): + def setup(self) -> None: super(UbuntuOSMountTools, self).setup() # NOTE(aznashwan): it's possible that if the Ubuntu minion has diff --git a/coriolis/osmorphing/redhat.py b/coriolis/osmorphing/redhat.py index 086fd3735..6f28980c5 100644 --- a/coriolis/osmorphing/redhat.py +++ b/coriolis/osmorphing/redhat.py @@ -5,6 +5,7 @@ import re import shlex import uuid +from typing import Any from oslo_log import log as logging @@ -37,14 +38,14 @@ def check_os_supported(cls, detected_os_info): def __init__( self, conn, - os_root_dir, - os_root_dev, + os_root_dir: str, + os_root_dev: str, hypervisor, event_manager, - detected_os_info, - osmorphing_parameters, + detected_os_info: dict[str, str], + osmorphing_parameters: dict[Any, Any], operation_timeout=None, - nics_set_dhcp=True, + nics_set_dhcp: bool = True, ): super(BaseRedHatMorphingTools, self).__init__( conn, @@ -178,7 +179,7 @@ def _comment_keys_from_ifcfg_files( ) LOG.debug("Commented all %s references from '%s'" % (keys, fullpath)) - def set_net_config(self, nics_info, dhcp): + def set_net_config(self, nics_info: list | None, dhcp: bool) -> None: if dhcp: nics_info = nics_info or [] if not nics_info: @@ -194,7 +195,7 @@ def set_net_config(self, nics_info, dhcp): LOG.info("Setting static IP configuration") self._setup_network_preservation(nics_info) - def get_installed_packages(self): + def get_installed_packages(self) -> None: cmd = 'rpm -qa --qf "%{NAME}\\n"' try: self.installed_packages = self._exec_cmd_chroot(cmd).splitlines() @@ -203,7 +204,7 @@ def get_installed_packages(self): LOG.trace(utils.get_exception_details()) self.installed_packages = [] - def _yum_install(self, package_names): + def _yum_install(self, package_names: list[str]) -> None: try: yum_cmd = 'yum install %s -y %s %s' % ( " ".join(package_names), @@ -230,7 +231,7 @@ def _yum_uninstall(self, package_names): package_names=package_names, package_manager='yum', error=str(err) ) from err - def _yum_clean_all(self): + def _yum_clean_all(self) -> None: try: self._exec_cmd_chroot("yum clean all") except exception.CoriolisException: @@ -244,10 +245,10 @@ def _yum_clean_all(self): if self._test_path('var/cache/yum'): self._exec_cmd_chroot("rm -rf /var/cache/yum") - def _get_repos_to_enable(self): + def _get_repos_to_enable(self) -> list[Any]: return [] - def enable_repos(self, repo_names): + def enable_repos(self, repo_names: list[Any]): """Enable repositories using subscription-manager for Red Hat Enterprise Linux. @@ -269,7 +270,7 @@ def enable_repos(self, repo_names): f"Error was: {utils.get_exception_details()}" ) - def pre_packages_install(self, package_names): + def pre_packages_install(self, package_names: list[str]) -> None: super(BaseRedHatMorphingTools, self).pre_packages_install(package_names) self._yum_clean_all() if 'grubby' not in self.installed_packages: @@ -277,13 +278,13 @@ def pre_packages_install(self, package_names): else: LOG.debug("Skipping package 'grubby' as it's already installed") - def post_packages_install(self, package_names): + def post_packages_install(self, package_names: list[str]) -> None: self._configure_cloud_init() self._run_dracut() self._set_selinux_autorelabel() super(BaseRedHatMorphingTools, self).post_packages_install(package_names) - def install_packages(self, package_names): + def install_packages(self, package_names: list[str]) -> None: repos_to_enable = self._get_repos_to_enable() self.enable_repos(repos_to_enable) self._yum_install(package_names) @@ -291,7 +292,7 @@ def install_packages(self, package_names): def uninstall_packages(self, package_names): self._yum_uninstall(package_names) - def _run_dracut(self): + def _run_dracut(self) -> None: self._exec_cmd_chroot("dracut --regenerate-all -f") def _set_network_nozeroconf_config(self): diff --git a/coriolis/osmorphing/rocky.py b/coriolis/osmorphing/rocky.py index 5a6f5355c..b38c0fcd7 100644 --- a/coriolis/osmorphing/rocky.py +++ b/coriolis/osmorphing/rocky.py @@ -1,6 +1,8 @@ # Copyright 2023 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from oslo_log import log as logging from coriolis import exception, utils @@ -15,14 +17,14 @@ class BaseRockyLinuxMorphingTools(centos.BaseCentOSMorphingTools): UEFI_GRUB_LOCATION = "/boot/efi/EFI/rocky" @classmethod - def check_os_supported(cls, detected_os_info): + def check_os_supported(cls, detected_os_info: dict[str, str]) -> bool: if detected_os_info['distribution_name'] != (ROCKY_LINUX_DISTRO_IDENTIFIER): return False return cls._version_supported_util( detected_os_info['release_version'], minimum=8 ) - def enable_repos(self, repo_names): + def enable_repos(self, repo_names: list[Any]) -> None: """Enable repositories for Rocky Linux. Uses dnf config-manager for all Rocky Linux versions (8+). diff --git a/coriolis/osmorphing/ubuntu.py b/coriolis/osmorphing/ubuntu.py index 6894774cb..9e3eec2b4 100644 --- a/coriolis/osmorphing/ubuntu.py +++ b/coriolis/osmorphing/ubuntu.py @@ -16,7 +16,7 @@ class BaseUbuntuMorphingTools(debian.BaseDebianMorphingTools): @classmethod - def check_os_supported(cls, detected_os_info): + def check_os_supported(cls, detected_os_info: dict[str, str]) -> bool: if detected_os_info['distribution_name'] != (UBUNTU_DISTRO_IDENTIFIER): return False diff --git a/coriolis/policies/base.py b/coriolis/policies/base.py index b8e3d4a58..7cb2bedbb 100644 --- a/coriolis/policies/base.py +++ b/coriolis/policies/base.py @@ -20,5 +20,5 @@ ] -def list_rules(): +def list_rules() -> list: return POLICY_DEFAULT_RULES diff --git a/coriolis/policies/deployments.py b/coriolis/policies/deployments.py index 63ad0a87a..4ee7c6d4c 100644 --- a/coriolis/policies/deployments.py +++ b/coriolis/policies/deployments.py @@ -9,7 +9,7 @@ DEPLOYMENTS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" -def get_deployments_policy_label(rule_label): +def get_deployments_policy_label(rule_label: str) -> str: return "%s:%s" % (DEPLOYMENTS_POLICY_PREFIX, rule_label) @@ -49,5 +49,5 @@ def get_deployments_policy_label(rule_label): ] -def list_rules(): +def list_rules() -> list[policy.DocumentedRuleDefault]: return DEPLOYMENTS_POLICY_DEFAULT_RULES diff --git a/coriolis/policies/diagnostics.py b/coriolis/policies/diagnostics.py index b5dae247e..528bb60ad 100644 --- a/coriolis/policies/diagnostics.py +++ b/coriolis/policies/diagnostics.py @@ -9,7 +9,7 @@ DIAGNOSTICS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" -def get_diagnostics_policy_label(rule_label): +def get_diagnostics_policy_label(rule_label: str) -> str: return "%s:%s" % (DIAGNOSTICS_POLICY_PREFIX, rule_label) @@ -23,5 +23,5 @@ def get_diagnostics_policy_label(rule_label): ] -def list_rules(): +def list_rules() -> list[policy.DocumentedRuleDefault]: return DIAGNOSTICS_POLICY_DEFAULT_RULES diff --git a/coriolis/policies/endpoints.py b/coriolis/policies/endpoints.py index fd54493cd..fc08597b5 100644 --- a/coriolis/policies/endpoints.py +++ b/coriolis/policies/endpoints.py @@ -10,7 +10,7 @@ ENDPOINTS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" -def get_endpoints_policy_label(rule_label): +def get_endpoints_policy_label(rule_label: str) -> str: return "%s:%s" % (ENDPOINTS_POLICY_PREFIX, rule_label) @@ -124,5 +124,5 @@ def get_endpoints_policy_label(rule_label): ] -def list_rules(): +def list_rules() -> list[policy.DocumentedRuleDefault]: return ENDPOINTS_POLICY_DEFAULT_RULES diff --git a/coriolis/policies/general.py b/coriolis/policies/general.py index 066835d21..e4c697d0a 100644 --- a/coriolis/policies/general.py +++ b/coriolis/policies/general.py @@ -9,7 +9,7 @@ PROVIDERS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" -def get_providers_policy_label(rule_label): +def get_providers_policy_label(rule_label: str) -> str: return "%s:%s" % (PROVIDERS_POLICY_PREFIX, rule_label) @@ -23,5 +23,5 @@ def get_providers_policy_label(rule_label): ] -def list_rules(): +def list_rules() -> list[policy.DocumentedRuleDefault]: return PROVIDERS_POLICY_DEFAULT_RULES diff --git a/coriolis/policies/minion_pools.py b/coriolis/policies/minion_pools.py index 749ef7ea9..f62e44951 100644 --- a/coriolis/policies/minion_pools.py +++ b/coriolis/policies/minion_pools.py @@ -10,7 +10,7 @@ MINION_POOLS_DEFAULT_RULE = "rule:admin_or_owner" -def get_minion_pools_policy_label(rule_label): +def get_minion_pools_policy_label(rule_label: str) -> str: return "%s:%s" % (MINION_POOLS_POLICY_PREFIX, rule_label) @@ -66,5 +66,5 @@ def get_minion_pools_policy_label(rule_label): ] -def list_rules(): +def list_rules() -> list[policy.DocumentedRuleDefault]: return MINION_POOLS_DEFAULT_RULES diff --git a/coriolis/policies/regions.py b/coriolis/policies/regions.py index df5026293..1b2ed5469 100644 --- a/coriolis/policies/regions.py +++ b/coriolis/policies/regions.py @@ -10,7 +10,7 @@ REGIONS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" -def get_regions_policy_label(rule_label): +def get_regions_policy_label(rule_label: str) -> str: return "%s:%s" % (REGIONS_POLICY_PREFIX, rule_label) @@ -48,5 +48,5 @@ def get_regions_policy_label(rule_label): ] -def list_rules(): +def list_rules() -> list[policy.DocumentedRuleDefault]: return REGIONS_POLICY_DEFAULT_RULES diff --git a/coriolis/policies/services.py b/coriolis/policies/services.py index 114cd0e1f..9b602cd86 100644 --- a/coriolis/policies/services.py +++ b/coriolis/policies/services.py @@ -10,7 +10,7 @@ SERVICES_POLICY_DEFAULT_RULE = "rule:admin_or_owner" -def get_services_policy_label(rule_label): +def get_services_policy_label(rule_label: str) -> str: return "%s:%s" % (SERVICES_POLICY_PREFIX, rule_label) @@ -48,5 +48,5 @@ def get_services_policy_label(rule_label): ] -def list_rules(): +def list_rules() -> list[policy.DocumentedRuleDefault]: return SERVICES_POLICY_DEFAULT_RULES diff --git a/coriolis/policies/transfer_schedules.py b/coriolis/policies/transfer_schedules.py index 524c9af85..0492d1527 100644 --- a/coriolis/policies/transfer_schedules.py +++ b/coriolis/policies/transfer_schedules.py @@ -11,7 +11,7 @@ TRANSFER_SCHEDULES_POLICY_DEFAULT_RULE = "rule:admin_or_owner" -def get_transfer_schedules_policy_label(rule_label): +def get_transfer_schedules_policy_label(rule_label: str) -> str: return "%s:%s" % (TRANSFER_SCHEDULES_POLICY_PREFIX, rule_label) @@ -59,5 +59,5 @@ def get_transfer_schedules_policy_label(rule_label): ] -def list_rules(): +def list_rules() -> list[policy.DocumentedRuleDefault]: return TRANSFER_SCHEDULES_POLICY_DEFAULT_RULES diff --git a/coriolis/policies/transfer_tasks_executions.py b/coriolis/policies/transfer_tasks_executions.py index 589b42cf2..25ca6e5e7 100644 --- a/coriolis/policies/transfer_tasks_executions.py +++ b/coriolis/policies/transfer_tasks_executions.py @@ -11,7 +11,7 @@ TRANSFER_EXECUTIONS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" -def get_transfer_executions_policy_label(rule_label): +def get_transfer_executions_policy_label(rule_label: str) -> str: return "%s:%s" % (TRANSFER_EXECUTIONS_POLICY_PREFIX, rule_label) @@ -66,5 +66,5 @@ def get_transfer_executions_policy_label(rule_label): ] -def list_rules(): +def list_rules() -> list[policy.DocumentedRuleDefault]: return TRANSFER_EXECUTIONS_POLICY_DEFAULT_RULES diff --git a/coriolis/policies/transfers.py b/coriolis/policies/transfers.py index 25398e451..e43a82869 100644 --- a/coriolis/policies/transfers.py +++ b/coriolis/policies/transfers.py @@ -9,7 +9,7 @@ TRANSFERS_POLICY_DEFAULT_RULE = "rule:admin_or_owner" -def get_transfers_policy_label(rule_label): +def get_transfers_policy_label(rule_label: str) -> str: return "%s:%s" % (TRANSFERS_POLICY_PREFIX, rule_label) @@ -55,5 +55,5 @@ def get_transfers_policy_label(rule_label): ] -def list_rules(): +def list_rules() -> list[policy.DocumentedRuleDefault]: return TRANSFERS_POLICY_DEFAULT_RULES diff --git a/coriolis/policy.py b/coriolis/policy.py index 50ca72f6e..19785f11e 100644 --- a/coriolis/policy.py +++ b/coriolis/policy.py @@ -42,14 +42,14 @@ ] -def reset(): +def reset() -> None: global _ENFORCER if _ENFORCER: _ENFORCER.clear() _ENFORCER = None -def init(): +def init() -> None: global _ENFORCER global saved_file_rules @@ -59,7 +59,7 @@ def init(): _ENFORCER.load_rules() -def register_rules(enforcer): +def register_rules(enforcer: policy.Enforcer) -> None: enforcer.register_defaults( itertools.chain(*[m.list_rules() for m in DEFAULT_POLICIES_MODULES]) ) @@ -70,7 +70,13 @@ def get_enforcer(): return _ENFORCER -def check_policy_for_context(context, action, target, exc=None, do_raise=True): +def check_policy_for_context( + context, + action: str, + target: dict[str, str], + exc: type[exception.PolicyNotAuthorized] | None = None, + do_raise: bool = True, +) -> bool: """Checks the validity of the given action of the given target based on set policies. On success, returns a value where bool(val) == True. diff --git a/coriolis/providers/api.py b/coriolis/providers/api.py index 0d2e23ea3..a443ba0fb 100644 --- a/coriolis/providers/api.py +++ b/coriolis/providers/api.py @@ -8,10 +8,15 @@ class API(object): def __init__(self): self._rpc_client = rpc_client.ConductorClient() - def get_available_providers(self, ctxt): + def get_available_providers(self, ctxt) -> dict[str, dict]: return self._rpc_client.get_available_providers(ctxt) - def get_provider_schemas(self, ctxt, platform_name, provider_type): + def get_provider_schemas( + self, + ctxt, + platform_name: str, + provider_type: str, + ) -> dict[str, dict]: return self._rpc_client.get_provider_schemas( ctxt, platform_name, int(provider_type) ) diff --git a/coriolis/providers/backup_writers.py b/coriolis/providers/backup_writers.py index 8c20f4790..2bf3e94d1 100644 --- a/coriolis/providers/backup_writers.py +++ b/coriolis/providers/backup_writers.py @@ -14,6 +14,7 @@ import threading import time import uuid +from typing import Iterator import paramiko from oslo_config import cfg @@ -85,7 +86,7 @@ _CHECKSUM_JOB_FAILED = "failed" -def _disable_lvm2_lvmetad(ssh): +def _disable_lvm2_lvmetad(ssh: paramiko.SSHClient) -> None: """Disables lvm2-lvmetad service. This service is responsible for automatically activating LVM2 volume groups when a disk is attached or when a volume group is created. During disk replication @@ -115,7 +116,7 @@ def _disable_lvm2_lvmetad(ssh): ) -def _disable_lvm_metad_udev_rule(ssh): +def _disable_lvm_metad_udev_rule(ssh: paramiko.SSHClient) -> None: """ Removes lvm-metad rule which creates lvm2-pvscan services for each disk detected with lvm partitions (after a transfer is complete). During normal @@ -137,7 +138,7 @@ def _disable_lvm_metad_udev_rule(ssh): utils.exec_ssh_cmd(ssh, "sudo rm %s" % path, get_pty=False) -def _check_deserialize_key(key): +def _check_deserialize_key(key: paramiko.RSAKey | str) -> paramiko.RSAKey: res = None if isinstance(key, paramiko.RSAKey): LOG.trace("Key is already in the proper format.") @@ -155,13 +156,13 @@ def _check_deserialize_key(key): class BackupWritersFactory(object): - def __init__(self, writer_connection_info, volumes_info): + def __init__(self, writer_connection_info: dict, volumes_info: list | None): self._validate_info(writer_connection_info) self._type = writer_connection_info["backend"] self._conn_info = writer_connection_info["connection_details"] self._volumes_info = volumes_info - def get_writer(self): + def get_writer(self) -> "HTTPBackupWriter|SSHBackupWriter": if self._type == BACKUP_WRITER_SSH: return SSHBackupWriter.from_connection_info( self._conn_info, self._volumes_info @@ -176,7 +177,7 @@ def get_writer(self): ) raise exception.CoriolisException("Invalid backup writer type: %s" % self._type) - def _validate_info(self, info): + def _validate_info(self, info: dict) -> None: if type(info) is not dict: raise exception.CoriolisException("Invalid backup writer connection info.") wrt_type = info.get("backend", None) @@ -237,19 +238,19 @@ def get_backup_writer_connection_info( class BaseBackupWriterImpl(with_metaclass(abc.ABCMeta)): - def __init__(self, path, disk_id): + def __init__(self, path: str, disk_id: str): self._path = path self._disk_id = disk_id @abc.abstractmethod - def _open(self): + def _open(self) -> None: pass def _handle_exception(self, ex): LOG.exception(ex) @abc.abstractmethod - def seek(self, pos): + def seek(self, pos: int) -> None: pass @abc.abstractmethod @@ -257,7 +258,7 @@ def truncate(self, size): pass @abc.abstractmethod - def write(self, data, encoding=None, uncompressed_size=None): + def write(self, data: bytes, encoding=None, uncompressed_size=None) -> None: """Write the given data chunk. Use the `seek` method to specify the write offset. @@ -270,7 +271,7 @@ def write(self, data, encoding=None, uncompressed_size=None): pass @abc.abstractmethod - def close(self): + def close(self) -> None: pass def get_disk_checksum(self, algorithm, start_offset=0, end_offset=0): @@ -288,11 +289,15 @@ def get_disk_checksum(self, algorithm, start_offset=0, end_offset=0): class BaseBackupWriter(with_metaclass(abc.ABCMeta)): @abc.abstractmethod - def _get_impl(self, path, disk_id): + def _get_impl( + self, path: str, disk_id: str + ) -> "HTTPBackupWriterImpl|SSHBackupWriterImpl": pass @contextlib.contextmanager - def open(self, path, disk_id): + def open( + self, path: str, disk_id: str + ) -> "Iterator[HTTPBackupWriterImpl|SSHBackupWriterImpl]": impl = None try: impl = self._get_impl(path, disk_id) @@ -359,7 +364,13 @@ def from_connection_info(cls, info, volumes_info): class SSHBackupWriterImpl(BaseBackupWriterImpl): - def __init__(self, path, disk_id, compress_transfer=False, encoder_count=3): + def __init__( + self, + path: str, + disk_id: str, + compress_transfer: bool = False, + encoder_count: int = 3, + ): self._msg_id = None self._stdin = None self._stdout = None @@ -384,18 +395,18 @@ def __init__(self, path, disk_id, compress_transfer=False, encoder_count=3): self._compress_transfer = CONF.compress_transfers super(SSHBackupWriterImpl, self).__init__(path, disk_id) - def _set_ssh_client(self, ssh): + def _set_ssh_client(self, ssh: paramiko.SSHClient) -> None: self._ssh = ssh @utils.retry_on_error() - def _exec_helper_cmd(self): + def _exec_helper_cmd(self) -> None: self._msg_id = 0 self._offset = 0 self._stdin, self._stdout, self._stderr = self._ssh.exec_command( "chmod +x write_data && sudo ./write_data" ) - def _encode_data(self, content, offset, msg_id): + def _encode_data(self, content: bytes, offset: int, msg_id: int) -> bytes: msg = data_transfer.encode_data( msg_id, self._path, offset, content, compress=self._compress_transfer ) @@ -412,13 +423,13 @@ def _encode_data(self, content, offset, msg_id): ) return msg - def _encode_eod(self): + def _encode_eod(self) -> bytes: msg = data_transfer.encode_eod(self._msg_id) LOG.debug("EOD message len: %d", len(msg)) return msg @utils.retry_on_error() - def _send_msg(self, data): + def _send_msg(self, data) -> bool | list | paramiko.SSHClient | None: # check if write_data is still alive if self._stdout.channel.exit_status_ready(): ret_val = self._stdout.channel.recv_exit_status() @@ -432,19 +443,19 @@ def _send_msg(self, data): self._stdin.flush() self._stdout.read(4) - def _open(self): + def _open(self) -> None: self._exec_helper_cmd() self._sender_thread = utils.start_thread(self._sender) for _ in range(self._encoder_cnt): self._encoder_threads.append(utils.start_thread(self._encoder)) - def seek(self, pos): + def seek(self, pos: int) -> None: self._offset = pos def truncate(self, size): pass - def _sender(self): + def _sender(self) -> None: LOG.debug("Backup sender started.") while not self._stopped: try: @@ -463,7 +474,7 @@ def _sender(self): del data LOG.debug("Backup sender stopped.") - def _encoder(self): + def _encoder(self) -> None: LOG.debug("Backup encoder started.") while not self._stopped: try: @@ -484,7 +495,7 @@ def _encoder(self): self._enc_q.task_done() LOG.debug("Backup encoder stopped.") - def write(self, data, encoding=None, uncompressed_size=None): + def write(self, data: bytes, encoding=None, uncompressed_size=None) -> None: """Write the given data chunk. Use the `seek` method to specify the write offset. @@ -515,7 +526,7 @@ def write(self, data, encoding=None, uncompressed_size=None): self._offset += len(data) self._msg_id += 1 - def _wait_for_queues(self): + def _wait_for_queues(self) -> None: LOG.info("Waiting for unfinished transfers to complete") timeout = datetime.datetime.now() + datetime.timedelta(seconds=600) while ( @@ -533,7 +544,7 @@ def _wait_for_queues(self): "Timed out waiting for data transfer to finish" ) - def close(self): + def close(self) -> None: self._closing = True self._wait_for_queues() self._stopped = True @@ -586,7 +597,15 @@ def _handle_exception(self, ex): class SSHBackupWriter(BaseBackupWriter): - def __init__(self, ip, port, username, pkey, password, volumes_info): + def __init__( + self, + ip: str, + port: int, + username: str, + pkey: paramiko.RSAKey, + password, + volumes_info: list | None, + ): self._ip = ip self._port = port self._username = username @@ -597,7 +616,9 @@ def __init__(self, ip, port, username, pkey, password, volumes_info): self._lock = threading.Lock() @classmethod - def from_connection_info(cls, info, volumes_info): + def from_connection_info( + cls, info: dict, volumes_info: list | None + ) -> "SSHBackupWriter": required = ["ip", "port", "username"] ip = info.get("ip") port = info.get("port") @@ -618,7 +639,7 @@ def from_connection_info(cls, info, volumes_info): return cls(ip, port, username, pkey, password, volumes_info) - def _get_impl(self, path, disk_id): + def _get_impl(self, path: str, disk_id: str) -> SSHBackupWriterImpl: ssh = self._connect_ssh() _disable_lvm_metad_udev_rule(ssh) _disable_lvm2_lvmetad(ssh) @@ -642,7 +663,7 @@ def _get_impl(self, path, disk_id): return impl @utils.retry_on_error() - def _copy_helper_cmd(self, ssh): + def _copy_helper_cmd(self, ssh: paramiko.SSHClient) -> None: with self._lock: sftp = ssh.open_sftp() local_path = os.path.join(utils.get_resources_bin_dir(), 'write_data') @@ -657,7 +678,7 @@ def _copy_helper_cmd(self, ssh): sftp.close() @utils.retry_on_error(sleep_seconds=30) - def _connect_ssh(self): + def _connect_ssh(self) -> paramiko.SSHClient: LOG.info( "Connecting to SSH host: %(ip)s:%(port)s" % {"ip": self._ip, "port": self._port} @@ -672,7 +693,13 @@ def _connect_ssh(self): class HTTPBackupWriterImpl(BaseBackupWriterImpl): - def __init__(self, path, disk_id, compress_transfer=None, compressor_count=3): + def __init__( + self, + path: str, + disk_id: str, + compress_transfer: bool = None, + compressor_count: int = 3, + ): self._offset = None self._session = None self._ip = None @@ -701,7 +728,7 @@ def __init__(self, path, disk_id, compress_transfer=None, compressor_count=3): self._compress_transfer = CONF.compress_transfers super(HTTPBackupWriterImpl, self).__init__(path, disk_id) - def _set_info(self, info): + def _set_info(self, info: dict) -> None: self._ip = info.get("ip") self._port = info.get("port") self._crt = info.get("client_crt") @@ -714,12 +741,12 @@ def _set_info(self, info): ) @property - def _uri(self): + def _uri(self) -> str: b64_path = base64.b64encode(self._path.encode()).decode() return "https://%s:%s/api/v2/device/%s" % (self._ip, self._port, b64_path) @utils.retry_on_error() - def _acquire(self): + def _acquire(self) -> None: self._ensure_session() uri = "%s/acquire" % self._uri headers = {"X-Client-Token": self._id} @@ -730,7 +757,7 @@ def _acquire(self): resp.raise_for_status() @utils.retry_on_error() - def _release(self): + def _release(self) -> list | paramiko.SSHClient | None: self._ensure_session() uri = "%s/release" % self._uri headers = {"X-Client-Token": self._id} @@ -740,7 +767,7 @@ def _release(self): LOG.debug("Returned code: %d. Msg: %s" % (resp.status_code, resp.content)) resp.raise_for_status() - def _init_session(self): + def _init_session(self) -> None: if self._session: self._session.close() sess = provider_utils.ProviderSession() @@ -748,7 +775,7 @@ def _init_session(self): sess.verify = self._ca self._session = sess - def _open(self): + def _open(self) -> None: self._closing = False self._init_session() self._acquire() @@ -759,13 +786,13 @@ def _open(self): for _ in range(self._compressor_count): self._compressor_threads.append(utils.start_thread(self._compressor)) - def seek(self, pos): + def seek(self, pos: int) -> None: self._offset = pos def truncate(self, size): pass - def _ensure_session(self): + def _ensure_session(self) -> None: if not self._session: self._init_session() return @@ -773,7 +800,7 @@ def _ensure_session(self): self._init_session() return - def _compressor(self): + def _compressor(self) -> None: LOG.debug("Backup compressor started.") while not self._stopped: try: @@ -803,7 +830,7 @@ def _compressor(self): self._comp_q.task_done() LOG.debug("Backup compressor stopped.") - def _sender(self): + def _sender(self) -> None: LOG.debug("Backup sender started.") while not self._stopped: try: @@ -825,7 +852,7 @@ def _sender(self): ) @utils.retry_on_error() - def send(): + def send() -> None: self._ensure_session() chunk = copy.copy(payload["chunk"]) LOG.debug( @@ -869,7 +896,7 @@ def send(): LOG.debug("Backup sender stopped.") @utils.retry_on_error() - def write(self, data, encoding=None, uncompressed_size=None): + def write(self, data: bytes, encoding=None, uncompressed_size=None) -> None: """Write the given data chunk. Use the `seek` method to specify the write offset. @@ -913,7 +940,7 @@ def write(self, data, encoding=None, uncompressed_size=None): raise exception.InvalidInput("Unsupported write encoding: %s" % encoding) self._offset += len(data) - def _wait_for_queues(self): + def _wait_for_queues(self) -> None: timeout = datetime.datetime.now() + datetime.timedelta(seconds=600) while ( self._comp_q.unfinished_tasks or self._sender_q.unfinished_tasks @@ -931,7 +958,9 @@ def _wait_for_queues(self): "Timed out waiting for data transfer to finish" ) - def _create_checksum_job(self, algorithm, start_offset=0, end_offset=0): + def _create_checksum_job( + self, algorithm: str, start_offset: int = 0, end_offset: int = 0 + ) -> str: """Creates a full-disk checksum job on the writer. The device must already be acquired. @@ -958,7 +987,7 @@ def _create_checksum_job(self, algorithm, start_offset=0, end_offset=0): return resp.json()["job_id"] - def _get_checksum_job_status(self, job_id): + def _get_checksum_job_status(self, job_id: str) -> dict: """Returns the current status of a writer checksum job.""" self._ensure_session() uri = "%s/checksumJob/%s" % (self._uri, job_id) @@ -968,7 +997,7 @@ def _get_checksum_job_status(self, job_id): return resp.json() - def _delete_checksum_job(self, job_id): + def _delete_checksum_job(self, job_id: str) -> None: """Deletes a writer checksum job.""" self._ensure_session() uri = "%s/checksumJob/%s" % (self._uri, job_id) @@ -976,7 +1005,9 @@ def _delete_checksum_job(self, job_id): resp = self._session.delete(uri, timeout=CONF.default_requests_timeout) resp.raise_for_status() - def get_disk_checksum(self, algorithm, start_offset=0, end_offset=0): + def get_disk_checksum( + self, algorithm: str, start_offset: int = 0, end_offset: int = 0 + ) -> dict[str, str]: """Computes and returns the checksum of the entire destination disk. Must be called while the device is acquired (inside open() context). @@ -1029,7 +1060,7 @@ def get_disk_checksum(self, algorithm, start_offset=0, end_offset=0): "Failed to delete checksum job %s for disk %s", job_id, self._path ) - def close(self): + def close(self) -> None: self._closing = True self._wait_for_queues() self._stopped = True @@ -1060,7 +1091,7 @@ def close(self): class HTTPBackupWriterBootstrapper(object): - def __init__(self, ssh_conn_info, writer_port): + def __init__(self, ssh_conn_info: dict, writer_port: int): self._lock = threading.Lock() self._writer_cmd = os.path.join("/usr/bin", _CORIOLIS_HTTP_WRITER_CMD) self._writer_port = writer_port @@ -1080,7 +1111,7 @@ def __init__(self, ssh_conn_info, writer_port): self._ssh = self._connect_ssh() @utils.retry_on_error(sleep_seconds=30) - def _connect_ssh(self): + def _connect_ssh(self) -> paramiko.SSHClient: LOG.info( "Connecting to SSH host: %(ip)s:%(port)s" % {"ip": self._ip, "port": self._port} @@ -1093,7 +1124,7 @@ def _connect_ssh(self): password=self._password, ) - def _inject_dport_allow_rule(self, ssh): + def _inject_dport_allow_rule(self, ssh: paramiko.SSHClient) -> None: cmd = ( "sudo nft insert rule ip filter INPUT tcp dport %(port)s counter " "accept || " @@ -1108,7 +1139,7 @@ def _inject_dport_allow_rule(self, ssh): utils.get_exception_details(), ) - def _add_firewalld_port(self, ssh): + def _add_firewalld_port(self, ssh: paramiko.SSHClient) -> None: cmd = "sudo firewall-cmd --add-port=%s/tcp" % self._writer_port try: utils.exec_ssh_cmd(ssh, cmd, get_pty=False) @@ -1118,7 +1149,7 @@ def _add_firewalld_port(self, ssh): utils.get_exception_details(), ) - def _change_binary_se_context(self, ssh): + def _change_binary_se_context(self, ssh: paramiko.SSHClient) -> None: cmd = "sudo chcon -t bin_t %s" % self._writer_cmd try: utils.exec_ssh_cmd(ssh, cmd, get_pty=False) @@ -1129,7 +1160,7 @@ def _change_binary_se_context(self, ssh): ) @utils.retry_on_error() - def _copy_writer(self, ssh): + def _copy_writer(self, ssh: paramiko.SSHClient) -> None: local_path = os.path.join( utils.get_resources_bin_dir(), _CORIOLIS_HTTP_WRITER_CMD ) @@ -1154,7 +1185,7 @@ def _copy_writer(self, ssh): finally: sftp.close() - def _setup_certificates(self, ssh): + def _setup_certificates(self, ssh: paramiko.SSHClient) -> dict[str, str]: remote_base_dir = "/etc/coriolis-writer" ca_crt_name = "ca-cert.pem" @@ -1202,13 +1233,13 @@ def _setup_certificates(self, ssh): "client_key": remote_client_key, } - def _read_remote_file_sudo(self, remote_path): + def _read_remote_file_sudo(self, remote_path: str) -> str: contents = utils.exec_ssh_cmd( self._ssh, 'sudo cat "%s"' % remote_path, get_pty=False ) return contents - def _init_writer(self, ssh, cert_paths): + def _init_writer(self, ssh: paramiko.SSHClient, cert_paths: dict[str, str]) -> None: cmdline = ( "%(cmd)s run -ca-cert %(ca_cert)s -key " "%(srv_key)s -cert %(srv_cert)s -listen-port " @@ -1225,7 +1256,7 @@ def _init_writer(self, ssh, cert_paths): self._inject_dport_allow_rule(ssh) self._add_firewalld_port(ssh) - def setup_writer(self): + def setup_writer(self) -> dict: _disable_lvm_metad_udev_rule(self._ssh) _disable_lvm2_lvmetad(self._ssh) self._copy_writer(self._ssh) @@ -1243,7 +1274,14 @@ def setup_writer(self): class HTTPBackupWriter(BaseBackupWriter): - def __init__(self, ip, port, volumes_info, certificates, compressor_count=3): + def __init__( + self, + ip: str, + port: int, + volumes_info: list | None, + certificates: dict[str, str], + compressor_count: int = 3, + ): self._ip = ip self._port = port self._volumes_info = volumes_info @@ -1258,7 +1296,9 @@ def __init__(self, ip, port, volumes_info, certificates, compressor_count=3): self._cert_paths = None @classmethod - def from_connection_info(cls, conn_info, volumes_info): + def from_connection_info( + cls, conn_info: dict, volumes_info: list | None + ) -> "HTTPBackupWriter": """Instantiate a HTTP backup writer from connection info. Connection info has the following schema: @@ -1300,21 +1340,21 @@ def from_connection_info(cls, conn_info, volumes_info): return cls(ip, port, volumes_info, certs) - def __del__(self): + def __del__(self) -> None: if self._crt_dir and os.path.isdir(self._crt_dir): try: shutil.rmtree(self._crt_dir) except BaseException: pass - def _wait_for_conn(self): + def _wait_for_conn(self) -> None: LOG.debug( "waiting for coriolis-writer connectivity %s:%s" % (self._ip, self._writer_port) ) utils.wait_for_port_connectivity(self._ip, self._writer_port) - def _write_cert_files(self): + def _write_cert_files(self) -> dict[str, str]: if not self._certificates: raise exception.CoriolisException("certificates not set") if self._cert_paths: @@ -1336,7 +1376,7 @@ def _write_cert_files(self): } return self._cert_paths - def _get_impl(self, path, disk_id): + def _get_impl(self, path: str, disk_id: str) -> HTTPBackupWriterImpl: cert_paths = self._write_cert_files() self._wait_for_conn() diff --git a/coriolis/providers/factory.py b/coriolis/providers/factory.py index 7420bc8d6..814710b95 100644 --- a/coriolis/providers/factory.py +++ b/coriolis/providers/factory.py @@ -49,7 +49,7 @@ } -def get_available_providers(): +def get_available_providers() -> dict[str, dict]: providers = {} for provider in CONF.providers: cls = utils.load_class(provider) @@ -65,7 +65,12 @@ def get_available_providers(): return providers -def get_provider(platform_name, provider_type, event_handler, raise_if_not_found=True): +def get_provider( + platform_name: str, + provider_type: int, + event_handler, + raise_if_not_found: bool = True, +): for provider in CONF.providers: cls = utils.load_class(provider) parent = PROVIDER_TYPE_MAP.get(provider_type) diff --git a/coriolis/providers/provider_utils.py b/coriolis/providers/provider_utils.py index e13be2723..8d568cfcf 100644 --- a/coriolis/providers/provider_utils.py +++ b/coriolis/providers/provider_utils.py @@ -1,6 +1,7 @@ # Copyright 2018 Cloudbase Solutions Srl # All Rights Reserved. import time +from typing import Any import paramiko import requests @@ -158,7 +159,15 @@ def check_changed_storage_mappings( class ProviderSession(requests.Session): - def merge_environment_settings(self, url, proxies, stream, verify, *args, **kwargs): + def merge_environment_settings( + self, + url: str, + proxies: dict[Any, Any], + stream, + verify: str | None, + *args, + **kwargs, + ) -> dict: verify = self.verify return super(ProviderSession, self).merge_environment_settings( url, proxies, stream, verify, *args, **kwargs diff --git a/coriolis/providers/replicator.py b/coriolis/providers/replicator.py index 7beca5643..b85e2a643 100644 --- a/coriolis/providers/replicator.py +++ b/coriolis/providers/replicator.py @@ -9,6 +9,7 @@ import tempfile import threading import time +from typing import Any import paramiko from oslo_config import cfg @@ -52,13 +53,13 @@ class _SSHTunnel(object): def __init__( self, - ssh_host, - ssh_port, - ssh_username, - ssh_pkey, + ssh_host: str, + ssh_port: int, + ssh_username: str, + ssh_pkey: paramiko.RSAKey, ssh_password, - remote_bind_address, - local_bind_address=None, + remote_bind_address: tuple[str, int], + local_bind_address: tuple[str, int] = None, ): self._ssh_host = ssh_host self._ssh_port = ssh_port @@ -71,7 +72,7 @@ def __init__( self._listen_sock = None self.local_bind_address = None - def start(self): + def start(self) -> None: client = utils.connect_ssh( self._ssh_host, self._ssh_port, @@ -104,7 +105,7 @@ def _accept_loop(self): t = threading.Thread(target=self._forward, args=(sock,), daemon=True) t.start() - def _forward(self, local_sock): + def _forward(self, local_sock) -> None: try: chan = self._transport.open_channel( 'direct-tcpip', @@ -161,13 +162,13 @@ def stop(self): class Client(object): def __init__( self, - ip, - port, - credentials, - ssh_conn_info, + ip: str, + port: int, + credentials: dict[str, str], + ssh_conn_info: dict, event_handler, - use_compression=False, - use_tunnel=False, + use_compression: bool = False, + use_tunnel: bool = False, ): self._ip = ip self._use_tunnel = use_tunnel @@ -183,29 +184,29 @@ def __init__( self._test_connection() @property - def repl_host(self): + def repl_host(self) -> str: if self._ip_via_tunnel is not None: return self._ip_via_tunnel return self._ip @property - def repl_port(self): + def repl_port(self) -> int: if self._port_via_tunnel is not None: return self._port_via_tunnel return self._port @property - def _base_uri(self): + def _base_uri(self) -> str: return "https://%s:%s" % (self.repl_host, self.repl_port) - def _setup_tunnel_connection(self): + def _setup_tunnel_connection(self) -> None: self._tunnel = self._get_ssh_tunnel() self._tunnel.start() host, port = self._tunnel.local_bind_address self._ip_via_tunnel = host self._port_via_tunnel = port - def _test_connection(self): + def _test_connection(self) -> None: """ Attempt to connect to the IP/port pair. If direct connection fails, set up a SSH tunnel and attempt a connection through that. @@ -244,7 +245,7 @@ def _test_connection(self): LOG.warning("failed to connect to replicator: %s" % err) raise - def _get_ssh_tunnel(self): + def _get_ssh_tunnel(self) -> _SSHTunnel: """ gets a SSH tunnel object. Note, this does not start the tunnel, it simply creates the object, without actually connecting. @@ -272,18 +273,20 @@ def _get_ssh_tunnel(self): ) return server - def raw_disk_uri(self, disk_name): + def raw_disk_uri(self, disk_name: str) -> str: diskUri = "%s/device/%s" % (self._base_uri, disk_name) return diskUri - def _get_session(self): + def _get_session(self) -> provider_utils.ProviderSession: sess = provider_utils.ProviderSession() sess.cert = (self._creds["client_cert"], self._creds["client_key"]) sess.verify = self._creds["ca_cert"] return sess @utils.retry_on_error(sleep_seconds=5) - def get_status(self, device=None, brief=True): + def get_status( + self, device: dict | None = None, brief: bool = True + ) -> list | paramiko.SSHClient: uri = "%s/api/v1/dev" % (self._base_uri) if device is not None: uri = "%s/%s/" % (uri, device) @@ -309,21 +312,21 @@ def get_chunks(self, device, skip_zeros=False): return chunks.json() @utils.retry_on_error() - def get_changes(self, device): + def get_changes(self, device: str) -> list[dict]: uri = "%s/api/v1/dev/%s/chunks/changes/" % (self._base_uri, device) chunks = self._cli.get(uri, timeout=CONF.replicator.default_requests_timeout) chunks.raise_for_status() return chunks.json() @utils.retry_on_error() - def get_disk_size(self, disk): + def get_disk_size(self, disk) -> int | paramiko.SSHClient | None: diskUri = self.raw_disk_uri(disk) info = self._cli.head(diskUri, timeout=CONF.replicator.default_requests_timeout) info.raise_for_status() return int(info.headers["Content-Length"]) @utils.retry_on_error() - def get_disk_checksum(self, device): + def get_disk_checksum(self, device: str) -> dict[str, str]: """Returns the total checksum of the given disk. :raises HTTPError: with HTTP 409 status if checksumming has not @@ -338,7 +341,7 @@ def get_disk_checksum(self, device): return resp.json() @utils.retry_on_error() - def download_chunk(self, disk, chunk): + def download_chunk(self, disk: str, chunk: dict) -> bytes: diskUri = self.raw_disk_uri(disk) offset = int(chunk["offset"]) @@ -360,18 +363,18 @@ def download_chunk(self, disk, chunk): class Replicator(object): def __init__( self, - conn_info, + conn_info: dict, event_manager, - volumes_info, - replica_state, + volumes_info: list, + replica_state: list | None, use_compression=None, - ignore_mounted=True, - hash_method=HASH_METHOD_SHA256, - watch_devices=True, - chunk_size=10485760, - use_tunnel=False, - disable_chunking=False, - _allow_loop_devices=False, + ignore_mounted: bool = True, + hash_method: str = HASH_METHOD_SHA256, + watch_devices: bool = True, + chunk_size: int = 10485760, + use_tunnel: bool = False, + disable_chunking: bool = False, + _allow_loop_devices: bool = False, ): self._event_manager = event_manager self._repl_state = replica_state @@ -395,11 +398,11 @@ def __init__( self._cli = None self._use_tunnel = use_tunnel - def __del__(self): + def __del__(self) -> None: if self._cert_dir is not None: utils.ignore_exceptions(shutil.rmtree)(self._cert_dir) - def _init_replicator_client(self, credentials): + def _init_replicator_client(self, credentials: dict[str, str]) -> None: """ Helper function to create an instance of the replicator client. @@ -416,19 +419,19 @@ def _init_replicator_client(self, credentials): use_tunnel=self._use_tunnel, ) - def _setup_ssh(self): + def _setup_ssh(self) -> paramiko.SSHClient: args = self._parse_source_ssh_conn_info(self._conn_info) ssh = self._get_ssh_client(args) return ssh - def _reconnect_ssh(self): + def _reconnect_ssh(self) -> paramiko.SSHClient: if self._ssh: utils.ignore_exceptions(self._ssh.close)() self._ssh = self._setup_ssh() return self._ssh - def init_replicator(self): + def init_replicator(self) -> None: try: self._credentials = utils.retry_on_error(sleep_seconds=5)( self._setup_replicator @@ -604,7 +607,7 @@ def attach_new_disk( return vol_info - def wait_for_chunks(self): + def wait_for_chunks(self) -> None: if self._cli is None: raise exception.CoriolisException( "replicator not initialized. Run init_replicator()" @@ -639,10 +642,10 @@ def start(self): def stop(self): utils.stop_service(self._ssh, REPLICATOR_SVC_NAME) - def restart(self): + def restart(self) -> None: utils.restart_service(self._ssh, REPLICATOR_SVC_NAME) - def update_state(self, state, restart=False): + def update_state(self, state: list[Any], restart: bool = False) -> None: fd, state_file = tempfile.mkstemp() try: os.close(fd) @@ -657,7 +660,7 @@ def update_state(self, state, restart=False): utils.ignore_exceptions(os.remove)(state_file) @utils.retry_on_error(sleep_seconds=5) - def _get_ssh_client(self, args): + def _get_ssh_client(self, args: dict) -> paramiko.SSHClient: """ gets a paramiko SSH client """ @@ -670,7 +673,7 @@ def _get_ssh_client(self, args): banner_timeout=args.get("banner_timeout"), ) - def _parse_source_ssh_conn_info(self, conn_info): + def _parse_source_ssh_conn_info(self, conn_info: dict) -> dict: # if we get valid SSH connection info we can # use it to copy the binary, and potentially # create a SSH tunnel through which we will @@ -707,7 +710,7 @@ def _parse_source_ssh_conn_info(self, conn_info): } return args - def _get_replicator_state_file(self): + def _get_replicator_state_file(self) -> str: """ Looks for replicator state in volumes_info. If none is found, replicator will be initialized without it. @@ -721,7 +724,7 @@ def _get_replicator_state_file(self): json.dump(state, fp) return filename - def _parse_replicator_conn_info(self, conn_info): + def _parse_replicator_conn_info(self, conn_info: dict) -> dict: # The IP should be the same one as the SSH IP. # Only the port will differ, and that is configurable. ip = conn_info.get("ip", None) @@ -731,7 +734,9 @@ def _parse_replicator_conn_info(self, conn_info): } @utils.retry_on_error() - def _copy_file(self, ssh, localPath, remotePath): + def _copy_file( + self, ssh, localPath, remotePath + ) -> bool | bytes | list | paramiko.SSHClient | None: tmp = tempfile.mktemp(dir="/tmp") sftp = paramiko.SFTPClient.from_transport(ssh.get_transport()) @@ -739,12 +744,14 @@ def _copy_file(self, ssh, localPath, remotePath): utils.exec_ssh_cmd(ssh, "sudo mv %s %s" % (tmp, remotePath), get_pty=False) sftp.close() - def _copy_replicator_cmd(self, ssh): + def _copy_replicator_cmd(self, ssh: paramiko.SSHClient) -> None: local_path = os.path.join(utils.get_resources_bin_dir(), 'replicator') self._copy_file(ssh, local_path, REPLICATOR_PATH) utils.exec_ssh_cmd(ssh, "sudo chmod +x %s" % REPLICATOR_PATH, get_pty=False) - def _setup_replicator_group(self, ssh, group_name=REPLICATOR_GROUP_NAME): + def _setup_replicator_group( + self, ssh: paramiko.SSHClient, group_name: str = REPLICATOR_GROUP_NAME + ) -> bool: """Sets up a group with the given name and adds the user we're connected as to it. @@ -769,7 +776,7 @@ def _setup_replicator_group(self, ssh, group_name=REPLICATOR_GROUP_NAME): return int(group_exists) == 1 - def _setup_replicator_user(self, ssh): + def _setup_replicator_user(self, ssh: paramiko.SSHClient) -> None: # check for and create replicator user: user_exists = utils.exec_ssh_cmd( ssh, @@ -787,7 +794,9 @@ def _setup_replicator_user(self, ssh): ssh, "sudo usermod -aG disk %s" % REPLICATOR_USERNAME, get_pty=False ) - def _exec_replicator(self, ssh, port, certs, state_file): + def _exec_replicator( + self, ssh: paramiko.SSHClient, port: int, certs: dict[str, str], state_file: str + ) -> None: cmdline = ( "%(replicator_path)s run -hash-method=%(hash_method)s " "-ignore-mounted-disks=%(ignore_mounted)s " @@ -818,14 +827,18 @@ def _exec_replicator(self, ssh, port, certs, state_file): ssh, cmdline, REPLICATOR_SVC_NAME, run_as=REPLICATOR_USERNAME, start=True ) - def _fetch_remote_file(self, ssh, remote_file, local_file): + def _fetch_remote_file( + self, ssh: paramiko.SSHClient, remote_file: str, local_file: str + ) -> None: # TODO(gsamfira): make this re-usable with open(local_file, 'wb') as fd: data = utils.retry_on_error()(utils.read_ssh_file)(ssh, remote_file) fd.write(data) @utils.retry_on_error(sleep_seconds=5) - def _setup_certificates(self, ssh, args): + def _setup_certificates( + self, ssh: paramiko.SSHClient, args: dict + ) -> dict[str, dict]: # TODO(gsamfira): coriolis-replicator and coriolis-writer share # the functionality of being able to generate certificates # This will either be replaced with proper certificate management @@ -920,7 +933,7 @@ def _setup_certificates(self, ssh, args): }, } - def _change_binary_se_context(self, ssh): + def _change_binary_se_context(self, ssh: paramiko.SSHClient) -> None: cmd = "sudo chcon -t bin_t %s" % REPLICATOR_PATH try: utils.exec_ssh_cmd(ssh, cmd, get_pty=False) @@ -930,7 +943,7 @@ def _change_binary_se_context(self, ssh): utils.get_exception_details(), ) - def _setup_replicator(self, ssh): + def _setup_replicator(self, ssh: paramiko.SSHClient) -> dict[str, str]: # copy the binary, set up the service, generate certificates, # start service state_file = self._get_replicator_state_file() @@ -953,19 +966,23 @@ def _setup_replicator(self, ssh): self._exec_replicator(ssh, args["port"], certs["remote"], REPLICATOR_STATE) return certs["local"] - def _get_size_from_chunks(self, chunks): + def _get_size_from_chunks(self, chunks: list[dict]) -> float: ret = 0 for chunk in chunks: ret += chunk["length"] return ret / units.Mi - def _find_vol_state(self, name, state): + def _find_vol_state(self, name: str, state: list[dict]) -> dict: for vol in state: if vol["device-name"] == name: return vol return None - def _verify_disk_checksum(self, dev_name, destination): + def _verify_disk_checksum( + self, + dev_name: str, + destination, + ) -> None: """Compares source and destination checksums for a transferred disk. Must be called while the device is still acquired on the writer side. @@ -1008,8 +1025,11 @@ def _verify_disk_checksum(self, dev_name, destination): ) def replicate_disks( - self, source_volumes_info, backup_writer, verify_checksum=False - ): + self, + source_volumes_info: list[dict], + backup_writer, + verify_checksum: bool = False, + ) -> list[dict]: """ Fetch the block diff and send it to the backup_writer. If the target_is_zeroed parameter is set to True, on initial diff --git a/coriolis/regions/api.py b/coriolis/regions/api.py index b6e1fda89..8e69bb4a6 100644 --- a/coriolis/regions/api.py +++ b/coriolis/regions/api.py @@ -8,21 +8,32 @@ class API(object): def __init__(self): self._rpc_client = rpc_client.ConductorClient() - def create(self, ctxt, region_name, description, enabled=True): + def create( + self, + ctxt, + region_name: str, + description: str, + enabled: bool = True, + ): return self._rpc_client.create_region( ctxt, region_name, description=description, enabled=enabled ) - def update(self, ctxt, region_id, updated_values): + def update( + self, + ctxt, + region_id: str, + updated_values: dict[str, str], + ): return self._rpc_client.update_region( ctxt, region_id, updated_values=updated_values ) - def delete(self, ctxt, region_id): + def delete(self, ctxt, region_id: str) -> None: self._rpc_client.delete_region(ctxt, region_id) def get_regions(self, ctxt): return self._rpc_client.get_regions(ctxt) - def get_region(self, ctxt, region_id): + def get_region(self, ctxt, region_id: str): return self._rpc_client.get_region(ctxt, region_id) diff --git a/coriolis/rpc.py b/coriolis/rpc.py index f3a4d3dc4..7bb3cc09a 100644 --- a/coriolis/rpc.py +++ b/coriolis/rpc.py @@ -34,20 +34,28 @@ class RequestContextSerializer(messaging.Serializer): def __init__(self, base): self._base = base - def serialize_entity(self, ctxt, entity): + def serialize_entity( + self, + ctxt: context.RequestContext, + entity, + ): if not self._base: return entity return self._base.serialize_entity(ctxt, entity) - def deserialize_entity(self, ctxt, entity): + def deserialize_entity( + self, + ctxt: context.RequestContext, + entity, + ): if not self._base: return entity return self._base.deserialize_entity(ctxt, entity) - def serialize_context(self, ctxt): + def serialize_context(self, ctxt: context.RequestContext) -> dict: return ctxt.to_dict() - def deserialize_context(self, ctxt): + def deserialize_context(self, ctxt: dict) -> context.RequestContext: return context.RequestContext.from_dict(ctxt) @@ -57,7 +65,11 @@ def _get_transport(): ) -def get_server(target, endpoints, serializer=None): +def get_server( + target: messaging.Target, + endpoints: list, + serializer: RequestContextSerializer | None = None, +): serializer = RequestContextSerializer(serializer) return messaging.get_rpc_server( _get_transport(), target, endpoints, executor='threading', serializer=serializer @@ -76,7 +88,9 @@ class BaseRPCClient(object): instantiates and cleans up transports for each call. """ - def __init__(self, target, timeout=None, serializer=None): + def __init__( + self, target: messaging.Target, timeout: int | None = None, serializer=None + ): self._target = target self._timeout = timeout if self._timeout is None: @@ -97,7 +111,7 @@ def _transport(self): else: return _TRANSPORT - def _rpc_client(self): + def _rpc_client(self) -> messaging.RPCClient: return messaging.RPCClient( self._transport, self._target, @@ -105,7 +119,7 @@ def _rpc_client(self): timeout=self._timeout, ) - def _call(self, ctxt, method, **kwargs): + def _call(self, ctxt: context.RequestContext, method: str, **kwargs): client = self._rpc_client() return client.call(ctxt, method, **kwargs) @@ -114,7 +128,7 @@ def _call_on_host(self, host, ctxt, method, **kwargs): cctxt = client.prepare(server=host) return cctxt.call(ctxt, method, **kwargs) - def _cast(self, ctxt, method, **kwargs): + def _cast(self, ctxt: context.RequestContext, method: str, **kwargs) -> None: client = self._rpc_client() client.cast(ctxt, method, **kwargs) diff --git a/coriolis/scheduler/filters/base.py b/coriolis/scheduler/filters/base.py index 2cbfe5cf5..a7556bd33 100644 --- a/coriolis/scheduler/filters/base.py +++ b/coriolis/scheduler/filters/base.py @@ -7,13 +7,13 @@ class BaseServiceFilter(with_metaclass(abc.ABCMeta)): - def is_service_acceptable(self, service): + def is_service_acceptable(self, service) -> bool: return self.rate_service(service) > 0 def filter_services(self, services): return [service for service in services if self.is_service_acceptable(service)] @abc.abstractmethod - def rate_service(self, service): + def rate_service(self, service) -> int: """Returns a rating out of 100 for the service.""" pass diff --git a/coriolis/scheduler/filters/trivial_filters.py b/coriolis/scheduler/filters/trivial_filters.py index 4acdcc1e3..8165e80a1 100644 --- a/coriolis/scheduler/filters/trivial_filters.py +++ b/coriolis/scheduler/filters/trivial_filters.py @@ -9,18 +9,18 @@ class RegionsFilter(base.BaseServiceFilter): - def __init__(self, regions, any_region=False): + def __init__(self, regions: list[str], any_region: bool = False): self._regions = regions self._any_region = any_region - def __repr__(self): + def __repr__(self) -> str: return "<%s(regions=%s, any_region=%s)>" % ( self.__class__.__name__, self._regions, self._any_region, ) - def rate_service(self, service): + def rate_service(self, service) -> int: if not self._regions: LOG.debug( "No regions specified for this filter (%s). Presuming service is valid." @@ -56,46 +56,46 @@ def rate_service(self, service): class TopicFilter(base.BaseServiceFilter): - def __init__(self, topic): + def __init__(self, topic: str): self._topic = topic def __repr__(self): return "<%s(topic=%s)>" % (self.__class__.__name__, self._topic) - def rate_service(self, service): + def rate_service(self, service) -> int: if service.topic == self._topic: return 100 return 0 class EnabledFilter(base.BaseServiceFilter): - def __init__(self, enabled=True): + def __init__(self, enabled: bool = True): self._enabled = enabled - def __repr__(self): + def __repr__(self) -> str: return "<%s(enabled=%s)>" % (self.__class__.__name__, self._enabled) - def rate_service(self, service): + def rate_service(self, service) -> int: if service.enabled == self._enabled: return 100 return 0 class ProviderTypesFilter(base.BaseServiceFilter): - def __init__(self, provider_requirements): + def __init__(self, provider_requirements: dict[str, list]): """Filters based on requested provider capabilities. :param provider_requirements: dict of the form { "": [constants.PROVIDER_TYPE_*, ...]} """ self._provider_requirements = provider_requirements - def __repr__(self): + def __repr__(self) -> str: return "<%s(provider_requirements=%s)>" % ( self.__class__.__name__, self._provider_requirements, ) - def rate_service(self, service): + def rate_service(self, service) -> int: for platform_type in self._provider_requirements: if platform_type not in service.providers: LOG.debug( diff --git a/coriolis/scheduler/rpc/client.py b/coriolis/scheduler/rpc/client.py index e66b3b42d..5fbc4da31 100644 --- a/coriolis/scheduler/rpc/client.py +++ b/coriolis/scheduler/rpc/client.py @@ -32,11 +32,15 @@ def __init__(self, timeout=None): timeout = CONF.scheduler.scheduler_rpc_timeout super(SchedulerClient, self).__init__(target, timeout=timeout) - def get_diagnostics(self, ctxt): + def get_diagnostics(self, ctxt) -> dict: return self._call(ctxt, 'get_diagnostics') def get_workers_for_specs( - self, ctxt, provider_requirements=None, region_sets=None, enabled=None + self, + ctxt, + provider_requirements: dict | None = None, + region_sets: list | None = None, + enabled: bool | None = None, ): return self._call( ctxt, @@ -46,7 +50,12 @@ def get_workers_for_specs( provider_requirements=provider_requirements, ) - def get_any_worker_service(self, ctxt, random_choice=False, raise_if_none=True): + def get_any_worker_service( + self, + ctxt, + random_choice: bool = False, + raise_if_none: bool = True, + ): services = self.get_workers_for_specs(ctxt) if not services: if raise_if_none: @@ -63,11 +72,11 @@ def get_any_worker_service(self, ctxt, random_choice=False, raise_if_none=True): def get_worker_service_for_specs( self, ctxt, - provider_requirements=None, - region_sets=None, - enabled=True, - random_choice=False, - raise_on_no_matches=True, + provider_requirements: dict | None = None, + region_sets: list | None = None, + enabled: bool = True, + random_choice: bool = False, + raise_on_no_matches: bool = True, ): """Utility method which ensures at least one service matching the provided requirements exists and is usable. @@ -113,12 +122,12 @@ def get_worker_service_for_specs( def get_worker_service_for_task( self, ctxt, - task, + task: dict[str, str], origin_endpoint, destination_endpoint, - retry_count=5, - retry_period=2, - random_choice=True, + retry_count: int = 5, + retry_period: int = 2, + random_choice: bool = True, ): """Gets a worker service for the task with the given properties and source/target endpoints. diff --git a/coriolis/scheduler/rpc/server.py b/coriolis/scheduler/rpc/server.py index 27b5f68b5..ffcd7f8f1 100644 --- a/coriolis/scheduler/rpc/server.py +++ b/coriolis/scheduler/rpc/server.py @@ -24,7 +24,7 @@ class SchedulerServerEndpoint(object): def __init__(self): self._rpc_conductor_client = rpc_conductor_client.ConductorClient() - def get_diagnostics(self, ctxt): + def get_diagnostics(self, ctxt) -> dict: return utils.get_diagnostics_info() def _get_all_worker_services(self, ctxt): @@ -38,8 +38,11 @@ def _get_all_worker_services(self, ctxt): return services def _get_weighted_filtered_services( - self, services, filters, minimum_per_filter_rating=1 - ): + self, + services, + filters: list, + minimum_per_filter_rating: int = 1, + ) -> list[tuple]: """Returns list of services and their scores for the given filters. Services which are rejected by any filter will be excluded. """ @@ -92,7 +95,12 @@ def _get_weighted_filtered_services( return sorted(scores, key=lambda s: s[1], reverse=True) def _filter_regions( - self, ctxt, region_ids, enabled=True, check_all_exist=True, regions_cache=None + self, + ctxt, + region_ids: list[str], + enabled: bool = True, + check_all_exist: bool = True, + regions_cache=None, ): found_regions = [] filtered_regions = [] @@ -120,10 +128,10 @@ def _filter_regions( def get_workers_for_specs( self, ctxt, - provider_requirements=None, - region_sets=None, - enabled=None, - filter_disabled_regions=True, + provider_requirements: dict | None = None, + region_sets: list | None = None, + enabled: bool | None = None, + filter_disabled_regions: bool = True, ): """Returns a list of enabled Worker Services with the specified parameters. diff --git a/coriolis/schemas.py b/coriolis/schemas.py index 3163b86cb..698615540 100644 --- a/coriolis/schemas.py +++ b/coriolis/schemas.py @@ -50,7 +50,11 @@ ) -def get_schema(package_name, schema_name, schemas_directory=DEFAULT_SCHEMAS_DIRECTORY): +def get_schema( + package_name: str, + schema_name: str, + schemas_directory: str = DEFAULT_SCHEMAS_DIRECTORY, +) -> dict: """Loads the schema using jinja2 template loading. Loads the schema with the given 'schema_name' using jinja2 template @@ -88,7 +92,9 @@ def get_schema(package_name, schema_name, schemas_directory=DEFAULT_SCHEMAS_DIRE return schema -def validate_value(val, schema, format_checker=None, raise_on_error=True): +def validate_value( + val: dict | list, schema: dict, format_checker=None, raise_on_error: bool = True +) -> bool: """Simple wrapper for jsonschema.validate for usability. NOTE: silently passes empty schemas. diff --git a/coriolis/service.py b/coriolis/service.py index 9c54d3937..8edcee4c9 100644 --- a/coriolis/service.py +++ b/coriolis/service.py @@ -170,7 +170,14 @@ def reset(self): class MessagingService(service.ServiceBase): - def __init__(self, topic, endpoints, version, worker_count=None, init_rpc=True): + def __init__( + self, + topic: str, + endpoints: list, + version: str, + worker_count: int = None, + init_rpc: bool = True, + ): if init_rpc: rpc.init() target = messaging.Target( @@ -189,7 +196,7 @@ def __init__(self, topic, endpoints, version, worker_count=None, init_rpc=True): def get_workers_count(self): return self._workers - def start(self): + def start(self) -> None: self._server.start() def stop(self): diff --git a/coriolis/services/api.py b/coriolis/services/api.py index 6e326088a..ab024d7f9 100644 --- a/coriolis/services/api.py +++ b/coriolis/services/api.py @@ -8,19 +8,32 @@ class API(object): def __init__(self): self._rpc_client = rpc_client.ConductorClient() - def create(self, ctxt, host, binary, topic, mapped_regions, enabled): + def create( + self, + ctxt, + host: str, + binary: str, + topic: str, + mapped_regions: list[str], + enabled: bool, + ): return self._rpc_client.register_service( ctxt, host, binary, topic, enabled, mapped_regions ) - def update(self, ctxt, service_id, updated_values): + def update( + self, + ctxt, + service_id: str, + updated_values: dict, + ): return self._rpc_client.update_service(ctxt, service_id, updated_values) - def delete(self, ctxt, region_id): + def delete(self, ctxt, region_id: str) -> None: self._rpc_client.delete_service(ctxt, region_id) def get_services(self, ctxt): return self._rpc_client.get_services(ctxt) - def get_service(self, ctxt, service_id): + def get_service(self, ctxt, service_id: str): return self._rpc_client.get_service(ctxt, service_id) diff --git a/coriolis/taskflow/base.py b/coriolis/taskflow/base.py index 8e98baa99..95538b892 100644 --- a/coriolis/taskflow/base.py +++ b/coriolis/taskflow/base.py @@ -29,7 +29,9 @@ class BaseCoriolisTaskflowTask(taskflow_tasks.Task): """Base class for all TaskFlow tasks within Coriolis.""" - def _get_error_str_for_flow_failures(self, flow_failures, full_tracebacks=True): + def _get_error_str_for_flow_failures( + self, flow_failures: dict[str, failure.Failure], full_tracebacks: bool = True + ) -> str: if not flow_failures: return "No flow failures provided." @@ -61,7 +63,7 @@ def _get_error_str_for_flow_failures(self, flow_failures, full_tracebacks=True): return res - def revert(self, *args, **kwargs): + def revert(self, *args, **kwargs) -> None: result = kwargs.get('result') if isinstance(result, failure.Failure): # it means that this is the task which error'd out: @@ -106,13 +108,13 @@ class BaseRunWorkerTask(BaseCoriolisTaskflowTask): def __init__( self, - task_name, - task_id, - task_instance, - main_task_runner_type, - cleanup_task_runner_type=None, + task_name: str, + task_id: str, + task_instance: str, + main_task_runner_type: str, + cleanup_task_runner_type: str | None = None, depends_on=None, - raise_on_cleanup_failure=False, + raise_on_cleanup_failure: bool = False, **kwargs, ): self._task_id = task_id @@ -126,7 +128,7 @@ def __init__( super(BaseRunWorkerTask, self).__init__(name=task_name, **kwargs) @property - def _scheduler_client(self): + def _scheduler_client(self) -> rpc_scheduler_client.SchedulerClient: if not getattr(self, '_scheduler_client_instance', None): self._scheduler_client_instance = rpc_scheduler_client.SchedulerClient() return self._scheduler_client_instance @@ -134,14 +136,14 @@ def _scheduler_client(self): def _get_worker_service_rpc_for_task( self, ctxt, - task_id, - task_type, - origin, - destination, - retry_count=5, - retry_period=2, - rpc_timeout=CONF.taskflow.worker_task_execution_timeout, - ): + task_id: str, + task_type: str, + origin: dict, + destination: dict, + retry_count: int = 5, + retry_period: int = 2, + rpc_timeout: int = CONF.taskflow.worker_task_execution_timeout, + ) -> rpc_worker_client.WorkerClient: task_info = {"id": task_id, "task_type": task_type} worker_service = self._scheduler_client.get_worker_service_for_task( ctxt, @@ -164,7 +166,15 @@ def _get_worker_service_rpc_for_task( worker_service, timeout=rpc_timeout ) - def _execute_task(self, ctxt, task_id, task_type, origin, destination, task_info): + def _execute_task( + self, + ctxt, + task_id: str, + task_type: str, + origin: dict, + destination: dict, + task_info: dict, + ) -> dict: worker_rpc = self._get_worker_service_rpc_for_task( ctxt, self._task_id, task_type, origin, destination ) @@ -200,7 +210,13 @@ def _execute_task(self, ctxt, task_id, task_type, origin, destination, task_info ) raise - def execute(self, context, origin, destination, task_info): + def execute( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + ) -> dict | None: res = self._execute_task( context, self._task_id, @@ -211,7 +227,14 @@ def execute(self, context, origin, destination, task_info): ) return res - def revert(self, context, origin, destination, task_info, **kwargs): + def revert( + self, + context, + origin: dict, + destination: dict, + task_info: dict, + **kwargs, + ) -> None: super(BaseRunWorkerTask, self).revert( context, origin, destination, task_info, **kwargs ) diff --git a/coriolis/taskflow/runner.py b/coriolis/taskflow/runner.py index ca953db25..a2cbfedd4 100644 --- a/coriolis/taskflow/runner.py +++ b/coriolis/taskflow/runner.py @@ -26,10 +26,10 @@ class TaskFlowRunner(object): def __init__( self, - service_name, - execution_order=TASKFLOW_EXECUTION_ORDER_PARALLEL, - executor=TASKFLOW_EXECUTOR_THREADED, - max_workers=1, + service_name: str, + execution_order: str = TASKFLOW_EXECUTION_ORDER_PARALLEL, + executor: str = TASKFLOW_EXECUTOR_THREADED, + max_workers: int = 1, ): self._service_name = service_name @@ -37,7 +37,7 @@ def __init__( self._executor = executor self._max_workers = max_workers - def _log_flow_transition(self, state, details): + def _log_flow_transition(self, state: str, details: dict) -> None: LOG.debug( "[TaskFlowRunner(%s)] Flow '%s' (internal UUID '%s') transitioned" " from '%s' state to '%s'", @@ -48,7 +48,7 @@ def _log_flow_transition(self, state, details): state, ) - def _log_task_transition(self, state, details): + def _log_task_transition(self, state: str, details: dict) -> None: LOG.debug( "[TaskFlowRunner(%s)] Task '%s' (internal UUID '%s') transitioned" " from '%s' state to '%s'", @@ -59,7 +59,7 @@ def _log_task_transition(self, state, details): state, ) - def _setup_engine_for_flow(self, flow, store=None): + def _setup_engine_for_flow(self, flow, store: dict | None = None): engine = engines.load( flow, store, @@ -71,7 +71,7 @@ def _setup_engine_for_flow(self, flow, store=None): engine.atom_notifier.register(notifier.Notifier.ANY, self._log_task_transition) return engine - def _run_flow(self, flow, store=None): + def _run_flow(self, flow, store: dict = None) -> None: LOG.debug("Ramping up to run flow with name '%s'", flow.name) engine = self._setup_engine_for_flow(flow, store=store) diff --git a/coriolis/taskflow/utils.py b/coriolis/taskflow/utils.py index 9c47e7f68..13f300055 100644 --- a/coriolis/taskflow/utils.py +++ b/coriolis/taskflow/utils.py @@ -9,10 +9,10 @@ class DummyDecider(object): """A callable to decide execution in a pre-defined manner.""" - def __init__(self, allow=True): + def __init__(self, allow: bool = True): self._allow = allow - def __call__(self, history): + def __call__(self, history: dict[str, dict]) -> bool: LOG.debug( "Dummy decider returning '%s'. Provided task history was: %s", self._allow, diff --git a/coriolis/tasks/base.py b/coriolis/tasks/base.py index f5c113491..4060c9e19 100644 --- a/coriolis/tasks/base.py +++ b/coriolis/tasks/base.py @@ -92,7 +92,15 @@ def get_required_platform(cls): ) @abc.abstractmethod - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict: """The actual logic run by the task. Should return a dict with all the fields declared by 'self.get_returned_task_info_properties'. @@ -103,7 +111,15 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): % (self.__class__) ) - def run(self, ctxt, instance, origin, destination, task_info, event_handler): + def run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict: """Runs the task with the given params and returns a dict with the results. NOTE: This should NOT modify the existing task_info in any way. @@ -169,12 +185,14 @@ def get_shared_lib_dirs_for_provider(ctxt, endpoint, event_handler): return [] -def get_connection_info(ctxt, data): +def get_connection_info(ctxt, data: dict) -> dict[str, str]: connection_info = data.get("connection_info") or {} return utils.get_secret_connection_info(ctxt, connection_info) -def marshal_migr_conn_info(migr_connection_info, private_key_field_name="pkey"): +def marshal_migr_conn_info( + migr_connection_info: dict, private_key_field_name: str = "pkey" +) -> dict: if migr_connection_info and (private_key_field_name in migr_connection_info): migr_connection_info = migr_connection_info.copy() pkey = migr_connection_info[private_key_field_name] @@ -185,7 +203,9 @@ def marshal_migr_conn_info(migr_connection_info, private_key_field_name="pkey"): return migr_connection_info -def unmarshal_migr_conn_info(migr_connection_info, private_key_field_name="pkey"): +def unmarshal_migr_conn_info( + migr_connection_info: dict, private_key_field_name: str = "pkey" +) -> dict: if migr_connection_info and (private_key_field_name in migr_connection_info): migr_connection_info = migr_connection_info.copy() pkey_str = migr_connection_info[private_key_field_name] diff --git a/coriolis/tasks/factory.py b/coriolis/tasks/factory.py index 601b6c109..c29bf6870 100644 --- a/coriolis/tasks/factory.py +++ b/coriolis/tasks/factory.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from coriolis import constants, exception from coriolis.tasks import ( migration_tasks, @@ -67,7 +69,7 @@ } -def get_task_runner_class(task_type): +def get_task_runner_class(task_type: str) -> type[Any]: cls = _TASKS_MAP.get(task_type) if not cls: raise exception.NotFound("TaskRunner not found for task type: %s" % task_type) diff --git a/coriolis/tasks/minion_pool_tasks.py b/coriolis/tasks/minion_pool_tasks.py index e91cad855..4b455d243 100644 --- a/coriolis/tasks/minion_pool_tasks.py +++ b/coriolis/tasks/minion_pool_tasks.py @@ -1,6 +1,8 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any, Callable + from oslo_log import log as logging from coriolis import constants, events, exception @@ -26,7 +28,9 @@ } -def _get_required_minion_pool_provider_types_for_platform(platform_type): +def _get_required_minion_pool_provider_types_for_platform( + platform_type: str, +) -> dict[str, list]: provider_type = None if platform_type == constants.PROVIDER_PLATFORM_SOURCE: provider_type = constants.PROVIDER_TYPE_SOURCE_MINION_POOL @@ -40,7 +44,9 @@ def _get_required_minion_pool_provider_types_for_platform(platform_type): return {platform_type: [provider_type]} -def _get_platform_to_target(required_platform, origin, destination): +def _get_platform_to_target( + required_platform: str, origin: dict, destination: dict +) -> dict: platform_to_target = None if required_platform == constants.TASK_PLATFORM_SOURCE: platform_to_target = origin @@ -54,8 +60,8 @@ def _get_platform_to_target(required_platform, origin, destination): def _check_missing_minion_properties( - provider_type, minion_properties, properties_to_check=None -): + provider_type: int, minion_properties: dict, properties_to_check: list | None = None +) -> None: if properties_to_check is None: properties_to_check = [] missing = [key for key in properties_to_check if key not in minion_properties] @@ -69,7 +75,7 @@ def _check_missing_minion_properties( ) -def _get_minion_conn_info(minion_properties): +def _get_minion_conn_info(minion_properties: dict) -> dict: minion_connection_info = {} if 'connection_info' in minion_properties: minion_connection_info = base.marshal_migr_conn_info( @@ -78,7 +84,7 @@ def _get_minion_conn_info(minion_properties): return minion_connection_info -def _get_minion_backup_writer_conn_info(minion_properties): +def _get_minion_backup_writer_conn_info(minion_properties: dict) -> dict: minion_backup_writer_conn = {} if 'backup_writer_connection_info' in minion_properties: minion_backup_writer_conn = minion_properties['backup_writer_connection_info'] @@ -99,15 +105,15 @@ def get_required_platform(cls): ) @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["pool_environment_options"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list: return [] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return _get_required_minion_pool_provider_types_for_platform( cls.get_required_platform() ) @@ -115,12 +121,12 @@ def get_required_provider_types(cls): def _run( self, ctxt, - minion_pool_machine_id, - origin, - destination, - task_info, + minion_pool_machine_id: str, + origin: dict, + destination: dict, + task_info: dict, event_handler, - ): + ) -> dict[Any, Any]: # NOTE: both origin or target endpoints would work: connection_info = base.get_connection_info(ctxt, destination) @@ -141,13 +147,13 @@ def _run( class ValidateSourceMinionPoolOptionsTask(_BaseValidateMinionPoolOptionsTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_SOURCE class ValidateDestinationMinionPoolOptionsTask(_BaseValidateMinionPoolOptionsTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_DESTINATION @@ -159,7 +165,7 @@ def get_required_platform(cls): ) @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return [ "pool_environment_options", "pool_shared_resources", @@ -168,7 +174,7 @@ def get_required_task_info_properties(cls): ] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return [ "minion_provider_properties", "minion_connection_info", @@ -176,7 +182,7 @@ def get_returned_task_info_properties(cls): ] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return _get_required_minion_pool_provider_types_for_platform( cls.get_required_platform() ) @@ -184,12 +190,12 @@ def get_required_provider_types(cls): def _run( self, ctxt, - minion_pool_machine_id, - origin, - destination, - task_info, + minion_pool_machine_id: str, + origin: dict, + destination: dict, + task_info: dict, event_handler, - ): + ) -> dict: # NOTE: both origin or target endpoints would work: connection_info = base.get_connection_info(ctxt, destination) @@ -236,13 +242,13 @@ def _run( class CreateSourceMinionMachineTask(_BaseCreateMinionMachineTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_SOURCE class CreateDestinationMinionMachineTask(_BaseCreateMinionMachineTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_DESTINATION @@ -252,15 +258,15 @@ def get_required_platform(cls): raise NotImplementedError("No minion deletion required platform specified.") @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["minion_provider_properties"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list: return [] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return _get_required_minion_pool_provider_types_for_platform( cls.get_required_platform() ) @@ -268,12 +274,12 @@ def get_required_provider_types(cls): def _run( self, ctxt, - minion_pool_machine_id, - origin, - destination, - task_info, + minion_pool_machine_id: str, + origin: dict, + destination: dict, + task_info: dict, event_handler, - ): + ) -> dict[Any, Any]: # NOTE: both origin or target endpoints would work: connection_info = base.get_connection_info(ctxt, destination) @@ -292,13 +298,13 @@ def _run( class DeleteSourceMinionMachineTask(_BaseDeleteMinionMachineTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_SOURCE class DeleteDestinationMinionMachineTask(_BaseDeleteMinionMachineTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_DESTINATION @@ -310,15 +316,15 @@ def get_required_platform(cls): ) @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["pool_environment_options", "pool_identifier"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["pool_shared_resources"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return _get_required_minion_pool_provider_types_for_platform( cls.get_required_platform() ) @@ -326,12 +332,12 @@ def get_required_provider_types(cls): def _run( self, ctxt, - minion_pool_machine_id, - origin, - destination, - task_info, + minion_pool_machine_id: str, + origin: dict, + destination: dict, + task_info: dict, event_handler, - ): + ) -> dict[str, dict]: # NOTE: both origin or target endpoints would work: connection_info = base.get_connection_info(ctxt, destination) @@ -353,13 +359,13 @@ def _run( class SetUpSourcePoolSupportingResourcesTask(_BaseSetUpPoolSupportingResourcesTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_SOURCE class SetUpDestinationPoolSupportingResources(_BaseSetUpPoolSupportingResourcesTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_DESTINATION @@ -371,15 +377,15 @@ def get_required_platform(cls): ) @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["pool_environment_options", "pool_shared_resources"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["pool_shared_resources"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return _get_required_minion_pool_provider_types_for_platform( cls.get_required_platform() ) @@ -387,12 +393,12 @@ def get_required_provider_types(cls): def _run( self, ctxt, - minion_pool_machine_id, - origin, - destination, - task_info, + minion_pool_machine_id: str, + origin: dict, + destination: dict, + task_info: dict, event_handler, - ): + ) -> dict[str, None]: # NOTE: both origin or target endpoints would work: connection_info = base.get_connection_info(ctxt, destination) @@ -416,7 +422,7 @@ class TearDownSourcePoolSupportingResourcesTask( _BaseTearDownPoolSupportingResourcesTask ): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_SOURCE @@ -424,7 +430,7 @@ class TearDownDestinationPoolSupportingResources( _BaseTearDownPoolSupportingResourcesTask ): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_DESTINATION @@ -439,18 +445,18 @@ def get_required_platform(cls): raise NotImplementedError("No minion disk attachment platform specified") @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: fields = list(cls._get_minion_task_info_field_mappings().keys()) return fields @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: fields = list(cls._get_minion_task_info_field_mappings().values()) fields.append(cls._get_minion_properties_task_info_field()) return fields @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return _get_required_minion_pool_provider_types_for_platform( cls.get_required_platform() ) @@ -481,7 +487,9 @@ def _get_provider_disk_operation(cls, provider): def _get_minion_task_info_field_mappings(cls): raise NotImplementedError("No minion task info field mappings provided.") - def _check_missing_disk_op_result_keys(self, platform_to_target, disk_op_res): + def _check_missing_disk_op_result_keys( + self, platform_to_target: dict, disk_op_res: dict + ) -> None: missing_result_props = [ prop for prop in ["volumes_info", "minion_properties"] @@ -498,7 +506,15 @@ def _check_missing_disk_op_result_keys(self, platform_to_target, disk_op_res): ) ) - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict: platform_to_target = _get_platform_to_target( self.get_required_platform(), origin, destination @@ -548,11 +564,11 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class _BaseAttachVolumesToTransferMinionTask(_BaseVolumesMinionMachineAttachmentTask): @classmethod - def _get_volumes_info_from_task_info(cls, task_info): + def _get_volumes_info_from_task_info(cls, task_info: dict) -> list[dict]: return task_info["volumes_info"] @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: fields = super( _BaseAttachVolumesToTransferMinionTask, cls ).get_required_task_info_properties() @@ -560,7 +576,7 @@ def get_required_task_info_properties(cls): return fields @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: fields = super( _BaseAttachVolumesToTransferMinionTask, cls ).get_returned_task_info_properties() @@ -602,15 +618,15 @@ def _get_provider_disk_operation(cls, provider): class AttachVolumesToDestinationMinionTask(_BaseAttachVolumesToTransferMinionTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def _get_minion_properties_task_info_field(cls): + def _get_minion_properties_task_info_field(cls) -> str: return "destination_minion_provider_properties" @classmethod - def _get_minion_connection_info_task_info_field(cls): + def _get_minion_connection_info_task_info_field(cls) -> str: return "destination_minion_connection_info" @classmethod @@ -618,19 +634,19 @@ def _get_provider_disk_operation(cls, provider): return provider.attach_volumes_to_minion @classmethod - def _get_minion_task_info_field_mappings(cls): + def _get_minion_task_info_field_mappings(cls) -> dict[str, str]: return TARGET_MINION_TASK_INFO_FIELD_MAPPINGS class DetachVolumesFromDestinationMinionTask(AttachVolumesToDestinationMinionTask): @classmethod - def _get_provider_disk_operation(cls, provider): + def _get_provider_disk_operation(cls, provider) -> Callable: return provider.detach_volumes_from_minion class AttachVolumesToOSMorphingMinionTask(_BaseVolumesMinionMachineAttachmentTask): @classmethod - def _get_volumes_info_from_task_info(cls, task_info): + def _get_volumes_info_from_task_info(cls, task_info: dict) -> list[dict]: # Similar to _BaseAttachVolumesToTransferMinionTask. if "volumes_info" in task_info["instance_deployment_info"]: # Use the updated volumes info reported by the provider. @@ -640,7 +656,7 @@ def _get_volumes_info_from_task_info(cls, task_info): return task_info["volumes_info"] @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: fields = super( AttachVolumesToOSMorphingMinionTask, cls ).get_required_task_info_properties() @@ -649,7 +665,7 @@ def get_required_task_info_properties(cls): return fields @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: fields = super( AttachVolumesToOSMorphingMinionTask, cls ).get_returned_task_info_properties() @@ -657,30 +673,38 @@ def get_returned_task_info_properties(cls): return fields @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def _get_minion_properties_task_info_field(cls): + def _get_minion_properties_task_info_field(cls) -> str: return "osmorphing_minion_provider_properties" @classmethod - def _get_minion_connection_info_task_info_field(cls): + def _get_minion_connection_info_task_info_field(cls) -> str: return "osmorphing_minion_connection_info" @classmethod - def _get_provider_disk_operation(cls, provider): + def _get_provider_disk_operation(cls, provider) -> Callable: return provider.attach_volumes_to_minion @classmethod - def _get_minion_task_info_field_mappings(cls): + def _get_minion_task_info_field_mappings(cls) -> dict[str, str]: return OSMOPRHING_MINION_TASK_INFO_FIELD_MAPPINGS @classmethod def _clear_mapped_minion_task_info_field(cls): return False - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict: res = super(AttachVolumesToOSMorphingMinionTask, self)._run( ctxt, instance, origin, destination, task_info, event_handler ) @@ -696,7 +720,7 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DetachVolumesFromOSMorphingMinionTask(AttachVolumesToOSMorphingMinionTask): @classmethod - def _get_provider_disk_operation(cls, provider): + def _get_provider_disk_operation(cls, provider) -> Callable: return provider.detach_volumes_from_minion @classmethod @@ -717,7 +741,7 @@ def get_required_platform(cls): raise NotImplementedError("No minion validation platform specified") @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: base_props = set( [ "export_info", @@ -731,11 +755,11 @@ def get_required_task_info_properties(cls): return list(base_props) @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return list(cls._get_minion_task_info_field_mappings().values()) @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return _get_required_minion_pool_provider_types_for_platform( cls.get_required_platform() ) @@ -747,7 +771,7 @@ def _get_provider_pool_validation_operation(cls, provider): ) @classmethod - def _get_transfer_properties_task_info_field(cls): + def _get_transfer_properties_task_info_field(cls) -> str: platform = cls.get_required_platform() if platform == constants.PROVIDER_PLATFORM_SOURCE: return "source_environment" @@ -765,7 +789,15 @@ def _get_minion_properties_task_info_field(cls): def _get_minion_task_info_field_mappings(cls): raise NotImplementedError("No minion task info field mappings provided.") - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict: platform_to_target = _get_platform_to_target( self.get_required_platform(), origin, destination @@ -792,55 +824,55 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class ValidateSourceMinionCompatibilityTask(_BaseValidateMinionCompatibilityTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_SOURCE @classmethod - def _get_minion_properties_task_info_field(cls): + def _get_minion_properties_task_info_field(cls) -> str: return "origin_minion_provider_properties" @classmethod - def _get_provider_pool_validation_operation(cls, provider): + def _get_provider_pool_validation_operation(cls, provider) -> Callable: return provider.validate_minion_compatibility_for_transfer @classmethod - def _get_minion_task_info_field_mappings(cls): + def _get_minion_task_info_field_mappings(cls) -> dict[str, str]: return SOURCE_MINION_TASK_INFO_FIELD_MAPPINGS class ValidateDestinationMinionCompatibilityTask(_BaseValidateMinionCompatibilityTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_DESTINATION @classmethod - def _get_minion_properties_task_info_field(cls): + def _get_minion_properties_task_info_field(cls) -> str: return "destination_minion_provider_properties" @classmethod - def _get_provider_pool_validation_operation(cls, provider): + def _get_provider_pool_validation_operation(cls, provider) -> Callable: return provider.validate_minion_compatibility_for_transfer @classmethod - def _get_minion_task_info_field_mappings(cls): + def _get_minion_task_info_field_mappings(cls) -> dict[str, str]: return TARGET_MINION_TASK_INFO_FIELD_MAPPINGS class ValidateOSMorphingMinionCompatibilityTask(_BaseValidateMinionCompatibilityTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_DESTINATION @classmethod - def _get_minion_properties_task_info_field(cls): + def _get_minion_properties_task_info_field(cls) -> str: return "osmorphing_minion_provider_properties" @classmethod - def _get_provider_pool_validation_operation(cls, provider): + def _get_provider_pool_validation_operation(cls, provider) -> Callable: return provider.validate_osmorphing_minion_compatibility_for_transfer @classmethod - def _get_minion_task_info_field_mappings(cls): + def _get_minion_task_info_field_mappings(cls) -> dict[str, str]: return OSMOPRHING_MINION_TASK_INFO_FIELD_MAPPINGS @@ -854,16 +886,16 @@ def get_required_platform(cls): raise NotImplementedError("No minion releasing platform specified") @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: prop_mappings = cls._get_minion_task_info_field_mappings() return list(set(prop_mappings.keys()).union(prop_mappings.values())) @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return cls.get_required_task_info_properties() @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return _get_required_minion_pool_provider_types_for_platform( cls.get_required_platform() ) @@ -872,7 +904,15 @@ def get_required_provider_types(cls): def _get_minion_task_info_field_mappings(cls): raise NotImplementedError("No minion task info field mappings provided.") - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, None]: event_manager = events.EventManager(event_handler) event_manager.progress_update("Releasing minion machine") return {field: None for field in self.get_returned_task_info_properties()} @@ -880,54 +920,62 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class ReleaseSourceMinionTask(_BaseReleaseMinionTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_SOURCE @classmethod - def _get_minion_task_info_field_mappings(cls): + def _get_minion_task_info_field_mappings(cls) -> dict[str, str]: return SOURCE_MINION_TASK_INFO_FIELD_MAPPINGS class ReleaseDestinationMinionTask(_BaseReleaseMinionTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_DESTINATION @classmethod - def _get_minion_task_info_field_mappings(cls): + def _get_minion_task_info_field_mappings(cls) -> dict[str, str]: return TARGET_MINION_TASK_INFO_FIELD_MAPPINGS class ReleaseOSMorphingMinionTask(_BaseReleaseMinionTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.PROVIDER_PLATFORM_DESTINATION @classmethod - def _get_minion_task_info_field_mappings(cls): + def _get_minion_task_info_field_mappings(cls) -> dict[str, str]: return OSMOPRHING_MINION_TASK_INFO_FIELD_MAPPINGS class CollectOSMorphingInfoTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return _get_required_minion_pool_provider_types_for_platform( cls.get_required_platform() ) @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["target_environment", "instance_deployment_info"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["osmorphing_info"] - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, dict]: provider_type = self.get_required_provider_types()[ self.get_required_platform() ][0] @@ -959,20 +1007,28 @@ def get_required_platform(cls): raise NotImplementedError("No minion healthcheck platform specified") @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["minion_provider_properties", "minion_connection_info"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list: return [] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return _get_required_minion_pool_provider_types_for_platform( cls.get_required_platform() ) - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[Any, Any]: platform_to_target = _get_platform_to_target( self.get_required_platform(), origin, destination @@ -1005,7 +1061,7 @@ def get_required_platform(cls): class HealthcheckDestinationMinionTask(_BaseHealthcheckMinionMachineTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @@ -1015,24 +1071,32 @@ def get_required_platform(cls): raise NotImplementedError("No minion power cycle platform specified") @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return _get_required_minion_pool_provider_types_for_platform( cls.get_required_platform() ) @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["minion_provider_properties"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list: return [] @classmethod def _get_minion_power_cycle_op(cls, provider): raise NotImplementedError("No minion power cycle operation implemented.") - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[Any, Any]: platform_to_target = _get_platform_to_target( self.get_required_platform(), origin, destination @@ -1053,35 +1117,35 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class _BasePowerOnMinionTask(_BasePowerCycleMinionTask): @classmethod - def _get_minion_power_cycle_op(cls, provider): + def _get_minion_power_cycle_op(cls, provider) -> Callable: return provider.start_minion class PowerOnSourceMinionTask(_BasePowerOnMinionTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_SOURCE class PowerOnDestinationMinionTask(_BasePowerOnMinionTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION class _BasePowerOffMinionTask(_BasePowerCycleMinionTask): @classmethod - def _get_minion_power_cycle_op(cls, provider): + def _get_minion_power_cycle_op(cls, provider) -> Callable: return provider.shutdown_minion class PowerOffSourceMinionTask(_BasePowerOffMinionTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_SOURCE class PowerOffDestinationMinionTask(_BasePowerOffMinionTask): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION diff --git a/coriolis/tasks/osmorphing_tasks.py b/coriolis/tasks/osmorphing_tasks.py index b8afd8965..96c670aa4 100644 --- a/coriolis/tasks/osmorphing_tasks.py +++ b/coriolis/tasks/osmorphing_tasks.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from oslo_log import log as logging from coriolis import constants, exception, schemas @@ -13,19 +15,19 @@ class OSMorphingTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["osmorphing_info", "osmorphing_connection_info", "user_scripts"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[Any]: return [] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_SOURCE: [ constants.PROVIDER_TYPE_TRANSFER_EXPORT @@ -35,7 +37,15 @@ def get_required_provider_types(cls): ], } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[Any, Any]: origin_provider = providers_factory.get_provider( origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler @@ -82,15 +92,15 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DeployOSMorphingResourcesTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["target_environment", "instance_deployment_info"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return [ "os_morphing_resources", "osmorphing_info", @@ -98,14 +108,22 @@ def get_returned_task_info_properties(cls): ] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_OS_MORPHING ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict: provider = providers_factory.get_provider( destination["type"], constants.PROVIDER_TYPE_OS_MORPHING, event_handler ) @@ -160,26 +178,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DeleteOSMorphingResourcesTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["target_environment", "os_morphing_resources"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["os_morphing_resources", "osmorphing_connection_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_OS_MORPHING ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, None]: provider = providers_factory.get_provider( destination["type"], constants.PROVIDER_TYPE_OS_MORPHING, event_handler ) diff --git a/coriolis/tasks/replica_tasks.py b/coriolis/tasks/replica_tasks.py index 3601f13fb..7eccbbcd4 100644 --- a/coriolis/tasks/replica_tasks.py +++ b/coriolis/tasks/replica_tasks.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from oslo_log import log as logging from coriolis import constants, events, exception, schemas, utils @@ -11,7 +13,7 @@ LOG = logging.getLogger(__name__) -def _get_volumes_info(task_info): +def _get_volumes_info(task_info: dict) -> list: volumes_info = task_info.get("volumes_info", []) if not volumes_info: raise exception.InvalidActionTasksExecutionState( @@ -20,7 +22,7 @@ def _get_volumes_info(task_info): return volumes_info -def _check_ensure_volumes_info_ordering(export_info, volumes_info): +def _check_ensure_volumes_info_ordering(export_info: dict, volumes_info: list) -> list: """Returns a new list of volumes_info, ensuring that the order of the disks in 'volumes_info' is consistent with the order that the disks appear in 'export_info[devices][disks]' @@ -83,7 +85,9 @@ def _check_ensure_volumes_info_ordering(export_info, volumes_info): return ordered_volumes_info -def _update_legacy_disk_ids_in_volumes_info(export_info, volumes_info): +def _update_legacy_disk_ids_in_volumes_info( + export_info: dict, volumes_info: list[dict] +) -> list[dict]: """Migrates volumes_info entries keyed on legacy disk identifiers. Source providers may change their disk identification methodology @@ -140,8 +144,10 @@ def _update_legacy_disk_ids_in_volumes_info(export_info, volumes_info): return volumes_info -def _preserve_old_export_info_nic_ips(old_export_info, new_export_info): - def _get_nic(nics_info, nic_id): +def _preserve_old_export_info_nic_ips( + old_export_info: dict, new_export_info: dict +) -> None: + def _get_nic(nics_info: list, nic_id: str) -> dict | None: for nic_info in nics_info: if nic_info['id'] == nic_id: return nic_info @@ -158,7 +164,7 @@ def _get_nic(nics_info, nic_id): new_info['ip_addresses'] = old_ips -def _preserve_hostname_info(old_export_info, new_export_info): +def _preserve_hostname_info(old_export_info: dict, new_export_info: dict) -> None: old_hostname = old_export_info.get("hostname", "") new_hostname = new_export_info.get("hostname", "") if not new_hostname: @@ -169,7 +175,7 @@ def _preserve_hostname_info(old_export_info, new_export_info): new_export_info['fqdn'] = old_fqdn -def _update_export_info(old_export_info, result_export_info): +def _update_export_info(old_export_info: dict, result_export_info: dict) -> None: _preserve_hostname_info(old_export_info, result_export_info) _preserve_old_export_info_nic_ips(old_export_info, result_export_info) @@ -178,26 +184,34 @@ class GetInstanceInfoTask(base.TaskRunner): """Task which gathers the export info for a VM.""" @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_SOURCE @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["source_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["export_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_SOURCE: [ constants.PROVIDER_TYPE_TRANSFER_EXPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, dict]: provider = providers_factory.get_provider( origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler ) @@ -220,26 +234,34 @@ class ShutdownInstanceTask(base.TaskRunner): """Task which shuts down a VM.""" @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_SOURCE @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["source_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[Any]: return [] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_SOURCE: [ constants.PROVIDER_TYPE_TRANSFER_EXPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[Any, Any]: provider = providers_factory.get_provider( origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler ) @@ -252,14 +274,14 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class ReplicateDisksTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: # NOTE: considering Replication reads from one end (be it PMR minion # or otherwise) to the disk writer minion on the destination, # replicate_disks would need access to both: return constants.TASK_PLATFORM_BILATERAL @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return [ "export_info", "volumes_info", @@ -270,18 +292,26 @@ def get_required_task_info_properties(cls): ] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["volumes_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_SOURCE: [ constants.PROVIDER_TYPE_TRANSFER_EXPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, list]: provider = providers_factory.get_provider( origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler ) @@ -341,26 +371,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DeployReplicaDisksTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["export_info", "volumes_info", "target_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["volumes_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_TRANSFER_IMPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, list]: target_environment = task_info['target_environment'] export_info = task_info["export_info"] @@ -393,26 +431,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DeleteReplicaSourceDiskSnapshotsTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_SOURCE @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["volumes_info", "source_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["volumes_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_SOURCE: [ constants.PROVIDER_TYPE_TRANSFER_EXPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, list]: event_manager = events.EventManager(event_handler) if not task_info.get("volumes_info"): LOG.debug("No volumes_info present. Skipping source snapshot deletion.") @@ -437,26 +483,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DeleteReplicaDisksTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["volumes_info", "target_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["volumes_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_TRANSFER_IMPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, list]: event_manager = events.EventManager(event_handler) if not task_info.get("volumes_info"): LOG.debug("No volumes_info present. Skipping disk deletion.") @@ -488,26 +542,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DeployReplicaSourceResourcesTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_SOURCE @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["source_environment", "export_info"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["source_resources", "source_resources_connection_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_SOURCE: [ constants.PROVIDER_TYPE_TRANSFER_EXPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict: provider = providers_factory.get_provider( origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler ) @@ -565,26 +627,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DeleteReplicaSourceResourcesTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_SOURCE @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["source_environment", "source_resources"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["source_resources", "source_resources_connection_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_SOURCE: [ constants.PROVIDER_TYPE_TRANSFER_EXPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, None]: provider = providers_factory.get_provider( origin["type"], constants.PROVIDER_TYPE_TRANSFER_EXPORT, event_handler ) @@ -605,26 +675,26 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DeployReplicaTargetResourcesTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["export_info", "volumes_info", "target_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["volumes_info", "target_resources", "target_resources_connection_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_TRANSFER_IMPORT ] } - def _validate_connection_info(self, migr_connection_info): + def _validate_connection_info(self, migr_connection_info: dict) -> None: try: backup_writers.BackupWritersFactory(migr_connection_info, None).get_writer() except Exception as err: @@ -648,7 +718,15 @@ def _validate_connection_info(self, migr_connection_info): raise_on_error=False, ) - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict: target_environment = task_info["target_environment"] export_info = task_info['export_info'] @@ -713,26 +791,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DeleteReplicaTargetResourcesTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["target_resources", "target_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["target_resources", "target_resources_connection_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_TRANSFER_IMPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, None]: provider = providers_factory.get_provider( destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler ) @@ -751,26 +837,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DeployReplicaInstanceResourcesTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["export_info", "target_environment", "clone_disks", "volumes_info"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["instance_deployment_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_TRANSFER_IMPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, dict]: target_environment = task_info["target_environment"] export_info = task_info["export_info"] @@ -798,26 +892,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class FinalizeReplicaInstanceDeploymentTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["target_environment", "instance_deployment_info"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["transfer_result"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_TRANSFER_IMPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, dict]: provider = providers_factory.get_provider( destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler ) @@ -844,11 +946,11 @@ def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["target_environment", "instance_deployment_info"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["instance_deployment_info"] @classmethod @@ -876,26 +978,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class CreateReplicaDiskSnapshotsTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["target_environment", "export_info", "volumes_info"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["volumes_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_TRANSFER_IMPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, list]: provider = providers_factory.get_provider( destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler ) @@ -917,26 +1027,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class DeleteReplicaTargetDiskSnapshotsTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["target_environment", "export_info", "volumes_info"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["volumes_info"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_TRANSFER_IMPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[str, list]: export_info = task_info['export_info'] provider = providers_factory.get_provider( destination["type"], constants.PROVIDER_TYPE_TRANSFER_IMPORT, event_handler @@ -962,11 +1080,11 @@ def get_required_platform(cls): return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["target_environment", "export_info", "volumes_info"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["volumes_info"] @classmethod @@ -999,26 +1117,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class ValidateReplicaExecutionSourceInputsTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_SOURCE @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["source_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[Any]: return [] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_SOURCE: [ constants.PROVIDER_TYPE_VALIDATE_TRANSFER_EXPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[Any, Any]: event_manager = events.EventManager(event_handler) origin_type = origin["type"] source_provider = providers_factory.get_provider( @@ -1046,19 +1172,19 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class ValidateReplicaExecutionDestinationInputsTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["export_info", "target_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[Any]: return [] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT @@ -1066,8 +1192,13 @@ def get_required_provider_types(cls): } def _validate_provider_replica_import_input( - self, provider, ctxt, conn_info, target_environment, export_info - ): + self, + provider, + ctxt, + conn_info: dict[str, str], + target_environment: dict, + export_info: dict, + ) -> None: provider.validate_replica_import_input( ctxt, conn_info, @@ -1077,7 +1208,15 @@ def _validate_provider_replica_import_input( check_final_vm_params=False, ) - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[Any, Any]: event_manager = events.EventManager(event_handler) destination_type = destination["type"] @@ -1128,26 +1267,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class ValidateReplicaDeploymentParametersTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["export_info", "target_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[Any]: return [] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict[Any, Any]: event_manager = events.EventManager(event_handler) destination_connection_info = base.get_connection_info(ctxt, destination) destination_type = destination["type"] @@ -1181,26 +1328,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class UpdateSourceReplicaTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_SOURCE @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["volumes_info", "source_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["volumes_info", "source_environment"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_SOURCE: [ constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict: event_manager = events.EventManager(event_handler) volumes_info = task_info.get("volumes_info", []) @@ -1251,26 +1406,34 @@ def _run(self, ctxt, instance, origin, destination, task_info, event_handler): class UpdateDestinationReplicaTask(base.TaskRunner): @classmethod - def get_required_platform(cls): + def get_required_platform(cls) -> str: return constants.TASK_PLATFORM_DESTINATION @classmethod - def get_required_task_info_properties(cls): + def get_required_task_info_properties(cls) -> list[str]: return ["export_info", "volumes_info", "target_environment"] @classmethod - def get_returned_task_info_properties(cls): + def get_returned_task_info_properties(cls) -> list[str]: return ["volumes_info", "target_environment"] @classmethod - def get_required_provider_types(cls): + def get_required_provider_types(cls) -> dict[str, list]: return { constants.PROVIDER_PLATFORM_DESTINATION: [ constants.PROVIDER_TYPE_DESTINATION_TRANSFER_UPDATE ] } - def _run(self, ctxt, instance, origin, destination, task_info, event_handler): + def _run( + self, + ctxt, + instance: str, + origin: dict, + destination: dict, + task_info: dict, + event_handler, + ) -> dict: event_manager = events.EventManager(event_handler) volumes_info = task_info.get("volumes_info", []) diff --git a/coriolis/transfer_cron/api.py b/coriolis/transfer_cron/api.py index 8db9a4cc3..b45c31f65 100644 --- a/coriolis/transfer_cron/api.py +++ b/coriolis/transfer_cron/api.py @@ -1,6 +1,7 @@ # Copyright 2017 Cloudbase Solutions Srl # All Rights Reserved. + from coriolis.conductor.rpc import client as rpc_client @@ -11,12 +12,12 @@ def __init__(self): def create( self, ctxt, - transfer_id, - schedule, - enabled, + transfer_id: str, + schedule: dict[str, int], + enabled: bool, exp_date, - shutdown_instance, - auto_deploy, + shutdown_instance: bool, + auto_deploy: bool, ): return self._rpc_client.create_transfer_schedule( ctxt, @@ -28,20 +29,42 @@ def create( auto_deploy, ) - def get_schedules(self, ctxt, transfer_id, expired=True): + def get_schedules( + self, + ctxt, + transfer_id: str, + expired: bool = True, + ): return self._rpc_client.get_transfer_schedules( ctxt, transfer_id, expired=expired ) - def get_schedule(self, ctxt, transfer_id, schedule_id, expired=True): + def get_schedule( + self, + ctxt, + transfer_id: str, + schedule_id: str, + expired: bool = True, + ): return self._rpc_client.get_transfer_schedule( ctxt, transfer_id, schedule_id, expired=expired ) - def update(self, ctxt, transfer_id, schedule_id, update_values): + def update( + self, + ctxt, + transfer_id: str, + schedule_id: str, + update_values: dict, + ): return self._rpc_client.update_transfer_schedule( ctxt, transfer_id, schedule_id, update_values ) - def delete(self, ctxt, transfer_id, schedule_id): + def delete( + self, + ctxt, + transfer_id: str, + schedule_id: str, + ) -> None: self._rpc_client.delete_transfer_schedule(ctxt, transfer_id, schedule_id) diff --git a/coriolis/transfer_cron/rpc/client.py b/coriolis/transfer_cron/rpc/client.py index 4e49dabaf..501053912 100644 --- a/coriolis/transfer_cron/rpc/client.py +++ b/coriolis/transfer_cron/rpc/client.py @@ -9,15 +9,23 @@ class TransferCronClient(rpc.BaseRPCClient): - def __init__(self, topic=constants.TRANSFER_CRON_MAIN_MESSAGING_TOPIC): + def __init__(self, topic: str = constants.TRANSFER_CRON_MAIN_MESSAGING_TOPIC): target = messaging.Target(topic=topic, version=VERSION) super(TransferCronClient, self).__init__(target) - def register(self, ctxt, schedule): + def register( + self, + ctxt, + schedule, + ) -> None: self._call(ctxt, 'register', schedule=schedule) - def unregister(self, ctxt, schedule): + def unregister( + self, + ctxt, + schedule, + ) -> None: self._call(ctxt, 'unregister', schedule=schedule) - def get_diagnostics(self, ctxt): + def get_diagnostics(self, ctxt) -> dict: return self._call(ctxt, 'get_diagnostics') diff --git a/coriolis/transfer_cron/rpc/server.py b/coriolis/transfer_cron/rpc/server.py index bc0ef886c..c28ca3fde 100644 --- a/coriolis/transfer_cron/rpc/server.py +++ b/coriolis/transfer_cron/rpc/server.py @@ -19,8 +19,12 @@ def _trigger_transfer( - ctxt, conductor_client, transfer_id, shutdown_instance, auto_deploy -): + ctxt: context.RequestContext, + conductor_client: rpc_client.ConductorClient, + transfer_id: str, + shutdown_instance: bool, + auto_deploy: bool, +) -> str | None: try: execution = conductor_client.execute_transfer_tasks( ctxt, @@ -59,7 +63,7 @@ def __init__(self): # leave this process up with the schedule loop never started. os.register_at_fork(after_in_child=self._ensure_cron_started) - def _ensure_cron_started(self): + def _ensure_cron_started(self) -> None: try: with self._cron_lock: if self._cron_started or self._loader_started: @@ -72,7 +76,7 @@ def _ensure_cron_started(self): if not self._cron_started: self._loader_started = False - def _load_schedules_until_started(self): + def _load_schedules_until_started(self) -> None: delay = 60 while True: try: @@ -90,7 +94,7 @@ def _load_schedules_until_started(self): LOG.info("Transfer schedules loaded, cron loop started") return - def _start_cron_loop(self): + def _start_cron_loop(self) -> None: with self._cron_lock: if self._cron_loop_started: return @@ -108,7 +112,11 @@ def _deserialize_schedule(self, sched): sched["schedule"] = json.loads(tmp) return sched - def _register_schedule(self, schedule, date=None): + def _register_schedule( + self, + schedule, + date=None, + ) -> None: date = date or timeutils.utcnow() sched = self._deserialize_schedule(schedule) expires = sched.get("expiration_date") @@ -134,7 +142,7 @@ def _register_schedule(self, schedule, date=None): ) self._cron.register(job) - def _init_cron(self): + def _init_cron(self) -> None: now = timeutils.utcnow() schedules = self._get_all_schedules() for schedule in schedules: @@ -148,13 +156,15 @@ def _init_cron(self): LOG.exception(err) self._start_cron_loop() - def _get_all_schedules(self): + def _get_all_schedules( + self, + ): schedules = self._rpc_client.get_transfer_schedules( self._admin_ctx, expired=False ) return schedules - def register(self, ctxt, schedule): + def register(self, ctxt: context.RequestContext, schedule: dict) -> None: self._ensure_cron_started() now = timeutils.utcnow() LOG.debug( @@ -162,11 +172,11 @@ def register(self, ctxt, schedule): ) self._register_schedule(schedule, date=now) - def unregister(self, ctxt, schedule): + def unregister(self, ctxt: context.RequestContext, schedule: dict) -> None: self._ensure_cron_started() schedule_id = schedule["id"] LOG.debug("removing schedule %s" % schedule_id) self._cron.unregister(schedule_id) - def get_diagnostics(self, ctxt): + def get_diagnostics(self, ctxt: context.RequestContext) -> dict: return utils.get_diagnostics_info() diff --git a/coriolis/transfer_tasks_executions/api.py b/coriolis/transfer_tasks_executions/api.py index 79e4933fe..a552b5a5e 100644 --- a/coriolis/transfer_tasks_executions/api.py +++ b/coriolis/transfer_tasks_executions/api.py @@ -8,17 +8,34 @@ class API(object): def __init__(self): self._rpc_client = rpc_client.ConductorClient() - def create(self, ctxt, transfer_id, shutdown_instances, auto_deploy): + def create( + self, + ctxt, + transfer_id: str, + shutdown_instances: bool, + auto_deploy: bool, + ) -> dict: return self._rpc_client.execute_transfer_tasks( ctxt, transfer_id, shutdown_instances, auto_deploy ) - def delete(self, ctxt, transfer_id, execution_id): + def delete( + self, + ctxt, + transfer_id: str, + execution_id: str, + ) -> None: self._rpc_client.delete_transfer_tasks_execution( ctxt, transfer_id, execution_id ) - def cancel(self, ctxt, transfer_id, execution_id, force): + def cancel( + self, + ctxt, + transfer_id: str, + execution_id: str, + force: bool, + ) -> None: self._rpc_client.cancel_transfer_tasks_execution( ctxt, transfer_id, execution_id, force ) @@ -26,14 +43,14 @@ def cancel(self, ctxt, transfer_id, execution_id, force): def get_executions( self, ctxt, - transfer_id, - include_tasks=False, - marker=None, - limit=None, - sort_keys=None, - sort_dirs=None, - filters=None, - ): + transfer_id: str, + include_tasks: bool = False, + marker: str | None = None, + limit: int | None = None, + sort_keys: list | None = None, + sort_dirs: list | None = None, + filters: dict | None = None, + ) -> list: return self._rpc_client.get_transfer_tasks_executions( ctxt, transfer_id, @@ -45,7 +62,12 @@ def get_executions( filters, ) - def get_execution(self, ctxt, transfer_id, execution_id): + def get_execution( + self, + ctxt, + transfer_id: str, + execution_id: str, + ) -> dict: return self._rpc_client.get_transfer_tasks_execution( ctxt, transfer_id, execution_id ) diff --git a/coriolis/transfers/api.py b/coriolis/transfers/api.py index 4678ce848..fc8be0b3d 100644 --- a/coriolis/transfers/api.py +++ b/coriolis/transfers/api.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + from coriolis.conductor.rpc import client as rpc_client @@ -11,22 +13,22 @@ def __init__(self): def create( self, ctxt, - transfer_scenario, - origin_endpoint_id, - destination_endpoint_id, - origin_minion_pool_id, - destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings, - source_environment, - destination_environment, - instances, - network_map, - storage_mappings, - notes=None, + transfer_scenario: str, + origin_endpoint_id: str, + destination_endpoint_id: str, + origin_minion_pool_id: str | None, + destination_minion_pool_id: str | None, + instance_osmorphing_minion_pool_mappings: dict[Any, Any], + source_environment: dict, + destination_environment: dict[str, dict], + instances: list[str], + network_map: dict[Any, Any], + storage_mappings: dict[Any, Any], + notes: str | None = None, user_scripts=None, - clone_disks=True, - skip_os_morphing=False, - ): + clone_disks: bool = True, + skip_os_morphing: bool = False, + ) -> dict: return self._rpc_client.create_instances_transfer( ctxt, transfer_scenario, @@ -46,23 +48,28 @@ def create( skip_os_morphing, ) - def update(self, ctxt, transfer_id, updated_properties): + def update( + self, + ctxt, + transfer_id: str, + updated_properties: dict, + ) -> dict: return self._rpc_client.update_transfer(ctxt, transfer_id, updated_properties) - def delete(self, ctxt, transfer_id): + def delete(self, ctxt, transfer_id: str) -> None: self._rpc_client.delete_transfer(ctxt, transfer_id) def get_transfers( self, ctxt, - include_tasks_executions=False, - include_task_info=False, - marker=None, - limit=None, - sort_keys=None, - sort_dirs=None, - filters=None, - ): + include_tasks_executions: bool = False, + include_task_info: bool = False, + marker: str | None = None, + limit: int | None = None, + sort_keys: list | None = None, + sort_dirs: list | None = None, + filters: dict | None = None, + ) -> list[dict]: return self._rpc_client.get_transfers( ctxt, include_tasks_executions, @@ -74,10 +81,15 @@ def get_transfers( filters=filters, ) - def get_transfer(self, ctxt, transfer_id, include_task_info=False): + def get_transfer( + self, + ctxt, + transfer_id: str, + include_task_info: bool = False, + ) -> dict: return self._rpc_client.get_transfer( ctxt, transfer_id, include_task_info=include_task_info ) - def delete_disks(self, ctxt, transfer_id): + def delete_disks(self, ctxt, transfer_id: str) -> dict: return self._rpc_client.delete_transfer_disks(ctxt, transfer_id) diff --git a/coriolis/utils.py b/coriolis/utils.py index 7ee7c821a..643d4ef0f 100644 --- a/coriolis/utils.py +++ b/coriolis/utils.py @@ -22,6 +22,7 @@ import traceback import uuid from io import StringIO +from typing import Any, Callable, Iterator, Sequence import netifaces import OpenSSL @@ -93,7 +94,7 @@ """ -def _get_local_ips(): +def _get_local_ips() -> list: ifaces = netifaces.interfaces() ret = [] for iface in ifaces: @@ -111,7 +112,7 @@ def _get_local_ips(): return ret -def _get_host_os_info(): +def _get_host_os_info() -> list[str]: info = None # This exists on all modern Linux systems if os.path.isfile("/etc/os-release"): @@ -120,7 +121,7 @@ def _get_host_os_info(): return info -def _get_release_tag(): +def _get_release_tag() -> str: release_tag = "latest" if os.path.isfile("/etc/coriolis/coriolis.release"): with open("/etc/coriolis/coriolis.release") as fd: @@ -128,7 +129,7 @@ def _get_release_tag(): return release_tag -def _filesystem_used_percentage(used, available): +def _filesystem_used_percentage(used: int, available: int) -> int: """Return the used-space percentage, rounded up. ``available`` is space a normal user can still use. The percentage is @@ -140,7 +141,7 @@ def _filesystem_used_percentage(used, available): return (used * 100 + total - 1) // total -def _get_filesystems(): +def _get_filesystems() -> list[dict]: """Return usage for mounted block devices, in bytes. The mount list is ``psutil.disk_partitions(all=False)``: devices with a @@ -177,7 +178,7 @@ def _get_filesystems(): return filesystems -def _get_memory(): +def _get_memory() -> dict: """Return memory usage in bytes. Values come from ``psutil``, which reads ``/proc/meminfo``. ``used`` and @@ -203,7 +204,7 @@ def _get_memory(): } -def _get_cpu_usage(): +def _get_cpu_usage() -> list[dict]: """Return recent CPU usage percentage for each core. The list is ordered by core index, matching ``psutil.cpu_percent``. @@ -212,7 +213,7 @@ def _get_cpu_usage(): return [{"core": core, "percent": percent} for core, percent in enumerate(percents)] -def get_diagnostics_info(): +def get_diagnostics_info() -> dict: """Return diagnostic details for this Coriolis service process. Host resource fields: @@ -234,7 +235,7 @@ def get_diagnostics_info(): } -def setup_logging(disable_file_handlers=False): +def setup_logging(disable_file_handlers: bool = False) -> None: """Configure oslo logging for Coriolis. :param disable_file_handlers: When True, clear log_dir/log_file and disable @@ -252,9 +253,9 @@ def setup_logging(disable_file_handlers=False): logging.setup(CONF, 'coriolis') -def ignore_exceptions(func): +def ignore_exceptions(func: Callable) -> Callable: @functools.wraps(func) - def _ignore_exceptions(*args, **kwargs): + def _ignore_exceptions(*args, **kwargs) -> None: try: return func(*args, **kwargs) except Exception: @@ -277,12 +278,12 @@ def get_single_result(lis): def retry_on_error( - max_attempts=5, - sleep_seconds=1, - terminal_exceptions=[], - retried_exceptions=(Exception,), -): - def _retry_on_error(func): + max_attempts: int = 5, + sleep_seconds: int = 1, + terminal_exceptions: Sequence[type[exception.MinionMachineCommandTimeout]] = [], + retried_exceptions: tuple = (Exception,), +) -> Callable: + def _retry_on_error(func: Callable) -> Callable: @functools.wraps(func) def _exec_retry(*args, **kwargs): i = 0 @@ -325,7 +326,7 @@ def get_udev_net_rules(net_ifaces_info): return content -def parse_os_release(ssh): +def parse_os_release(ssh: paramiko.SSHClient) -> tuple[str, str]: os_release_info = exec_ssh_cmd( ssh, "[ -f '/etc/os-release' ] && cat /etc/os-release || true" ) @@ -349,7 +350,7 @@ def parse_lsb_release(ssh): return (dist_id[0], release[0]) -def get_linux_os_info(ssh): +def get_linux_os_info(ssh: paramiko.SSHClient) -> tuple[str, str]: info = parse_os_release(ssh) if info is None: # Fall back to lsb_release @@ -358,7 +359,10 @@ def get_linux_os_info(ssh): @retry_on_error() -def test_ssh_path(ssh, remote_path): +def test_ssh_path( + ssh, + remote_path, +) -> bool | bytes | paramiko.SSHClient | None: sftp = ssh.open_sftp() try: sftp.stat(remote_path) @@ -370,13 +374,17 @@ def test_ssh_path(ssh, remote_path): @retry_on_error() -def read_ssh_file(ssh, remote_path): +def read_ssh_file(ssh: paramiko.SSHClient, remote_path: str) -> bytes: sftp = ssh.open_sftp() return sftp.open(remote_path, 'rb').read() @retry_on_error() -def write_ssh_file(ssh, remote_path, content): +def write_ssh_file( + ssh, + remote_path, + content, +) -> bool | bytes | list | paramiko.SSHClient | None: sftp = ssh.open_sftp() fd = sftp.open(remote_path, 'wb') # Enabling pipelined transfers here will make @@ -418,7 +426,7 @@ def write_winrm_file(conn, remote_path, content, overwrite=True): @retry_on_error() -def list_ssh_dir(ssh, remote_path): +def list_ssh_dir(ssh: paramiko.SSHClient, remote_path: str) -> list[str]: """List directory contents via SFTP, falling back to ls on encoding errors.""" try: @@ -433,7 +441,7 @@ def list_ssh_dir(ssh, remote_path): return [f for f in output.splitlines() if f.strip()] -def get_env_command_prefix(environment): +def get_env_command_prefix(environment: dict[str, str]) -> str: """ Each variable is shell-quoted individually, so values containing spaces or other shell-sensitive characters are passed through @@ -447,7 +455,7 @@ def get_env_command_prefix(environment): ) -def check_env_command(ssh): +def check_env_command(ssh: paramiko.SSHClient) -> None: """ Checks that env(1) exists on the given machine, as it is what carries the environment variables over to privileged and chrooted commands. @@ -467,7 +475,13 @@ def check_env_command(ssh): ) from ex -def _exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False, timeout=None): +def _exec_ssh_cmd( + ssh: paramiko.SSHClient, + cmd: str, + environment: dict | None = None, + get_pty: bool = False, + timeout=None, +) -> str: sanitized_cmd = strutils.mask_password(cmd) remote_str = "" if timeout is not None: @@ -520,19 +534,19 @@ def _exec_ssh_cmd(ssh, cmd, environment=None, get_pty=False, timeout=None): def exec_ssh_cmd( - ssh, - cmd, - environment=None, - get_pty=False, + ssh: paramiko.SSHClient, + cmd: str, + environment: dict | None = None, + get_pty: bool = False, timeout=None, - max_attempts=5, - retry_interval=1, - terminal_exceptions=( + max_attempts: int = 5, + retry_interval: int = 1, + terminal_exceptions: tuple = ( exception.MinionMachineCommandTimeout, exception.SSHCommandNotFoundException, ), - retried_exceptions=(paramiko.ssh_exception.SSHException,), -): + retried_exceptions: tuple = (paramiko.ssh_exception.SSHException,), +) -> str: # By default, we'll perform retries only in case of SSH failures. # # Add "SSHCommandFailed" to the list of retried exceptions if failed @@ -544,15 +558,20 @@ def exec_ssh_cmd( terminal_exceptions=terminal_exceptions, retried_exceptions=retried_exceptions, ) - def wrapper(): + def wrapper() -> str: return _exec_ssh_cmd(ssh, cmd, environment, get_pty, timeout) return wrapper() def exec_ssh_cmd_chroot( - ssh, chroot_dir, cmd, environment=None, get_pty=False, timeout=None -): + ssh: paramiko.SSHClient, + chroot_dir: str, + cmd: str, + environment: dict | None = None, + get_pty: bool = False, + timeout=None, +) -> str: return exec_ssh_cmd( ssh, "sudo %schroot %s %s" @@ -563,7 +582,7 @@ def exec_ssh_cmd_chroot( ) -def check_fs(ssh, fs_type, dev_path): +def check_fs(ssh: paramiko.SSHClient, fs_type: str, dev_path: str) -> None: try: out = exec_ssh_cmd( ssh, "sudo fsck -p -t %s %s" % (fs_type, dev_path), get_pty=False @@ -593,7 +612,7 @@ def run_xfs_repair(ssh, dev_path): LOG.warn("xfs_repair returned an error:\n%s", str(ex)) -def _check_port_open(host, port): +def _check_port_open(host: str, port: int) -> bool: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.settimeout(1) @@ -605,7 +624,7 @@ def _check_port_open(host, port): s.close() -def wait_for_port_connectivity(address, port, max_wait=300): +def wait_for_port_connectivity(address: str, port: int, max_wait: int = 300) -> None: i = 0 while not _check_port_open(address, port) and i < max_wait: time.sleep(1) @@ -614,7 +633,7 @@ def wait_for_port_connectivity(address, port, max_wait=300): raise exception.CoriolisException("Connection failed on port %s" % port) -def exec_process(args): +def exec_process(args: list[str]) -> bytes: p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) std_out, std_err = p.communicate() if p.returncode: @@ -661,19 +680,21 @@ def convert_disk_format(disk_path, target_disk_path, target_format, preallocated raise -def get_hostname(): +def get_hostname() -> str: return socket.gethostname() -def get_binary_name(): +def get_binary_name() -> str: return os.path.basename(sys.argv[0]) -def get_exception_details(): +def get_exception_details() -> str: return traceback.format_exc() -def walk_class_hierarchy(clazz, encountered=None): +def walk_class_hierarchy( + clazz: type[exc.HTTPClientError], encountered: list | None = None +) -> Iterator[None]: """Walk class hierarchy, yielding most derived classes first.""" if not encountered: encountered = [] @@ -698,34 +719,34 @@ def get_ssl_cert_thumbprint(context, host, port=443, digest_algorithm="sha1"): return x509.digest(digest_algorithm).decode() -def get_resources_dir(): +def get_resources_dir() -> str: return os.path.join(os.path.dirname(os.path.abspath(__file__)), "resources") -def get_resources_bin_dir(): +def get_resources_bin_dir() -> str: return os.path.join(get_resources_dir(), "bin") -def serialize_key(key, password=None): +def serialize_key(key: paramiko.RSAKey, password=None) -> str: key_io = io.StringIO() key.write_private_key(key_io, password) return key_io.getvalue() -def deserialize_key(key_bytes, password=None): +def deserialize_key(key_bytes: str, password=None) -> paramiko.RSAKey: key_io = io.StringIO(key_bytes) return paramiko.RSAKey.from_private_key(key_io, password) def connect_ssh( - hostname, - port, - username, - pkey=None, + hostname: str, + port: int, + username: str, + pkey: paramiko.RSAKey | None = None, password=None, - connect_timeout=None, - banner_timeout=None, -): + connect_timeout: int | None = None, + banner_timeout: int | None = None, +) -> paramiko.SSHClient: """Open and return a connected paramiko SSHClient. :param pkey: a paramiko.PKey instance, a serialized PEM string, or None. @@ -760,7 +781,7 @@ def connect_ssh( return ssh -def is_serializable(obj): +def is_serializable(obj: dict) -> None: pickle.dumps(obj) @@ -780,7 +801,7 @@ def _to_primitive( return jsonutils.loads(jsonutils.dumps(obj, default=_to_primitive)) -def load_class(class_path): +def load_class(class_path: str) -> type[Any]: LOG.debug('Loading class \'%s\'' % class_path) parts = class_path.rsplit('.', 1) module = __import__(parts[0], fromlist=parts[1]) @@ -795,7 +816,7 @@ def check_md5(data, md5): raise exception.CoriolisException("MD5 check failed") -def get_secret_connection_info(ctxt, connection_info): +def get_secret_connection_info(ctxt, connection_info: dict[str, str]) -> dict[str, str]: secret_ref = connection_info.get("secret_ref") if secret_ref: LOG.info("Retrieving connection info from secret: %s", secret_ref) @@ -803,14 +824,14 @@ def get_secret_connection_info(ctxt, connection_info): return connection_info -def parse_int_value(value): +def parse_int_value(value: str) -> int: try: return int(str(value)) except ValueError: raise exception.InvalidInput("Invalid integer: %s" % value) -def decode_base64_param(value, is_json=False): +def decode_base64_param(value: str, is_json: bool = False) -> dict | str: try: decoded = base64.urlsafe_b64decode(value).decode() if is_json: @@ -895,9 +916,11 @@ def get_unique_option_ids(resources, id_key="id", name_key="name"): return identifiers -def bad_request_on_error(error_message): - def _bad_request_on_error(func): - def wrapper(*args, **kwargs): +def bad_request_on_error(error_message: str) -> Callable[[Callable], Callable]: + def _bad_request_on_error( + func, + ): + def wrapper(*args, **kwargs) -> tuple[bool, None]: (is_valid, message) = func(*args, **kwargs) if not is_valid: raise exc.HTTPBadRequest(explanation=(error_message % message)) @@ -908,7 +931,7 @@ def wrapper(*args, **kwargs): return _bad_request_on_error -def sanitize_task_info(task_info): +def sanitize_task_info(task_info: dict) -> dict: """Returns a copy of the given task info with any chunking info for volumes and sensitive credentials removed. """ @@ -948,7 +971,7 @@ def sanitize_task_info(task_info): return new -def parse_ini_config(file_contents): +def parse_ini_config(file_contents: str) -> dict[str, str]: """Parses the contents of the given .ini config file and returns a dict with the options/values within it. """ @@ -962,7 +985,9 @@ def parse_ini_config(file_contents): return config -def read_ssh_ini_config_file(ssh, path, check_exists=False): +def read_ssh_ini_config_file( + ssh: paramiko.SSHClient, path: str, check_exists: bool = False +) -> dict[str, str]: """Reads and parses the contents of an .ini file at the given path.""" if not check_exists or test_ssh_path(ssh, path): content = read_ssh_file(ssh, path).decode() @@ -971,14 +996,20 @@ def read_ssh_ini_config_file(ssh, path, check_exists=False): return {} -def _get_systemd_unit_path(ssh): +def _get_systemd_unit_path(ssh: paramiko.SSHClient) -> str: """Returns the systemd unit directory path on the remote system.""" if test_ssh_path(ssh, "/lib/systemd/system"): return "/lib/systemd/system" return "/usr/lib/systemd/system" -def _write_systemd(ssh, cmdline, svcname, run_as=None, start=True): +def _write_systemd( + ssh: paramiko.SSHClient, + cmdline: str, + svcname: str, + run_as: str | None = None, + start: bool = True, +) -> None: systemd_unit_dir = _get_systemd_unit_path(ssh) serviceFilePath = "%s/%s.service" % (systemd_unit_dir, svcname) @@ -988,7 +1019,7 @@ def _write_systemd(ssh, cmdline, svcname, run_as=None, start=True): exec_ssh_cmd(ssh, "sudo systemctl start %s" % svcname, get_pty=False) return - def _reload_and_start(start=True): + def _reload_and_start(start: bool = True) -> None: exec_ssh_cmd(ssh, "sudo systemctl daemon-reload", get_pty=False) # NOTE: the service must be enabled so that it comes back up on its own after # the underlying instance is rebooted or power-cycled (e.g.: minion pool @@ -997,7 +1028,7 @@ def _reload_and_start(start=True): if start: exec_ssh_cmd(ssh, "sudo systemctl start %s" % svcname, get_pty=False) - def _correct_selinux_label(): + def _correct_selinux_label() -> None: cmd = "sudo restorecon -v %s" % serviceFilePath try: exec_ssh_cmd(ssh, cmd, get_pty=False) @@ -1050,14 +1081,20 @@ def _write_upstart(ssh, cmdline, svcname, run_as=None, start=True): exec_ssh_cmd(ssh, "start %s" % svcname) -def _has_systemd(ssh): +def _has_systemd(ssh: paramiko.SSHClient) -> bool: """Check if the remote system uses systemd as its init system.""" return test_ssh_path(ssh, "/lib/systemd/system") or test_ssh_path( ssh, "/usr/lib/systemd/system" ) -def create_service(ssh, cmdline, svcname, run_as=None, start=True): +def create_service( + ssh: paramiko.SSHClient, + cmdline: str, + svcname: str, + run_as: str | None = None, + start: bool = True, +) -> None: # Simplistic check for init system. We usually use official images, # and none of the supported operating systems come with both upstart # and systemd installed side by side. So if /lib/systemd/system @@ -1071,7 +1108,7 @@ def create_service(ssh, cmdline, svcname, run_as=None, start=True): raise exception.CoriolisException("could not determine init system") -def restart_service(ssh, svcname): +def restart_service(ssh: paramiko.SSHClient, svcname: str) -> None: if _has_systemd(ssh): exec_ssh_cmd(ssh, "sudo systemctl restart %s" % svcname, get_pty=False) elif test_ssh_path(ssh, "/etc/init"): @@ -1323,7 +1360,12 @@ def dump(self): return tmp.read() -def start_thread(target, args=(), kwargs=None, daemon=True): +def start_thread( + target: Callable, + args: list | tuple = (), + kwargs: dict | None = None, + daemon: bool = True, +): """Convenience helper for one-liner thread spawning.""" thread = threading.Thread( target=target, diff --git a/coriolis/worker/rpc/client.py b/coriolis/worker/rpc/client.py index db9ba3a78..0f83ced21 100644 --- a/coriolis/worker/rpc/client.py +++ b/coriolis/worker/rpc/client.py @@ -1,6 +1,8 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +from typing import Any + import oslo_messaging as messaging from oslo_config import cfg @@ -23,9 +25,9 @@ class WorkerClient(rpc.BaseRPCClient): def __init__( self, - timeout=None, - host=None, - base_worker_topic=constants.WORKER_MAIN_MESSAGING_TOPIC, + timeout: int | None = None, + host: str | None = None, + base_worker_topic: str = constants.WORKER_MAIN_MESSAGING_TOPIC, ): topic = base_worker_topic if host is not None: @@ -38,7 +40,12 @@ def __init__( super(WorkerClient, self).__init__(target, timeout=timeout) @classmethod - def from_service_definition(cls, service, timeout=None, topic_override=None): + def from_service_definition( + cls, + service, + timeout: int | None = None, + topic_override=None, + ) -> "WorkerClient": if service.get('topic') != constants.WORKER_MAIN_MESSAGING_TOPIC: raise ValueError( "Unknown topic '%s' for worker service client. Only " @@ -51,8 +58,15 @@ def from_service_definition(cls, service, timeout=None, topic_override=None): return cls(timeout=timeout, base_worker_topic=topic, host=service.get('host')) def begin_task( - self, ctxt, task_id, task_type, origin, destination, instance, task_info - ): + self, + ctxt, + task_id: str, + task_type: str, + origin: dict, + destination: dict, + instance: str, + task_info: dict, + ) -> None: self._cast( ctxt, 'exec_task', @@ -66,8 +80,15 @@ def begin_task( ) def run_task( - self, ctxt, task_id, task_type, origin, destination, instance, task_info - ): + self, + ctxt, + task_id: str, + task_type: str, + origin: dict, + destination: dict, + instance: str, + task_info: dict, + ) -> dict: return self._call( ctxt, 'exec_task', @@ -80,7 +101,13 @@ def run_task( report_to_conductor=False, ) - def cancel_task(self, ctxt, task_id, process_id, force): + def cancel_task( + self, + ctxt, + task_id: str, + process_id: int, + force: bool, + ) -> None: return self._call( ctxt, 'cancel_task', task_id=task_id, process_id=process_id, force=force ) @@ -88,14 +115,14 @@ def cancel_task(self, ctxt, task_id, process_id, force): def get_endpoint_instances( self, ctxt, - platform_name, - connection_info, - source_environment, + platform_name: str, + connection_info: dict[str, str], + source_environment: dict[Any, Any], marker=None, limit=None, - instance_name_pattern=None, - refresh=False, - ): + instance_name_pattern: str | None = None, + refresh: bool = False, + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_instances', @@ -109,8 +136,13 @@ def get_endpoint_instances( ) def get_endpoint_instance( - self, ctxt, platform_name, connection_info, source_environment, instance_name - ): + self, + ctxt, + platform_name: str, + connection_info: dict[str, str], + source_environment: dict[Any, Any], + instance_name: str, + ) -> dict: return self._call( ctxt, 'get_endpoint_instance', @@ -121,8 +153,13 @@ def get_endpoint_instance( ) def get_endpoint_destination_options( - self, ctxt, platform_name, connection_info, env, option_names - ): + self, + ctxt, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_destination_options', @@ -133,8 +170,13 @@ def get_endpoint_destination_options( ) def get_endpoint_source_options( - self, ctxt, platform_name, connection_info, env, option_names - ): + self, + ctxt, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_source_options', @@ -144,7 +186,13 @@ def get_endpoint_source_options( option_names=option_names, ) - def get_endpoint_networks(self, ctxt, platform_name, connection_info, env): + def get_endpoint_networks( + self, + ctxt, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_networks', @@ -153,7 +201,12 @@ def get_endpoint_networks(self, ctxt, platform_name, connection_info, env): env=env, ) - def validate_endpoint_connection(self, ctxt, platform_name, connection_info): + def validate_endpoint_connection( + self, + ctxt, + platform_name: str, + connection_info: dict[str, str], + ) -> tuple: return self._call( ctxt, 'validate_endpoint_connection', @@ -161,7 +214,12 @@ def validate_endpoint_connection(self, ctxt, platform_name, connection_info): connection_info=connection_info, ) - def validate_endpoint_target_environment(self, ctxt, platform_name, target_env): + def validate_endpoint_target_environment( + self, + ctxt, + platform_name: str, + target_env: dict, + ) -> tuple[bool, None]: return self._call( ctxt, 'validate_endpoint_target_environment', @@ -169,7 +227,12 @@ def validate_endpoint_target_environment(self, ctxt, platform_name, target_env): target_env=target_env, ) - def validate_endpoint_source_environment(self, ctxt, platform_name, source_env): + def validate_endpoint_source_environment( + self, + ctxt, + platform_name: str, + source_env: dict, + ) -> tuple[bool, None]: return self._call( ctxt, 'validate_endpoint_source_environment', @@ -177,7 +240,13 @@ def validate_endpoint_source_environment(self, ctxt, platform_name, source_env): source_env=source_env, ) - def get_endpoint_storage(self, ctxt, platform_name, connection_info, env): + def get_endpoint_storage( + self, + ctxt, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + ) -> dict[str, list]: return self._call( ctxt, 'get_endpoint_storage', @@ -187,8 +256,12 @@ def get_endpoint_storage(self, ctxt, platform_name, connection_info, env): ) def get_endpoint_inventory_csv( - self, ctxt, platform_name, connection_info, source_environment - ): + self, + ctxt, + platform_name: str, + connection_info: dict[str, str], + source_environment: dict[Any, Any], + ) -> str: return self._call( ctxt, 'get_endpoint_inventory_csv', @@ -197,10 +270,15 @@ def get_endpoint_inventory_csv( source_environment=source_environment, ) - def get_available_providers(self, ctxt): + def get_available_providers(self, ctxt) -> dict[str, dict]: return self._call(ctxt, 'get_available_providers') - def get_provider_schemas(self, ctxt, platform_name, provider_type): + def get_provider_schemas( + self, + ctxt, + platform_name: str, + provider_type: int, + ) -> dict[str, dict]: return self._call( ctxt, 'get_provider_schemas', @@ -208,15 +286,20 @@ def get_provider_schemas(self, ctxt, platform_name, provider_type): provider_type=provider_type, ) - def get_diagnostics(self, ctxt): + def get_diagnostics(self, ctxt) -> dict: return self._call(ctxt, 'get_diagnostics') - def get_service_status(self, ctxt): + def get_service_status(self, ctxt) -> dict: return self._call(ctxt, 'get_service_status') def get_endpoint_source_minion_pool_options( - self, ctxt, platform_name, connection_info, env, option_names - ): + self, + ctxt, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_source_minion_pool_options', @@ -227,8 +310,13 @@ def get_endpoint_source_minion_pool_options( ) def get_endpoint_destination_minion_pool_options( - self, ctxt, platform_name, connection_info, env, option_names - ): + self, + ctxt, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: return self._call( ctxt, 'get_endpoint_destination_minion_pool_options', @@ -239,8 +327,11 @@ def get_endpoint_destination_minion_pool_options( ) def validate_endpoint_source_minion_pool_options( - self, ctxt, platform_name, pool_environment - ): + self, + ctxt, + platform_name: str, + pool_environment: dict[Any, Any], + ) -> tuple[bool, None]: return self._call( ctxt, 'validate_endpoint_source_minion_pool_options', @@ -249,8 +340,11 @@ def validate_endpoint_source_minion_pool_options( ) def validate_endpoint_destination_minion_pool_options( - self, ctxt, platform_name, pool_environment - ): + self, + ctxt, + platform_name: str, + pool_environment: dict[Any, Any], + ) -> tuple[bool, None]: return self._call( ctxt, 'validate_endpoint_destination_minion_pool_options', diff --git a/coriolis/worker/rpc/server.py b/coriolis/worker/rpc/server.py index a7e0230ee..73ecd232e 100644 --- a/coriolis/worker/rpc/server.py +++ b/coriolis/worker/rpc/server.py @@ -8,6 +8,7 @@ import threading import time from logging import handlers +from typing import Any import psutil from oslo_config import cfg @@ -32,7 +33,12 @@ # TODO(aznashwan): parametrize the event handler provided during task execution # to decouple what gets notified from the task running logic itself: -def _get_event_handler_for_task_type(task_type, ctxt, task_object_id): +def _get_event_handler_for_task_type( + task_type: str, ctxt: context.RequestContext, task_object_id: str +) -> ( + rpc_conductor_client.ConductorTaskRpcEventHandler + | rpc_minion_manager_client.MinionManagerPoolRpcEventHandler +): if task_type in constants.MINION_POOL_OPERATIONS_TASKS: return rpc_minion_manager_client.MinionManagerPoolRpcEventHandler( ctxt, task_object_id @@ -48,7 +54,7 @@ def __init__(self): self._lock = threading.Lock() @property - def _rpc_conductor_client(self): + def _rpc_conductor_client(self) -> rpc_conductor_client.ConductorClient: # NOTE(aznashwan): it is unsafe to fork processes with pre-instantiated # oslo_messaging clients as the underlying thread queues will # be invalidated. Considering this class both serves from a "main @@ -91,7 +97,7 @@ def _register_worker_service(self): self._service_registration = service_registration return service_registration - def get_service_status(self, ctxt): + def get_service_status(self, ctxt: context.RequestContext) -> dict: diagnostics = self.get_diagnostics(ctxt) status = { "host": diagnostics["hostname"], @@ -103,7 +109,9 @@ def get_service_status(self, ctxt): return status - def cancel_task(self, ctxt, task_id, process_id, force): + def cancel_task( + self, ctxt: context.RequestContext, task_id: str, process_id: int, force: bool + ) -> None: LOG.debug( "Received request to cancel task '%s' (process %s)", task_id, process_id ) @@ -323,15 +331,15 @@ def _exec_task_process( def exec_task( self, - ctxt, - task_id, - task_type, - origin, - destination, - instance, - task_info, - report_to_conductor=True, - ): + ctxt: context.RequestContext, + task_id: str, + task_type: str, + origin: dict, + destination: dict, + instance: str, + task_info: dict, + report_to_conductor: bool = True, + ) -> dict | None: try: task_result = self._exec_task_process( ctxt, @@ -393,15 +401,15 @@ def exec_task( def get_endpoint_instances( self, - ctxt, - platform_name, - connection_info, - source_environment, + ctxt: context.RequestContext, + platform_name: str, + connection_info: dict[str, str], + source_environment: dict[Any, Any], marker, limit, - instance_name_pattern, - refresh=False, - ): + instance_name_pattern: str | None, + refresh: bool = False, + ) -> list[dict]: export_provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_ENDPOINT_INSTANCES, None ) @@ -425,8 +433,13 @@ def get_endpoint_instances( return instances_info def get_endpoint_instance( - self, ctxt, platform_name, connection_info, source_environment, instance_name - ): + self, + ctxt: context.RequestContext, + platform_name: str, + connection_info: dict[str, str], + source_environment: dict[Any, Any], + instance_name: str, + ) -> dict: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_ENDPOINT_INSTANCES, None ) @@ -442,8 +455,13 @@ def get_endpoint_instance( return instance_info def get_endpoint_destination_options( - self, ctxt, platform_name, connection_info, env, option_names - ): + self, + ctxt: context.RequestContext, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_DESTINATION_ENDPOINT_OPTIONS, @@ -469,8 +487,13 @@ def get_endpoint_destination_options( return options def get_endpoint_source_minion_pool_options( - self, ctxt, platform_name, connection_info, env, option_names - ): + self, + ctxt: context.RequestContext, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_SOURCE_MINION_POOL, @@ -497,8 +520,13 @@ def get_endpoint_source_minion_pool_options( return options def get_endpoint_destination_minion_pool_options( - self, ctxt, platform_name, connection_info, env, option_names - ): + self, + ctxt: context.RequestContext, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_DESTINATION_MINION_POOL, @@ -525,8 +553,13 @@ def get_endpoint_destination_minion_pool_options( return options def get_endpoint_source_options( - self, ctxt, platform_name, connection_info, env, option_names - ): + self, + ctxt: context.RequestContext, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + option_names: dict[Any, Any], + ) -> list[dict]: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_SOURCE_ENDPOINT_OPTIONS, @@ -551,7 +584,13 @@ def get_endpoint_source_options( return options - def get_endpoint_networks(self, ctxt, platform_name, connection_info, env): + def get_endpoint_networks( + self, + ctxt: context.RequestContext, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + ) -> list[dict]: env = env or {} provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_ENDPOINT_NETWORKS, None @@ -565,7 +604,13 @@ def get_endpoint_networks(self, ctxt, platform_name, connection_info, env): return networks_info - def get_endpoint_storage(self, ctxt, platform_name, connection_info, env): + def get_endpoint_storage( + self, + ctxt: context.RequestContext, + platform_name: str, + connection_info: dict[str, str], + env: dict[Any, Any], + ) -> dict[str, list]: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_ENDPOINT_STORAGE, None ) @@ -579,8 +624,12 @@ def get_endpoint_storage(self, ctxt, platform_name, connection_info, env): return storage def get_endpoint_inventory_csv( - self, ctxt, platform_name, connection_info, source_environment - ): + self, + ctxt: context.RequestContext, + platform_name: str, + connection_info: dict[str, str], + source_environment: dict[Any, Any], + ) -> str: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_ENDPOINT_INVENTORY_EXPORT, None ) @@ -596,10 +645,12 @@ def get_endpoint_inventory_csv( ctxt, secret_connection_info, source_environment ) - def get_available_providers(self, ctxt): + def get_available_providers(self, ctxt: context.RequestContext) -> dict[str, dict]: return providers_factory.get_available_providers() - def validate_endpoint_target_environment(self, ctxt, platform_name, target_env): + def validate_endpoint_target_environment( + self, ctxt: context.RequestContext, platform_name: str, target_env: dict + ) -> tuple[bool, None]: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_OS_MORPHING, None ) @@ -616,7 +667,9 @@ def validate_endpoint_target_environment(self, ctxt, platform_name, target_env): return (is_valid, message) - def validate_endpoint_source_environment(self, ctxt, platform_name, source_env): + def validate_endpoint_source_environment( + self, ctxt: context.RequestContext, platform_name: str, source_env: dict + ) -> tuple[bool, None]: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_TRANSFER_EXPORT, None ) @@ -634,8 +687,11 @@ def validate_endpoint_source_environment(self, ctxt, platform_name, source_env): return (is_valid, message) def validate_endpoint_source_minion_pool_options( - self, ctxt, platform_name, pool_environment - ): + self, + ctxt: context.RequestContext, + platform_name: str, + pool_environment: dict[Any, Any], + ) -> tuple[bool, None]: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_SOURCE_MINION_POOL, None ) @@ -653,8 +709,11 @@ def validate_endpoint_source_minion_pool_options( return (is_valid, message) def validate_endpoint_destination_minion_pool_options( - self, ctxt, platform_name, pool_environment - ): + self, + ctxt: context.RequestContext, + platform_name: str, + pool_environment: dict[Any, Any], + ) -> tuple[bool, None]: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_DESTINATION_MINION_POOL, None ) @@ -671,7 +730,12 @@ def validate_endpoint_destination_minion_pool_options( return (is_valid, message) - def validate_endpoint_connection(self, ctxt, platform_name, connection_info): + def validate_endpoint_connection( + self, + ctxt: context.RequestContext, + platform_name: str, + connection_info: dict[str, str], + ) -> tuple: provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_ENDPOINT, None ) @@ -707,7 +771,9 @@ def validate_endpoint_connection(self, ctxt, platform_name, connection_info): return (is_valid, message) - def get_provider_schemas(self, ctxt, platform_name, provider_type): + def get_provider_schemas( + self, ctxt: context.RequestContext, platform_name: str, provider_type: int + ) -> dict[str, dict]: provider = providers_factory.get_provider(platform_name, provider_type, None) schemas = {} @@ -734,7 +800,7 @@ def get_provider_schemas(self, ctxt, platform_name, provider_type): return schemas - def get_diagnostics(self, ctxt): + def get_diagnostics(self, ctxt: context.RequestContext) -> dict: return utils.get_diagnostics_info()